- - name: reportUsage
- value: 'true'
- repoRef:
- name: manifests
- path: common/spartakus
- name: spartakus
- - kustomizeConfig:
- overlays:
- - istio
- repoRef:
- name: manifests
- path: tensorboard
- name: tensorboard
- - kustomizeConfig:
- overlays:
- - application
- repoRef:
- name: manifests
- path: tf-training/tf-job-crds
- name: tf-job-crds
- - kustomizeConfig:
- overlays:
- - application
- repoRef:
- name: manifests
- path: tf-training/tf-job-operator
- name: tf-job-operator
- - kustomizeConfig:
- overlays:
- - application
- repoRef:
- name: manifests
- path: katib/katib-crds
- name: katib-crds
- - kustomizeConfig:
- overlays:
- - application
- - istio
- repoRef:
- name: manifests
- path: katib/katib-controller
- name: katib-controller
- - kustomizeConfig:
- overlays:
- - application
- repoRef:
- name: manifests
- path: pipeline/api-service
- name: api-service
- - kustomizeConfig:
- overlays:
- - application
- parameters:
- - name: minioPvcName
- value: minio-pv-claim
- repoRef:
- name: manifests
- path: pipeline/minio
- name: minio
- - kustomizeConfig:
- overlays:
- - application
- parameters:
- - name: mysqlPvcName
- value: mysql-pv-claim
- repoRef:
- name: manifests
- path: pipeline/mysql
- name: mysql
- - kustomizeConfig:
- overlays:
- - application
- repoRef:
- name: manifests
- path: pipeline/persistent-agent
- name: persistent-agent
- - kustomizeConfig:
- overlays:
- - application
- repoRef:
- name: manifests
- path: pipeline/pipelines-runner
- name: pipelines-runner
- - kustomizeConfig:
- overlays:
- - istio
- - application
- repoRef:
- name: manifests
- path: pipeline/pipelines-ui
- name: pipelines-ui
- - kustomizeConfig:
- overlays:
- - application
- repoRef:
- name: manifests
- path: pipeline/pipelines-viewer
- name: pipelines-viewer
- - kustomizeConfig:
- overlays:
- - application
- repoRef:
- name: manifests
- path: pipeline/scheduledworkflow
- name: scheduledworkflow
- - kustomizeConfig:
- overlays:
- - application
- repoRef:
- name: manifests
- path: pipeline/pipeline-visualization-service
- name: pipeline-visualization-service
- - kustomizeConfig:
- overlays:
- - application
- - istio
- repoRef:
- name: manifests
- path: profiles
- name: profiles
- - kustomizeConfig:
- overlays:
- - application
- repoRef:
- name: manifests
- path: seldon/seldon-core-operator
- name: seldon-core
- - kustomizeConfig:
- overlays:
- - application
- repoRef:
- name: manifests
- path: mpi-job/mpi-operator
- name: mpi-operator
- - kustomizeConfig:
- overlays:
- - application
- parameters:
- - name: clusterName
- value: kubeflow-aws
- repoRef:
- name: manifests
- path: aws/aws-alb-ingress-controller
- name: aws-alb-ingress-controller
- - kustomizeConfig:
- overlays:
- - application
- repoRef:
- name: manifests
- path: aws/nvidia-device-plugin
- name: nvidia-device-plugin
- plugins:
- - kind: KfAwsPlugin
- metadata:
- name: aws
- spec:
- auth:
- basicAuth:
- password:
- name: password
- username: admin
- region: {{ .Region }}
- roles:
- - {{ .NodeInstanceRoleName }}
- repos:
- - name: manifests
- uri: https://github.com/kubeflow/manifests/archive/v1.0.2.tar.gz
- version: v1.0.2
-
-`
-
-// https://www.kubeflow.org/docs/aws/deploy/install-kubeflow/
-func (ts *tester) installKfConfig() error {
- args := []string{
- ts.cfg.EKSConfig.AddOnKubeflow.KfctlPath,
- "apply",
- "--verbose",
- "--file=" + ts.cfg.EKSConfig.AddOnKubeflow.KfctlConfigPath,
- }
- cmdTxt := strings.Join(args, " ")
-
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
- defer cancel()
- cmd := osexec.CommandContext(ctx, args[0], args[1:]...)
- cmd.Dir = ts.cfg.EKSConfig.AddOnKubeflow.KfDir
- cmd.Env = []string{
- "KUBECONFIG=" + ts.cfg.EKSConfig.KubeConfigPath,
- "AWS_CLUSTER_NAME=" + ts.cfg.EKSConfig.Name,
- "KF_NAME=" + ts.cfg.EKSConfig.Name,
- "BASE_DIR=" + ts.cfg.EKSConfig.AddOnKubeflow.BaseDir,
- "KF_DIR=" + ts.cfg.EKSConfig.AddOnKubeflow.KfDir,
- "CONFIG_FILE=" + ts.cfg.EKSConfig.AddOnKubeflow.KfctlConfigPath,
- }
- for _, ev := range cmd.Env {
- ss := strings.Split(ev, "=")
- os.Setenv(ss[0], ss[1])
- defer os.Unsetenv(ss[0])
- }
-
- pwd, _ := os.Getwd()
- defer func() {
- err := os.Chdir(pwd)
- ts.cfg.Logger.Info("chdir", zap.String("dir", pwd), zap.Error(err))
- }()
- if err := os.Chdir(ts.cfg.EKSConfig.AddOnKubeflow.KfDir); err != nil {
- ts.cfg.Logger.Warn("chdir failed", zap.String("dir", ts.cfg.EKSConfig.AddOnKubeflow.KfDir), zap.Error(err))
- return err
- }
- ts.cfg.Logger.Info("chdir", zap.String("dir", ts.cfg.EKSConfig.AddOnKubeflow.KfDir))
-
- ts.cfg.Logger.Info("kfctl applying", zap.String("command", strings.Join(args, " ")))
- output, err := cmd.Output()
- if err != nil {
- // not working...
- // e.g. Definitions:apiextensions.JSONSchemaDefinitions(nil), ExternalDocs:(*apiextensions.ExternalDocumentation)(nil), Example:(*apiextensions.JSON)(nil)}: must only have "properties", "required" or "description" at the root if the status subresource is enabled] filename="kustomize/kustomize.go:202"
- ts.cfg.Logger.Warn("kfctl apply failed", zap.String("command", strings.Join(args, " ")), zap.Error(err))
- }
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n'%s' (env %q) output:\n\n%s\n\n", cmdTxt, cmd.Env, out)
-
- if err != nil {
- // TODO: fix
- fmt.Fprintf(ts.cfg.LogWriter, "kfctl apply failed... try yourself...")
- fmt.Fprintln(ts.cfg.LogWriter, "1. install aws-iam-authenticator")
- fmt.Fprintln(ts.cfg.LogWriter, "2. install eksctl")
- fmt.Fprintln(ts.cfg.LogWriter, "3. run following")
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n%s\n\n%s\n\n", strings.Join(cmd.Env, "\n"), cmdTxt)
- }
- return nil
-}
diff --git a/eks/kubeflow/kubeflow_test.go b/eks/kubeflow/kubeflow_test.go
deleted file mode 100644
index 03a5e49a4..000000000
--- a/eks/kubeflow/kubeflow_test.go
+++ /dev/null
@@ -1,40 +0,0 @@
-package kubeflow
-
-import (
- "fmt"
- "io/ioutil"
- "os"
- "testing"
- "time"
-
- "github.com/aws/aws-k8s-tester/eksconfig"
- "go.uber.org/zap"
-)
-
-func Test_downloadInstallKfctl(t *testing.T) {
- tt := &tester{cfg: Config{
- Logger: zap.NewExample(),
- EKSConfig: eksconfig.NewDefault(),
- }}
- os.RemoveAll(tt.cfg.EKSConfig.AddOnKubeflow.KfctlPath)
- if err := tt.downloadInstallKfctl(); err != nil {
- t.Fatal(err)
- }
-}
-
-func Test_writeKfctlConfig(t *testing.T) {
- tt := &tester{cfg: Config{
- Logger: zap.NewExample(),
- EKSConfig: eksconfig.NewDefault(),
- }}
- f, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("%X", time.Now().UnixNano()))
- if err != nil {
- t.Fatal(err)
- }
- tt.cfg.EKSConfig.AddOnKubeflow.KfctlConfigPath = f.Name()
- f.Close()
- os.RemoveAll(tt.cfg.EKSConfig.AddOnKubeflow.KfctlConfigPath)
- if err := tt.writeKfctlConfig(); err != nil {
- t.Fatal(err)
- }
-}
diff --git a/eks/kubernetes-dashboard/dashboard.go b/eks/kubernetes-dashboard/dashboard.go
deleted file mode 100644
index 15b0423e5..000000000
--- a/eks/kubernetes-dashboard/dashboard.go
+++ /dev/null
@@ -1,395 +0,0 @@
-package kubernetesdashboard
-
-import (
- "context"
- "errors"
- "fmt"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/pkg/fileutil"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "go.uber.org/zap"
- "k8s.io/utils/exec"
-)
-
-// ref. https://docs.aws.amazon.com/eks/latest/userguide/dashboard-tutorial.html
-// ref. https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta8/aio/deploy/recommended.yaml
-// ref. https://github.com/kubernetes/dashboard/blob/master/aio/deploy/recommended.yaml
-const dashboardYAML = `
-apiVersion: v1
-kind: Namespace
-metadata:
- name: kubernetes-dashboard
-
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- labels:
- k8s-app: kubernetes-dashboard
- name: kubernetes-dashboard
- namespace: kubernetes-dashboard
-
----
-
-kind: Service
-apiVersion: v1
-metadata:
- labels:
- k8s-app: kubernetes-dashboard
- name: kubernetes-dashboard
- namespace: kubernetes-dashboard
-spec:
- ports:
- - port: 443
- targetPort: 8443
- selector:
- k8s-app: kubernetes-dashboard
-
----
-
-apiVersion: v1
-kind: Secret
-metadata:
- labels:
- k8s-app: kubernetes-dashboard
- name: kubernetes-dashboard-certs
- namespace: kubernetes-dashboard
-type: Opaque
-
----
-
-apiVersion: v1
-kind: Secret
-metadata:
- labels:
- k8s-app: kubernetes-dashboard
- name: kubernetes-dashboard-csrf
- namespace: kubernetes-dashboard
-type: Opaque
-data:
- csrf: ""
-
----
-
-apiVersion: v1
-kind: Secret
-metadata:
- labels:
- k8s-app: kubernetes-dashboard
- name: kubernetes-dashboard-key-holder
- namespace: kubernetes-dashboard
-type: Opaque
-
----
-
-kind: ConfigMap
-apiVersion: v1
-metadata:
- labels:
- k8s-app: kubernetes-dashboard
- name: kubernetes-dashboard-settings
- namespace: kubernetes-dashboard
-
----
-
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
- labels:
- k8s-app: kubernetes-dashboard
- name: kubernetes-dashboard
- namespace: kubernetes-dashboard
-rules:
- # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
- - apiGroups: [""]
- resources: ["secrets"]
- resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
- verbs: ["get", "update", "delete"]
- # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
- - apiGroups: [""]
- resources: ["configmaps"]
- resourceNames: ["kubernetes-dashboard-settings"]
- verbs: ["get", "update"]
- # Allow Dashboard to get metrics.
- - apiGroups: [""]
- resources: ["services"]
- resourceNames: ["heapster", "dashboard-metrics-scraper"]
- verbs: ["proxy"]
- - apiGroups: [""]
- resources: ["services/proxy"]
- resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
- verbs: ["get"]
-
----
-
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
- labels:
- k8s-app: kubernetes-dashboard
- name: kubernetes-dashboard
-rules:
- # Allow Metrics Scraper to get metrics from the Metrics server
- - apiGroups: ["metrics.k8s.io"]
- resources: ["pods", "nodes"]
- verbs: ["get", "list", "watch"]
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
- labels:
- k8s-app: kubernetes-dashboard
- name: kubernetes-dashboard
- namespace: kubernetes-dashboard
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: Role
- name: kubernetes-dashboard
-subjects:
- - kind: ServiceAccount
- name: kubernetes-dashboard
- namespace: kubernetes-dashboard
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- name: kubernetes-dashboard
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: kubernetes-dashboard
-subjects:
- - kind: ServiceAccount
- name: kubernetes-dashboard
- namespace: kubernetes-dashboard
-
----
-
-kind: Deployment
-apiVersion: apps/v1
-metadata:
- labels:
- k8s-app: kubernetes-dashboard
- name: kubernetes-dashboard
- namespace: kubernetes-dashboard
-spec:
- replicas: 1
- revisionHistoryLimit: 10
- selector:
- matchLabels:
- k8s-app: kubernetes-dashboard
- template:
- metadata:
- labels:
- k8s-app: kubernetes-dashboard
- spec:
- containers:
- - name: kubernetes-dashboard
- image: kubernetesui/dashboard:v2.0.0-rc7
- imagePullPolicy: Always
- ports:
- - containerPort: 8443
- protocol: TCP
- args:
- - --auto-generate-certificates
- - --namespace=kubernetes-dashboard
- # Uncomment the following line to manually specify Kubernetes API server Host
- # If not specified, Dashboard will attempt to auto discover the API server and connect
- # to it. Uncomment only if the default does not work.
- # - --apiserver-host=http://my-address:port
- volumeMounts:
- - name: kubernetes-dashboard-certs
- mountPath: /certs
- # Create on-disk volume to store exec logs
- - mountPath: /tmp
- name: tmp-volume
- livenessProbe:
- httpGet:
- scheme: HTTPS
- path: /
- port: 8443
- initialDelaySeconds: 30
- timeoutSeconds: 30
- securityContext:
- allowPrivilegeEscalation: false
- readOnlyRootFilesystem: true
- runAsUser: 1001
- runAsGroup: 2001
- volumes:
- - name: kubernetes-dashboard-certs
- secret:
- secretName: kubernetes-dashboard-certs
- - name: tmp-volume
- emptyDir: {}
- serviceAccountName: kubernetes-dashboard
- nodeSelector:
- "kubernetes.io/os": linux
- # Comment the following tolerations if Dashboard must not be deployed on master
- tolerations:
- - key: node-role.kubernetes.io/master
- effect: NoSchedule
-
----
-
-kind: Service
-apiVersion: v1
-metadata:
- labels:
- k8s-app: dashboard-metrics-scraper
- name: dashboard-metrics-scraper
- namespace: kubernetes-dashboard
-spec:
- ports:
- - port: 8000
- targetPort: 8000
- selector:
- k8s-app: dashboard-metrics-scraper
-
----
-kind: Deployment
-apiVersion: apps/v1
-metadata:
- labels:
- k8s-app: dashboard-metrics-scraper
- name: dashboard-metrics-scraper
- namespace: kubernetes-dashboard
-spec:
- replicas: 1
- revisionHistoryLimit: 10
- selector:
- matchLabels:
- k8s-app: dashboard-metrics-scraper
- template:
- metadata:
- labels:
- k8s-app: dashboard-metrics-scraper
- annotations:
- seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
- spec:
- containers:
- - name: dashboard-metrics-scraper
- image: kubernetesui/metrics-scraper:v1.0.4
- ports:
- - containerPort: 8000
- protocol: TCP
- livenessProbe:
- httpGet:
- scheme: HTTP
- path: /
- port: 8000
- initialDelaySeconds: 30
- timeoutSeconds: 30
- volumeMounts:
- - mountPath: /tmp
- name: tmp-volume
- securityContext:
- allowPrivilegeEscalation: false
- readOnlyRootFilesystem: true
- runAsUser: 1001
- runAsGroup: 2001
- serviceAccountName: kubernetes-dashboard
- nodeSelector:
- "kubernetes.io/os": linux
- # Comment the following tolerations if Dashboard must not be deployed on master
- tolerations:
- - key: node-role.kubernetes.io/master
- effect: NoSchedule
- volumes:
- - name: tmp-volume
- emptyDir: {}
-
-`
-
-func (ts *tester) installDashboard() error {
- ts.cfg.Logger.Info("writing dashboard YAML")
- fpath, err := fileutil.WriteTempFile([]byte(dashboardYAML))
- if err != nil {
- ts.cfg.Logger.Warn("failed to write dashboard YAML", zap.Error(err))
- return err
- }
- ts.cfg.Logger.Info("applying dashboard YAML", zap.String("path", fpath))
-
- var output []byte
- waitDur := 5 * time.Minute
- retryStart := time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("create dashboard aborted")
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(
- ctx,
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig="+ts.cfg.EKSConfig.KubeConfigPath,
- "apply", "--filename="+fpath,
- ).CombinedOutput()
- cancel()
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\"kubectl apply\" dashboard output:\n%s\n", out)
- if err == nil {
- break
- }
- if strings.Contains(out, " created") || strings.Contains(out, " unchanged") {
- err = nil
- break
- }
-
- ts.cfg.Logger.Warn("create dashboard failed", zap.Error(err))
- ts.cfg.EKSConfig.RecordStatus(fmt.Sprintf("create dashboard failed (%v)", err))
- }
- if err != nil {
- return fmt.Errorf("'kubectl apply' failed %v (output %q)", err, string(output))
- }
-
- ts.cfg.Logger.Info("created dashboard")
-
- return ts.waitDeploymentDashboard()
-}
-
-func (ts *tester) waitDeploymentDashboard() (err error) {
- timeout := 7 * time.Minute
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- _, err = k8s_client.WaitForDeploymentCompletes(
- ctx,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.Stopc,
- ts.cfg.K8SClient,
- time.Minute,
- 20*time.Second,
- "kubernetes-dashboard",
- "kubernetes-dashboard",
- 1,
- k8s_client.WithQueryFunc(func() {
- descArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=kubernetes-dashboard",
- "describe",
- "deployment",
- "kubernetes-dashboard",
- }
- descCmd := strings.Join(descArgs, " ")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe deployment' failed", zap.Error(err))
- }
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", descCmd, out)
- }),
- )
- cancel()
- return err
-}
diff --git a/eks/kubernetes-dashboard/eks-admin.go b/eks/kubernetes-dashboard/eks-admin.go
deleted file mode 100644
index ffffdafd7..000000000
--- a/eks/kubernetes-dashboard/eks-admin.go
+++ /dev/null
@@ -1,125 +0,0 @@
-package kubernetesdashboard
-
-import (
- "context"
- "errors"
- "fmt"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/pkg/fileutil"
- "go.uber.org/zap"
- "k8s.io/utils/exec"
-)
-
-// ref. https://docs.aws.amazon.com/eks/latest/userguide/dashboard-tutorial.html
-const eksAdminYAML = `
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: eks-admin
- namespace: kube-system
-
----
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRoleBinding
-metadata:
- name: eks-admin
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: cluster-admin
-subjects:
-- kind: ServiceAccount
- name: eks-admin
- namespace: kube-system
-
-`
-
-func (ts *tester) installEKSAdmin() error {
- ts.cfg.Logger.Info("writing eks-admin YAML")
- fpath, err := fileutil.WriteTempFile([]byte(eksAdminYAML))
- if err != nil {
- ts.cfg.Logger.Warn("failed to write eks-admin YAML", zap.Error(err))
- return err
- }
- ts.cfg.Logger.Info("applying eks-admin YAML", zap.String("path", fpath))
-
- var output []byte
- waitDur := 5 * time.Minute
- retryStart := time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("create eks-admin aborted")
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(
- ctx,
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig="+ts.cfg.EKSConfig.KubeConfigPath,
- "apply", "--filename="+fpath,
- ).CombinedOutput()
- cancel()
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\"kubectl apply\" eks-admin output:\n%s\n", out)
- if err == nil {
- break
- }
- if strings.Contains(out, " created") || strings.Contains(out, " unchanged") {
- err = nil
- break
- }
-
- ts.cfg.Logger.Warn("create eks-admin failed", zap.Error(err))
- ts.cfg.EKSConfig.RecordStatus(fmt.Sprintf("create eks-admin failed (%v)", err))
- }
- if err != nil {
- return fmt.Errorf("'kubectl apply' failed %v (output %q)", err, string(output))
- }
-
- ts.cfg.Logger.Info("created eks-admin")
- return ts.fetchAuthenticationToken()
-}
-
-func (ts *tester) fetchAuthenticationToken() error {
- ts.cfg.Logger.Info("fetching authentication token")
-
- var token []byte
- waitDur := time.Minute
- retryStart := time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("check aborted")
- case <-time.After(15 * time.Second):
- }
-
- ls, err := ts.cfg.K8SClient.ListSecrets("kube-system", 10, 5*time.Second)
- if err != nil {
- return fmt.Errorf("failed to list secrets (%v)", err)
- }
- for _, v := range ls {
- if !strings.HasPrefix(v.Name, "eks-admin") {
- continue
- }
- token = v.Data["token"]
- break
- }
- if len(token) > 0 {
- break
- }
- }
- if len(token) == 0 {
- return errors.New("authentication token not found")
- }
- ts.cfg.Logger.Info("fetched authentication token")
-
- ts.cfg.EKSConfig.AddOnKubernetesDashboard.AuthenticationToken = string(token)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\nKubernetes Dashboard Token:\n%s\n\n\n", ts.cfg.EKSConfig.AddOnKubernetesDashboard.AuthenticationToken)
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
diff --git a/eks/kubernetes-dashboard/kubernetes-dashboard.go b/eks/kubernetes-dashboard/kubernetes-dashboard.go
deleted file mode 100644
index ed911ccdd..000000000
--- a/eks/kubernetes-dashboard/kubernetes-dashboard.go
+++ /dev/null
@@ -1,210 +0,0 @@
-// Package kubernetesdashboard implements Kubernetes dashboard add-on.
-package kubernetesdashboard
-
-import (
- "context"
- "errors"
- "fmt"
- "io"
- "io/ioutil"
- "os"
- "os/exec"
- "reflect"
- "strings"
- "syscall"
- "time"
-
- eks_tester "github.com/aws/aws-k8s-tester/eks/tester"
- "github.com/aws/aws-k8s-tester/eksconfig"
- "github.com/aws/aws-k8s-tester/pkg/httputil"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "go.uber.org/zap"
-)
-
-// Config defines Dashboard configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-func New(cfg Config) eks_tester.Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
- proxyCmd *exec.Cmd
- proxyCancel func()
-}
-
-func (ts *tester) Create() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnKubernetesDashboard() {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
- if ts.cfg.EKSConfig.AddOnKubernetesDashboard.Created {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Create", zap.String("tester", pkgName))
- ts.cfg.EKSConfig.AddOnKubernetesDashboard.Created = true
- ts.cfg.EKSConfig.Sync()
- createStart := time.Now()
- defer func() {
- createEnd := time.Now()
- ts.cfg.EKSConfig.AddOnKubernetesDashboard.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- if err := ts.installDashboard(); err != nil {
- return err
- }
- if err := ts.installEKSAdmin(); err != nil {
- return err
- }
- // TODO: use ingress
- if err := ts.startProxy(false); err != nil {
- return err
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) Delete() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnKubernetesDashboard() {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnKubernetesDashboard.Created {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Delete", zap.String("tester", pkgName))
- deleteStart := time.Now()
- defer func() {
- deleteEnd := time.Now()
- ts.cfg.EKSConfig.AddOnKubernetesDashboard.TimeFrameDelete = timeutil.NewTimeFrame(deleteStart, deleteEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- var errs []string
-
- if err := ts.stopProxy(); err != nil {
- errs = append(errs, err.Error())
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- ts.cfg.EKSConfig.AddOnKubernetesDashboard.Created = false
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) startProxy(enable bool) error {
- proxyArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "proxy",
- }
- proxyCmd := strings.Join(proxyArgs, " ")
-
- if enable {
- ts.cfg.Logger.Info("starting Kubernetes Dashboard proxy", zap.String("cmd-path", ts.cfg.EKSConfig.KubectlPath))
- ctx, cancel := context.WithCancel(context.Background())
- ts.proxyCmd = exec.CommandContext(ctx, proxyArgs[0], proxyArgs[1:]...)
- ts.proxyCmd.Stderr = os.Stderr
- ts.proxyCmd.Stdout = os.Stdout
- ts.proxyCancel = cancel
- if err := ts.proxyCmd.Start(); err != nil {
- ts.cfg.Logger.Warn("failed to start kubectl proxy command", zap.Error(err))
- ts.proxyCancel()
- if ts.proxyCmd.Process != nil {
- ts.proxyCmd.Process.Kill()
- }
- return err
- }
- ts.cfg.EKSConfig.AddOnKubernetesDashboard.KubectlProxyPID = ts.proxyCmd.Process.Pid
- ts.cfg.Logger.Info("started Kubernetes Dashboard proxy", zap.Int("pid", ts.cfg.EKSConfig.AddOnKubernetesDashboard.KubectlProxyPID))
-
- waitDur := time.Minute
- retryStart := time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("Kubernetes Dashboard proxy creation aborted")
- case <-time.After(5 * time.Second):
- }
-
- out, err := httputil.ReadInsecure(ts.cfg.Logger, ioutil.Discard, ts.cfg.EKSConfig.AddOnKubernetesDashboard.URL)
- if err != nil {
- ts.cfg.Logger.Warn("failed to read Kubernetes Dashboard proxy; retrying", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
- httpOutput := string(out)
- fmt.Fprintf(ts.cfg.LogWriter, "\nKubernetes Dashboard proxy output:\n%s\n", httpOutput)
-
- if strings.Contains(httpOutput, `The Kubernetes Authors`) {
- ts.cfg.Logger.Info("read Kubernetes Dashboard proxy; exiting")
- break
- }
-
- ts.cfg.Logger.Warn("unexpected Kubernetes Dashboard proxy output; retrying")
- }
- }
-
- fmt.Fprintf(ts.cfg.LogWriter, "\nkubectl proxy command:\n%s\n", proxyCmd)
- fmt.Fprintf(ts.cfg.LogWriter, "\nKubernetes Dashboard Token:\n%s\n", ts.cfg.EKSConfig.AddOnKubernetesDashboard.AuthenticationToken)
- fmt.Fprintf(ts.cfg.LogWriter, "\nKubernetes Dashboard URL:\n%s\n\n", ts.cfg.EKSConfig.AddOnKubernetesDashboard.URL)
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) stopProxy() error {
- if ts.proxyCmd == nil || ts.cfg.EKSConfig.AddOnKubernetesDashboard.KubectlProxyPID == 0 {
- return nil
- }
-
- ts.cfg.Logger.Info("stopping Kubernetes Dashboard proxy")
-
- if ts.proxyCancel != nil {
- ts.proxyCancel()
- }
-
- if ts.proxyCmd != nil && ts.proxyCmd.Process != nil {
- err := ts.proxyCmd.Process.Kill()
- if err != nil {
- ts.cfg.Logger.Warn("proxyCmd.Process.Kill failed", zap.Error(err))
- } else {
- ts.cfg.Logger.Info("ran proxyCmd.Process.Kill")
- }
- }
-
- if ts.cfg.EKSConfig.AddOnKubernetesDashboard.KubectlProxyPID != 0 {
- err := syscall.Kill(-ts.cfg.EKSConfig.AddOnKubernetesDashboard.KubectlProxyPID, syscall.SIGKILL)
- if err != nil {
- ts.cfg.Logger.Warn("syscall.Kill failed", zap.Error(err))
- } else {
- ts.cfg.Logger.Info("ran syscall.Kill")
- }
- }
-
- ts.cfg.Logger.Info("stopped Kubernetes Dashboard proxy")
-
- return nil
-}
diff --git a/eks/metrics-server/addon.go b/eks/metrics-server/addon.go
deleted file mode 100644
index 7fb011191..000000000
--- a/eks/metrics-server/addon.go
+++ /dev/null
@@ -1,58 +0,0 @@
-// Package metricsserver implements Kubernetes metrics server.
-// ref. https://github.com/kubernetes-sigs/metrics-server/releases
-package metricsserver
-
-import (
- "fmt"
-
- "github.com/aws/aws-k8s-tester/eksconfig"
- k8sclient "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- gotemplate "github.com/aws/aws-k8s-tester/pkg/util"
-)
-
-// MetricsServer is an addon that installs Metrics Server
-type MetricsServer struct {
- K8sClient k8sclient.EKS
- Config *eksconfig.Config
-}
-
-// IsEnabled returns true if enabled
-func (c *MetricsServer) IsEnabled() bool {
- return c.Config.Spec.MetricsServer != nil
-}
-
-// Apply installs the addon
-func (c *MetricsServer) Apply() (err error) {
- template, err := gotemplate.FromLocalDirectory(c.Config.Spec.MetricsServer)
- if err != nil {
- return fmt.Errorf("while building templates, %v", err)
- }
- if err := c.K8sClient.Apply(template.String()); err != nil {
- return fmt.Errorf("while applying resources, %v", err)
- }
- c.Config.Status.MetricsServer = &eksconfig.MetricsServerStatus{
- AddonStatus: eksconfig.AddonStatus{
- Installed: true,
- Ready: true,
- },
- }
- return nil
-}
-
-// Delete removes the addon
-func (c *MetricsServer) Delete() (err error) {
- template, err := gotemplate.FromLocalDirectory(c.Config.Spec.MetricsServer)
- if err != nil {
- return fmt.Errorf("while building templates, %v", err)
- }
- if err := c.K8sClient.Delete(template.String()); err != nil {
- return fmt.Errorf("while deleting resources, %v", err)
- }
- c.Config.Status.MetricsServer = &eksconfig.MetricsServerStatus{
- AddonStatus: eksconfig.AddonStatus{
- Installed: false,
- Ready: false,
- },
- }
- return nil
-}
diff --git a/eks/metrics-server/metrics-server.go b/eks/metrics-server/metrics-server.go
deleted file mode 100644
index ceef4b8ff..000000000
--- a/eks/metrics-server/metrics-server.go
+++ /dev/null
@@ -1,461 +0,0 @@
-// Package metricsserver implements Kubernetes metrics server.
-// ref. https://github.com/kubernetes-sigs/metrics-server/releases
-package metricsserver
-
-import (
- "context"
- "errors"
- "fmt"
- "io"
- "reflect"
- "strings"
- "time"
-
- eks_tester "github.com/aws/aws-k8s-tester/eks/tester"
- "github.com/aws/aws-k8s-tester/eksconfig"
- "github.com/aws/aws-k8s-tester/pkg/fileutil"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/aws"
- "go.uber.org/zap"
- apierrs "k8s.io/apimachinery/pkg/api/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/utils/exec"
-)
-
-// Config defines Dashboard configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-func New(cfg Config) eks_tester.Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
-}
-
-func (ts *tester) Create() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnMetricsServer() {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
- if ts.cfg.EKSConfig.AddOnMetricsServer.Created {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Create", zap.String("tester", pkgName))
- ts.cfg.EKSConfig.AddOnMetricsServer.Created = true
- ts.cfg.EKSConfig.Sync()
- createStart := time.Now()
- defer func() {
- createEnd := time.Now()
- ts.cfg.EKSConfig.AddOnMetricsServer.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- if err := ts.createMetricsServer(); err != nil {
- return err
- }
- if err := ts.waitDeployment(); err != nil {
- return err
- }
- if err := ts.checkMetrics(); err != nil {
- return err
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) Delete() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnMetricsServer() {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnMetricsServer.Created {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Delete", zap.String("tester", pkgName))
- deleteStart := time.Now()
- defer func() {
- deleteEnd := time.Now()
- ts.cfg.EKSConfig.AddOnMetricsServer.TimeFrameDelete = timeutil.NewTimeFrame(deleteStart, deleteEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- var errs []string
-
- if err := ts.deleteDeployment(); err != nil {
- errs = append(errs, err.Error())
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- ts.cfg.EKSConfig.AddOnMetricsServer.Created = false
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// ref. https://docs.aws.amazon.com/eks/latest/userguide/dashboard-tutorial.html
-// ref. https://github.com/kubernetes-sigs/metrics-server/releases
-// ref. https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.3.6/components.yaml
-const metricsServerYAML = `
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- name: system:aggregated-metrics-reader
- labels:
- rbac.authorization.k8s.io/aggregate-to-view: "true"
- rbac.authorization.k8s.io/aggregate-to-edit: "true"
- rbac.authorization.k8s.io/aggregate-to-admin: "true"
-rules:
-- apiGroups: ["metrics.k8s.io"]
- resources: ["pods", "nodes"]
- verbs: ["get", "list", "watch"]
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- name: metrics-server:system:auth-delegator
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: system:auth-delegator
-subjects:
-- kind: ServiceAccount
- name: metrics-server
- namespace: kube-system
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
- name: metrics-server-auth-reader
- namespace: kube-system
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: Role
- name: extension-apiserver-authentication-reader
-subjects:
-- kind: ServiceAccount
- name: metrics-server
- namespace: kube-system
-
----
-apiVersion: apiregistration.k8s.io/v1beta1
-kind: APIService
-metadata:
- name: v1beta1.metrics.k8s.io
-spec:
- service:
- name: metrics-server
- namespace: kube-system
- group: metrics.k8s.io
- version: v1beta1
- insecureSkipTLSVerify: true
- groupPriorityMinimum: 100
- versionPriority: 100
-
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: metrics-server
- namespace: kube-system
-
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: metrics-server
- namespace: kube-system
- labels:
- k8s-app: metrics-server
-spec:
- selector:
- matchLabels:
- k8s-app: metrics-server
- template:
- metadata:
- name: metrics-server
- labels:
- k8s-app: metrics-server
- spec:
- serviceAccountName: metrics-server
- volumes:
- # mount in tmp so we can safely use from-scratch images and/or read-only containers
- - name: tmp-dir
- emptyDir: {}
- containers:
- - name: metrics-server
- image: k8s.gcr.io/metrics-server-amd64:v0.3.6
- imagePullPolicy: IfNotPresent
- args:
- - --cert-dir=/tmp
- - --secure-port=4443
- - --kubelet-insecure-tls
- - --kubelet-preferred-address-types=InternalIP
- ports:
- - name: main-port
- containerPort: 4443
- protocol: TCP
- securityContext:
- readOnlyRootFilesystem: true
- runAsNonRoot: true
- runAsUser: 1000
- volumeMounts:
- - name: tmp-dir
- mountPath: /tmp
- nodeSelector:
- kubernetes.io/os: linux
- kubernetes.io/arch: "amd64"
-
----
-apiVersion: v1
-kind: Service
-metadata:
- name: metrics-server
- namespace: kube-system
- labels:
- kubernetes.io/name: "Metrics-server"
- kubernetes.io/cluster-service: "true"
-spec:
- selector:
- k8s-app: metrics-server
- ports:
- - port: 443
- protocol: TCP
- targetPort: main-port
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- name: system:metrics-server
-rules:
-- apiGroups:
- - ""
- resources:
- - pods
- - nodes
- - nodes/stats
- - namespaces
- - configmaps
- verbs:
- - get
- - list
- - watch
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- name: system:metrics-server
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: system:metrics-server
-subjects:
-- kind: ServiceAccount
- name: metrics-server
- namespace: kube-system
-
-
-`
-
-const (
- deploymentName = "metrics-server"
-)
-
-// ref. https://github.com/kubernetes-sigs/metrics-server
-func (ts *tester) createMetricsServer() error {
- ts.cfg.Logger.Info("writing metrics-server YAML")
- fpath, err := fileutil.WriteTempFile([]byte(metricsServerYAML))
- if err != nil {
- ts.cfg.Logger.Warn("failed to write metrics-server YAML", zap.Error(err))
- return err
- }
- ts.cfg.Logger.Info("applying metrics-server YAML", zap.String("path", fpath))
-
- applyArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "apply",
- "--filename=" + fpath,
- }
- applyCmd := strings.Join(applyArgs, " ")
-
- var output []byte
- waitDur := 5 * time.Minute
- retryStart := time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("create metrics-server aborted")
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, applyArgs[0], applyArgs[1:]...).CombinedOutput()
- cancel()
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\"%s\" output:\n%s\n", applyCmd, out)
- if err == nil {
- break
- }
- if strings.Contains(out, " created") || strings.Contains(out, " unchanged") {
- err = nil
- break
- }
-
- ts.cfg.Logger.Warn("create metrics-server failed", zap.Error(err))
- ts.cfg.EKSConfig.RecordStatus(fmt.Sprintf("create metrics-server failed (%v)", err))
- }
- if err != nil {
- return fmt.Errorf("'kubectl apply' failed %v (output %q)", err, string(output))
- }
-
- ts.cfg.Logger.Info("created metrics-server")
- return nil
-}
-
-func (ts *tester) waitDeployment() (err error) {
- timeout := 7 * time.Minute
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- _, err = k8s_client.WaitForDeploymentCompletes(
- ctx,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.Stopc,
- ts.cfg.K8SClient,
- time.Minute,
- 20*time.Second,
- "kube-system",
- deploymentName,
- 1,
- k8s_client.WithQueryFunc(func() {
- descArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=kube-system",
- "describe",
- "deployment",
- deploymentName,
- }
- descCmd := strings.Join(descArgs, " ")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe deployment' failed", zap.Error(err))
- }
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", descCmd, out)
- }),
- )
- cancel()
- return err
-}
-
-func (ts *tester) checkMetrics() error {
- logArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=kube-system",
- "logs",
- "--selector=k8s-app=metrics-server",
- "--all-containers=true",
- "--timestamps",
- }
- logsCmd := strings.Join(logArgs, " ")
-
- topNodeArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "top",
- "node",
- }
- topNodeCmd := strings.Join(topNodeArgs, " ")
-
- topNodeReady := false
- waitDur, retryStart := 30*time.Minute, time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("check aborted")
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, logArgs[0], logArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("failed to run kubectl logs", zap.Error(err))
- continue
- }
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", logsCmd, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
- output, err = exec.New().CommandContext(ctx, topNodeArgs[0], topNodeArgs[1:]...).CombinedOutput()
- out = string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", topNodeCmd, out)
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("failed to run kubectl top node", zap.Error(err))
- continue
- }
- if strings.Contains(out, "MEMORY") {
- topNodeReady = true
- break
- }
- }
- if !topNodeReady {
- return fmt.Errorf("%q not ready", topNodeCmd)
- }
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) deleteDeployment() error {
- ts.cfg.Logger.Info("deleting deployment")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- AppsV1().
- Deployments("kube-system").
- Delete(
- ctx,
- deploymentName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return err
- }
- ts.cfg.Logger.Info("deleted deployment")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
diff --git a/eks/metrics-server/metrics-server.gotmpl b/eks/metrics-server/metrics-server.gotmpl
deleted file mode 100644
index 6130be92c..000000000
--- a/eks/metrics-server/metrics-server.gotmpl
+++ /dev/null
@@ -1,208 +0,0 @@
-# ref. https://docs.aws.amazon.com/eks/latest/userguide/dashboard-tutorial.html
-# ref. https://github.com/kubernetes-sigs/metrics-server/releases
-# ref. https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.3.6/components.yaml
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- labels:
- k8s-app: metrics-server
- rbac.authorization.k8s.io/aggregate-to-admin: "true"
- rbac.authorization.k8s.io/aggregate-to-edit: "true"
- rbac.authorization.k8s.io/aggregate-to-view: "true"
- name: system:aggregated-metrics-reader
-rules:
-- apiGroups:
- - metrics.k8s.io
- resources:
- - pods
- - nodes
- verbs:
- - get
- - list
- - watch
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- labels:
- k8s-app: metrics-server
- name: metrics-server:system:auth-delegator
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: system:auth-delegator
-subjects:
-- kind: ServiceAccount
- name: metrics-server
- namespace: kube-system
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
- labels:
- k8s-app: metrics-server
- name: metrics-server-auth-reader
- namespace: kube-system
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: Role
- name: extension-apiserver-authentication-reader
-subjects:
-- kind: ServiceAccount
- name: metrics-server
- namespace: kube-system
-
----
-apiVersion: apiregistration.k8s.io/v1
-kind: APIService
-metadata:
- labels:
- k8s-app: metrics-server
- name: v1beta1.metrics.k8s.io
-spec:
- group: metrics.k8s.io
- groupPriorityMinimum: 100
- insecureSkipTLSVerify: true
- service:
- name: metrics-server
- namespace: kube-system
- version: v1beta1
- versionPriority: 100
-
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- labels:
- k8s-app: metrics-server
- name: metrics-server
- namespace: kube-system
-
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- labels:
- k8s-app: metrics-server
- name: metrics-server
- namespace: kube-system
-spec:
- selector:
- matchLabels:
- k8s-app: metrics-server
- strategy:
- rollingUpdate:
- maxUnavailable: 0
- template:
- metadata:
- labels:
- k8s-app: metrics-server
- spec:
- containers:
- - args:
- - --cert-dir=/tmp
- - --secure-port=4443
- - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- - --kubelet-use-node-status-port
- - --metric-resolution=15s
- image: k8s.gcr.io/metrics-server/metrics-server:v0.6.1
- imagePullPolicy: IfNotPresent
- livenessProbe:
- failureThreshold: 3
- httpGet:
- path: /livez
- port: https
- scheme: HTTPS
- periodSeconds: 10
- name: metrics-server
- ports:
- - containerPort: 4443
- name: https
- protocol: TCP
- readinessProbe:
- failureThreshold: 3
- httpGet:
- path: /readyz
- port: https
- scheme: HTTPS
- initialDelaySeconds: 20
- periodSeconds: 10
- resources:
- requests:
- cpu: 100m
- memory: 200Mi
- securityContext:
- allowPrivilegeEscalation: false
- readOnlyRootFilesystem: true
- runAsNonRoot: true
- runAsUser: 1000
- volumeMounts:
- - mountPath: /tmp
- name: tmp-dir
- nodeSelector:
- kubernetes.io/os: linux
- priorityClassName: system-cluster-critical
- serviceAccountName: metrics-server
- volumes:
- - emptyDir: {}
- name: tmp-dir
-
----
-apiVersion: v1
-kind: Service
-metadata:
- labels:
- k8s-app: metrics-server
- name: metrics-server
- namespace: kube-system
-spec:
- ports:
- - name: https
- port: 443
- protocol: TCP
- targetPort: https
- selector:
- k8s-app: metrics-server
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- labels:
- k8s-app: metrics-server
- name: system:metrics-server
-rules:
-- apiGroups:
- - ""
- resources:
- - nodes/metrics
- verbs:
- - get
-- apiGroups:
- - ""
- resources:
- - pods
- - nodes
- verbs:
- - get
- - list
- - watch
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- labels:
- k8s-app: metrics-server
- name: system:metrics-server
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: system:metrics-server
-subjects:
-- kind: ServiceAccount
- name: metrics-server
- namespace: kube-system
diff --git a/eks/mng/enis.go b/eks/mng/enis.go
deleted file mode 100644
index 7f082ff78..000000000
--- a/eks/mng/enis.go
+++ /dev/null
@@ -1,145 +0,0 @@
-package mng
-
-import (
- "context"
- "errors"
- "fmt"
- "strings"
- "time"
-
- aws_v2 "github.com/aws/aws-sdk-go-v2/aws"
- aws_ec2_v2 "github.com/aws/aws-sdk-go-v2/service/ec2"
- aws_ec2_v2_types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
- smithy "github.com/aws/smithy-go"
- "go.uber.org/zap"
-)
-
-func (ts *tester) deleteENIs(name string) bool {
- cur, ok := ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[name]
- if !ok {
- return false
- }
- if cur.RemoteAccessSecurityGroupID == "" {
- return false
- }
-
- ts.cfg.Logger.Info("deleting ENIs for security group", zap.String("mng-name", name), zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- out, err := ts.cfg.EC2APIV2.DescribeNetworkInterfaces(
- context.Background(),
- &aws_ec2_v2.DescribeNetworkInterfacesInput{
- Filters: []aws_ec2_v2_types.Filter{
- {
- Name: aws_v2.String("group-id"),
- Values: []string{cur.RemoteAccessSecurityGroupID},
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to describe ENIs", zap.Error(err))
- return false
- }
-
- enis := make([]aws_ec2_v2_types.NetworkInterface, 0)
- for _, eni := range out.NetworkInterfaces {
- enis = append(enis, eni)
- ts.cfg.Logger.Info("found ENI", zap.String("eni", aws_v2.ToString(eni.NetworkInterfaceId)))
- }
-
- // detacth and delete ENIs
- deleted := false
- for _, eni := range enis {
- eniID := aws_v2.ToString(eni.NetworkInterfaceId)
- if _, ok := ts.cfg.EKSConfig.Status.DeletedResources[eniID]; ok {
- continue
- }
-
- ts.cfg.Logger.Warn("detaching ENI", zap.String("eni", eniID))
- out, err := ts.cfg.EC2APIV2.DescribeNetworkInterfaces(
- context.Background(),
- &aws_ec2_v2.DescribeNetworkInterfacesInput{
- NetworkInterfaceIds: []string{eniID},
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to describe ENI", zap.Error(err))
- continue
- }
- if len(out.NetworkInterfaces) != 1 {
- ts.cfg.Logger.Warn("expected 1 ENI", zap.String("eni", eniID), zap.Int("enis", len(out.NetworkInterfaces)))
- continue
- }
- if out.NetworkInterfaces[0].Attachment == nil {
- ts.cfg.Logger.Warn("no attachment found for ENI", zap.String("eni", eniID))
- } else {
- for i := 0; i < 5; i++ {
- time.Sleep(5 * time.Second)
- _, err = ts.cfg.EC2APIV2.DetachNetworkInterface(
- context.Background(),
- &aws_ec2_v2.DetachNetworkInterfaceInput{
- AttachmentId: out.NetworkInterfaces[0].Attachment.AttachmentId,
- Force: aws_v2.Bool(true),
- })
- if err == nil {
- ts.cfg.Logger.Info("successfully detached ENI", zap.String("eni", eniID))
- break
- }
- ts.cfg.Logger.Warn("failed to detach ENI", zap.String("eni", eniID), zap.Error(err))
- }
- }
-
- for i := 0; i < 5; i++ {
- // may take awhile for delete to success upon detach
- time.Sleep(10 * time.Second)
- ts.cfg.Logger.Info("deleting ENI", zap.String("eni", eniID))
- _, err = ts.cfg.EC2APIV2.DeleteNetworkInterface(
- context.Background(),
- &aws_ec2_v2.DeleteNetworkInterfaceInput{
- NetworkInterfaceId: aws_v2.String(eniID),
- })
- if err == nil {
- ts.cfg.Logger.Info("successfully deleted ENI", zap.String("eni", eniID))
- deleted = true
- break
- }
- ts.cfg.Logger.Warn("failed to delete ENI", zap.String("eni", eniID), zap.Error(err))
- }
-
- // confirm ENI deletion
- retryStart := time.Now()
- for time.Since(retryStart) < 5*time.Minute {
- time.Sleep(5 * time.Second)
- _, err = ts.cfg.EC2APIV2.DescribeNetworkInterfaces(
- context.Background(),
- &aws_ec2_v2.DescribeNetworkInterfacesInput{
- NetworkInterfaceIds: []string{eniID},
- })
- if err == nil {
- _, derr := ts.cfg.EC2APIV2.DeleteNetworkInterface(
- context.Background(),
- &aws_ec2_v2.DeleteNetworkInterfaceInput{
- NetworkInterfaceId: aws_v2.String(eniID),
- })
- ts.cfg.Logger.Warn("ENI still exists", zap.String("eni", eniID), zap.Error(derr))
- continue
- }
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- ts.cfg.EKSConfig.Status.DeletedResources[eniID] = "AddOnManagedNodeGroups.ENI"
- ts.cfg.EKSConfig.Sync()
- deleted = true
- break
- }
- }
-
- _, derr := ts.cfg.EC2APIV2.DeleteNetworkInterface(
- context.Background(),
- &aws_ec2_v2.DeleteNetworkInterfaceInput{
- NetworkInterfaceId: aws_v2.String(eniID),
- })
- ts.cfg.Logger.Warn("ENI still exists", zap.String("eni", eniID), zap.String("errors", fmt.Sprintf("%v, %v", err, derr)))
- }
- }
- return deleted
-}
diff --git a/eks/mng/logs.go b/eks/mng/logs.go
deleted file mode 100644
index 2103d2872..000000000
--- a/eks/mng/logs.go
+++ /dev/null
@@ -1,505 +0,0 @@
-package mng
-
-import (
- "context"
- "fmt"
- "os"
- "path/filepath"
- "sort"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/pkg/fileutil"
- "github.com/aws/aws-k8s-tester/pkg/randutil"
- "github.com/aws/aws-k8s-tester/ssh"
- "github.com/dustin/go-humanize"
- "github.com/mholt/archiver/v3"
- "go.uber.org/zap"
- "golang.org/x/time/rate"
-)
-
-var defaultLogs = map[string]string{
- // kernel logs
- "sudo journalctl --no-pager --output=short-precise -k": "kernel.out.log",
-
- // full journal logs (e.g. disk mounts)
- "sudo journalctl --no-pager --output=short-precise": "journal.out.log",
-
- // other systemd services
- "sudo systemctl list-units -t service --no-pager --no-legend --all": "list-units-systemctl.out.log",
-}
-
-// FetchLogs downloads logs from managed node group instances.
-func (ts *tester) FetchLogs() (err error) {
- if !ts.cfg.EKSConfig.IsEnabledAddOnManagedNodeGroups() {
- ts.cfg.Logger.Info("skipping fetch logs for node groups")
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnManagedNodeGroups.FetchLogs {
- ts.cfg.Logger.Info("skipping fetch logs for node groups")
- return nil
- }
-
- ts.logsMu.Lock()
- defer ts.logsMu.Unlock()
-
- err = os.MkdirAll(ts.cfg.EKSConfig.AddOnManagedNodeGroups.LogsDir, 0700)
- if err != nil {
- ts.cfg.Logger.Warn("failed to mkdir", zap.Error(err))
- return err
- }
-
- err = ts.fetchLogs(250, 10)
- if err != nil {
- ts.cfg.Logger.Warn("failed to fetch logs", zap.Error(err))
- return err
- }
-
- ts.cfg.Logger.Info("gzipping logs dir", zap.String("logs-dir", ts.cfg.EKSConfig.AddOnManagedNodeGroups.LogsDir), zap.String("file-path", ts.cfg.EKSConfig.AddOnManagedNodeGroups.LogsTarGzPath))
- err = os.RemoveAll(ts.cfg.EKSConfig.AddOnManagedNodeGroups.LogsTarGzPath)
- if err != nil {
- ts.cfg.Logger.Warn("failed to remove temp file", zap.Error(err))
- return err
- }
- err = archiver.Archive([]string{ts.cfg.EKSConfig.AddOnManagedNodeGroups.LogsDir}, ts.cfg.EKSConfig.AddOnManagedNodeGroups.LogsTarGzPath)
- if err != nil {
- ts.cfg.Logger.Warn("archive failed", zap.Error(err))
- return err
- }
- stat, err := os.Stat(ts.cfg.EKSConfig.AddOnManagedNodeGroups.LogsTarGzPath)
- if err != nil {
- ts.cfg.Logger.Warn("failed to os stat", zap.Error(err))
- return err
- }
- sz := humanize.Bytes(uint64(stat.Size()))
- ts.cfg.Logger.Info("gzipped logs dir", zap.String("logs-dir", ts.cfg.EKSConfig.AddOnManagedNodeGroups.LogsDir), zap.String("file-path", ts.cfg.EKSConfig.AddOnManagedNodeGroups.LogsTarGzPath), zap.String("file-size", sz))
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) fetchLogs(qps float32, burst int) error {
- logsDir := ts.cfg.EKSConfig.AddOnManagedNodeGroups.LogsDir
- sshOptLog := ssh.WithVerbose(ts.cfg.EKSConfig.LogLevel == "debug")
- rateLimiter := rate.NewLimiter(rate.Limit(qps), burst)
- rch, waits := make(chan instanceLogs, 10), 0
-
- for name, nodeGroup := range ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs {
- ts.cfg.Logger.Info("fetching logs from managed node group",
- zap.String("mng-name", name),
- zap.Int("nodes", len(nodeGroup.Instances)),
- )
- waits += len(nodeGroup.Instances)
-
- for instID, cur := range nodeGroup.Instances {
- pfx := instID + "-"
-
- go func(instID, logsDir, pfx string, cur ec2config.Instance) {
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("exiting fetch logger", zap.String("prefix", pfx))
- return
- default:
- }
-
- if !rateLimiter.Allow() {
- ts.cfg.Logger.Debug("waiting for rate limiter before SSH into the machine",
- zap.Float32("qps", qps),
- zap.Int("burst", burst),
- zap.String("instance-id", instID),
- )
- werr := rateLimiter.Wait(context.Background())
- ts.cfg.Logger.Debug("waited for rate limiter",
- zap.Float32("qps", qps),
- zap.Int("burst", burst),
- zap.Error(werr),
- )
- }
-
- sh, err := ssh.New(ssh.Config{
- Logger: ts.cfg.Logger,
- KeyPath: ts.cfg.EKSConfig.RemoteAccessPrivateKeyPath,
- PublicIP: cur.PublicIP,
- PublicDNSName: cur.PublicDNSName,
- UserName: cur.RemoteAccessUserName,
- })
- if err != nil {
- rch <- instanceLogs{mngName: name, errs: []string{err.Error()}}
- return
- }
- defer sh.Close()
- if err = sh.Connect(); err != nil {
- rch <- instanceLogs{mngName: name, errs: []string{err.Error()}}
- return
- }
-
- data := instanceLogs{mngName: name, instanceID: instID}
- // fetch default logs
- for cmd, fileName := range defaultLogs {
- if !rateLimiter.Allow() {
- ts.cfg.Logger.Debug("waiting for rate limiter before fetching file")
- werr := rateLimiter.Wait(context.Background())
- ts.cfg.Logger.Debug("waited for rate limiter", zap.Error(werr))
- }
- out, oerr := sh.Run(cmd, sshOptLog)
- if oerr != nil {
- data.errs = append(data.errs, fmt.Sprintf("failed to run command %q for %q (error %v)", cmd, instID, oerr))
- continue
- }
-
- fpath := filepath.Join(logsDir, shorten(ts.cfg.Logger, pfx+fileName))
- f, err := os.Create(fpath)
- if err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to create a file %q for %q (error %v)",
- fpath,
- instID,
- err,
- ))
- continue
- }
- if _, err = f.Write(out); err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to write to a file %q for %q (error %v)",
- fpath,
- instID,
- err,
- ))
- f.Close()
- continue
- }
- f.Close()
- ts.cfg.Logger.Debug("wrote", zap.String("file-path", fpath))
- data.paths = append(data.paths, fpath)
- }
-
- if !rateLimiter.Allow() {
- ts.cfg.Logger.Debug("waiting for rate limiter before fetching file")
- werr := rateLimiter.Wait(context.Background())
- ts.cfg.Logger.Debug("waited for rate limiter", zap.Error(werr))
- }
- ts.cfg.Logger.Info("listing systemd service units", zap.String("instance-id", instID))
- listCmd := "sudo systemctl list-units -t service --no-pager --no-legend --all"
- out, oerr := sh.Run(listCmd, sshOptLog)
- if oerr != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to run command %q for %q (error %v)",
- listCmd,
- instID,
- oerr,
- ))
- } else {
- /*
- auditd.service loaded active running Security Auditing Service
- auth-rpcgss-module.service loaded inactive dead Kernel Module supporting RPCSEC_GSS
- */
- svcCmdToFileName := make(map[string]string)
- for _, line := range strings.Split(string(out), "\n") {
- fields := strings.Fields(line)
- if len(fields) == 0 || fields[0] == "" || len(fields) < 5 {
- continue
- }
- if fields[1] == "not-found" {
- continue
- }
- if fields[2] == "inactive" {
- continue
- }
- svc := fields[0]
- svcCmd := "sudo journalctl --no-pager --output=cat -u " + svc
- svcFileName := svc + ".out.log"
- svcCmdToFileName[svcCmd] = svcFileName
- }
- for cmd, fileName := range svcCmdToFileName {
- if !rateLimiter.Allow() {
- ts.cfg.Logger.Debug("waiting for rate limiter before fetching file")
- werr := rateLimiter.Wait(context.Background())
- ts.cfg.Logger.Debug("waited for rate limiter", zap.Error(werr))
- }
- out, oerr := sh.Run(cmd, sshOptLog)
- if oerr != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to run command %q for %q (error %v)",
- listCmd,
- instID,
- oerr,
- ))
- continue
- }
-
- fpath := filepath.Join(logsDir, shorten(ts.cfg.Logger, pfx+fileName))
- f, err := os.Create(fpath)
- if err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to create a file %q for %q (error %v)",
- fpath,
- instID,
- err,
- ))
- continue
- }
- if _, err = f.Write(out); err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to write to a file %q for %q (error %v)",
- fpath,
- instID,
- err,
- ))
- f.Close()
- continue
- }
- f.Close()
- ts.cfg.Logger.Debug("wrote", zap.String("file-path", fpath))
- data.paths = append(data.paths, fpath)
- }
- }
-
- if !rateLimiter.Allow() {
- ts.cfg.Logger.Debug("waiting for rate limiter before fetching file")
- werr := rateLimiter.Wait(context.Background())
- ts.cfg.Logger.Debug("waited for rate limiter", zap.Error(werr))
- }
- // https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/troubleshooting.md#ipamd-debugging-commands
- // https://github.com/aws/amazon-vpc-cni-k8s/blob/master/scripts/aws-cni-support.sh
- ts.cfg.Logger.Info("fetching ENI information", zap.String("instance-id", instID))
- eniCmd := "curl -s http://localhost:61679/v1/enis"
- out, oerr = sh.Run(eniCmd, sshOptLog)
- if oerr != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to run command %q for %q (error %v)",
- eniCmd,
- instID,
- oerr,
- ))
- } else {
- v1ENIOutputPath := filepath.Join(logsDir, shorten(ts.cfg.Logger, pfx+"v1-enis.out.log"))
- f, err := os.Create(v1ENIOutputPath)
- if err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to create a file %q for %q (error %v)",
- v1ENIOutputPath,
- instID,
- err,
- ))
- } else {
- if _, err = f.Write(out); err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to write to a file %q for %q (error %v)",
- v1ENIOutputPath,
- instID,
- err,
- ))
- } else {
- ts.cfg.Logger.Debug("wrote", zap.String("file-path", v1ENIOutputPath))
- data.paths = append(data.paths, v1ENIOutputPath)
- }
- f.Close()
- }
- }
-
- ts.cfg.Logger.Info("running /opt/cni/bin/aws-cni-support.sh", zap.String("instance-id", instID))
- cniCmd := "sudo /opt/cni/bin/aws-cni-support.sh || true"
- out, oerr = sh.Run(cniCmd, sshOptLog)
- if oerr != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to run command %q for %q (error %v)",
- cniCmd,
- instID,
- oerr,
- ))
- } else {
- ts.cfg.Logger.Info("ran /opt/cni/bin/aws-cni-support.sh", zap.String("instance-id", instID), zap.String("output", string(out)))
- }
-
- if !rateLimiter.Allow() {
- ts.cfg.Logger.Debug("waiting for rate limiter before fetching file")
- werr := rateLimiter.Wait(context.Background())
- ts.cfg.Logger.Debug("waited for rate limiter", zap.Error(werr))
- }
- ts.cfg.Logger.Info("listing /var/log", zap.String("instance-id", instID))
- findCmd := "sudo find /var/log ! -type d"
- out, oerr = sh.Run(findCmd, sshOptLog, ssh.WithRetry(5, 3*time.Second))
- if oerr != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to run command %q for %q (error %v)",
- findCmd,
- instID,
- oerr,
- ))
- } else {
- varLogPaths := make(map[string]string)
- for _, line := range strings.Split(string(out), "\n") {
- if len(line) == 0 {
- // last value
- continue
- }
- logCmd := "sudo cat " + line
- logPath := filepath.Base(line)
- varLogPaths[logCmd] = logPath
- }
- for cmd, logPath := range varLogPaths {
- if !rateLimiter.Allow() {
- ts.cfg.Logger.Debug("waiting for rate limiter before fetching file")
- werr := rateLimiter.Wait(context.Background())
- ts.cfg.Logger.Debug("waited for rate limiter", zap.Error(werr))
- }
- // e.g. "read tcp 10.119.223.210:58688->54.184.39.156:22: read: connection timed out"
- out, oerr := sh.Run(cmd, sshOptLog, ssh.WithRetry(2, 3*time.Second))
- if oerr != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to run command %q for %q (error %v)",
- cmd,
- instID,
- oerr,
- ))
- continue
- }
-
- fpath := filepath.Join(logsDir, shorten(ts.cfg.Logger, pfx+logPath))
- f, err := os.Create(fpath)
- if err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to create a file %q for %q (error %v)",
- fpath,
- instID,
- err,
- ))
- continue
- }
- if _, err = f.Write(out); err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to write to a file %q for %q (error %v)",
- fpath,
- instID,
- err,
- ))
- f.Close()
- continue
- }
- f.Close()
- ts.cfg.Logger.Debug("wrote", zap.String("file-path", fpath))
- data.paths = append(data.paths, fpath)
- }
- }
- rch <- data
- }(instID, logsDir, pfx, cur)
- }
- }
-
- ts.cfg.Logger.Info("waiting for log fetcher goroutines", zap.Int("waits", waits))
- total := 0
- for i := 0; i < waits; i++ {
- var data instanceLogs
- select {
- case data = <-rch:
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("exiting fetch logger")
- ts.cfg.EKSConfig.Sync()
- return nil
- }
- if len(data.errs) > 0 {
- ts.cfg.Logger.Warn("failed to fetch logs, but keeping whatever available",
- zap.String("mng-name", data.mngName),
- zap.String("instance-id", data.instanceID),
- zap.Strings("errors", data.errs),
- )
- }
- cur, ok := ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[data.mngName]
- if !ok {
- return fmt.Errorf("EKS Managed Node Group name %q is unknown", data.mngName)
- }
- if cur.Logs == nil {
- cur.Logs = make(map[string][]string)
- }
-
- // existing logs are already written out to disk, merge/list them all
- var logs []string
- logs, ok = cur.Logs[data.instanceID]
- if ok {
- ts.cfg.Logger.Warn("managed node group already has existing logs; merging",
- zap.String("mng-name", data.mngName),
- zap.String("instance-id", data.instanceID),
- )
- }
- all := make(map[string]struct{})
- for _, v := range logs {
- all[v] = struct{}{}
- }
- for _, v := range data.paths {
- all[v] = struct{}{}
- }
- logs = make([]string, 0, len(all))
- for k := range all {
- logs = append(logs, k)
- }
- sort.Strings(logs)
- cur.Logs[data.instanceID] = logs
- files := len(logs)
-
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[data.mngName] = cur
- ts.cfg.EKSConfig.Sync()
-
- total += files
- ts.cfg.Logger.Info("wrote log files",
- zap.String("instance-id", data.instanceID),
- zap.Int("files", files),
- zap.Int("total-downloaded-files", total),
- zap.Int("total-goroutines-to-wait", waits),
- zap.Int("current-waited-goroutines", i),
- )
- }
-
- ts.cfg.Logger.Info("wrote all log files",
- zap.String("log-dir", logsDir),
- zap.Int("total-downloaded-files", total),
- )
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-type instanceLogs struct {
- mngName string
- instanceID string
- paths []string
- errs []string
-}
-
-func (ts *tester) DownloadClusterLogs(artifactDir string) error {
- err := ts.FetchLogs()
- if err != nil {
- return err
- }
-
- ts.logsMu.RLock()
- defer ts.logsMu.RUnlock()
-
- for _, cur := range ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs {
- for _, fpaths := range cur.Logs {
- for _, fpath := range fpaths {
- newPath := filepath.Join(artifactDir, filepath.Base(fpath))
- if err := fileutil.Copy(fpath, newPath); err != nil {
- return err
- }
- }
- }
- }
-
- return fileutil.Copy(
- ts.cfg.EKSConfig.ConfigPath,
- filepath.Join(artifactDir, filepath.Base(ts.cfg.EKSConfig.ConfigPath)),
- )
-}
-
-func shorten(lg *zap.Logger, name string) string {
- if len(name) < 240 {
- return name
- }
-
- ext := filepath.Ext(name)
- oldName := name
-
- name = name[:230] + randutil.String(5) + ext
- lg.Info("file name too long; renamed", zap.String("old", oldName), zap.String("new", name))
- return name
-}
diff --git a/eks/mng/mng.go b/eks/mng/mng.go
deleted file mode 100644
index 6197b771c..000000000
--- a/eks/mng/mng.go
+++ /dev/null
@@ -1,296 +0,0 @@
-// Package mng defines AWS EKS Managed Node Group configuration.
-package mng
-
-import (
- "errors"
- "fmt"
- "io"
- "reflect"
- "strings"
- "sync"
- "time"
-
- "github.com/aws/aws-k8s-tester/eks/mng/scale"
- version_upgrade "github.com/aws/aws-k8s-tester/eks/mng/version-upgrade"
- "github.com/aws/aws-k8s-tester/eks/mng/wait"
- "github.com/aws/aws-k8s-tester/eksconfig"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- aws_asg_v2 "github.com/aws/aws-sdk-go-v2/service/autoscaling"
- aws_ec2_v2 "github.com/aws/aws-sdk-go-v2/service/ec2"
- aws_eks_v2 "github.com/aws/aws-sdk-go-v2/service/eks"
- aws_iam_v2 "github.com/aws/aws-sdk-go-v2/service/iam"
- "github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface"
- "github.com/aws/aws-sdk-go/service/eks/eksiface"
- "go.uber.org/zap"
-)
-
-// Config defines Managed Node Group configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
-
- IAMAPIV2 *aws_iam_v2.Client
- EC2APIV2 *aws_ec2_v2.Client
- ASGAPIV2 *aws_asg_v2.Client
- EKSAPI eksiface.EKSAPI
- EKSAPIV2 *aws_eks_v2.Client
-
- CFNAPI cloudformationiface.CloudFormationAPI
-}
-
-// Tester implements EKS "Managed Node Group" for "kubetest2" Deployer.
-// ref. https://github.com/kubernetes/test-infra/blob/master/kubetest2/pkg/types/types.go
-// ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
-// ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html
-type Tester interface {
- // Name returns the name of the tester.
- Name() string
- // Create creates EKS "Managed Node Group", and waits for completion.
- Create() error
- // Delete deletes all EKS "Managed Node Group" resources.
- Delete() error
- // Scale runs all scale up/down operations.
- Scale() error
- // UpgradeVersion upgrades EKS "Managed Node Group" version, and waits for completion.
- UpgradeVersion() error
-
- // FetchLogs fetches logs from all worker nodes.
- FetchLogs() error
- // DownloadClusterLogs dumps all logs to artifact directory.
- // Let default kubetest log dumper handle all artifact uploads.
- // See https://github.com/kubernetes/test-infra/pull/9811/files#r225776067.
- DownloadClusterLogs(artifactDir string) error
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-// New creates a new Job tester.
-func New(cfg Config) Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{
- cfg: cfg,
- nodeWaiter: wait.New(wait.Config{
- Logger: cfg.Logger,
- LogWriter: cfg.LogWriter,
- Stopc: cfg.Stopc,
- EKSConfig: cfg.EKSConfig,
- K8SClient: cfg.K8SClient,
- EC2APIV2: cfg.EC2APIV2,
- ASGAPIV2: cfg.ASGAPIV2,
- EKSAPI: cfg.EKSAPI,
- }),
- scaler: scale.New(scale.Config{
- Logger: cfg.Logger,
- LogWriter: cfg.LogWriter,
- Stopc: cfg.Stopc,
- EKSConfig: cfg.EKSConfig,
- EKSAPI: cfg.EKSAPI,
- }),
- versionUpgrader: version_upgrade.New(version_upgrade.Config{
- Logger: cfg.Logger,
- LogWriter: cfg.LogWriter,
- Stopc: cfg.Stopc,
- EKSConfig: cfg.EKSConfig,
- K8SClient: cfg.K8SClient,
- EKSAPI: cfg.EKSAPI,
- }),
- logsMu: new(sync.RWMutex),
- deleteRequested: make(map[string]struct{}),
- }
-}
-
-type tester struct {
- cfg Config
- nodeWaiter wait.NodeWaiter
- scaler scale.Scaler
- versionUpgrader version_upgrade.Upgrader
- logsMu *sync.RWMutex
- deleteRequested map[string]struct{}
-}
-
-func (ts *tester) Create() (err error) {
- if !ts.cfg.EKSConfig.IsEnabledAddOnManagedNodeGroups() {
- ts.cfg.Logger.Info("managed node group is disabled; skipping creation")
- return nil
- }
- if ts.cfg.EKSConfig.AddOnManagedNodeGroups.Created {
- ts.cfg.Logger.Info("managed node group is already created; skipping creation")
- return nil
- }
- if len(ts.cfg.EKSConfig.VPC.PublicSubnetIDs) == 0 {
- return errors.New("empty EKSConfig.VPC.PublicSubnetIDs")
- }
-
- ts.cfg.Logger.Info("starting tester.Create", zap.String("tester", pkgName))
- createStart := time.Now()
- defer func() {
- createEnd := time.Now()
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- if err = ts.createRole(); err != nil {
- return err
- }
- if err = ts.createMNGs(); err != nil {
- return err
- }
- for mngName := range ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs {
- if err = ts.authorizeSecurityGroups(mngName); err != nil {
- return err
- }
- }
-
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.Created = true
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) Scale() (err error) {
- for mngName := range ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs {
- if err = ts.scaler.Scale(mngName); err != nil {
- return err
- }
- if err = ts.nodeWaiter.Wait(mngName, 3); err != nil {
- return err
- }
- }
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) UpgradeVersion() (err error) {
- if !ts.cfg.EKSConfig.IsEnabledAddOnManagedNodeGroups() {
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnManagedNodeGroups.Created {
- ts.cfg.Logger.Info("ManagedNodeGroup is not created; skipping upgrade")
- return nil
- }
-
- for mngName, cur := range ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs {
- if cur.VersionUpgrade == nil || !cur.VersionUpgrade.Enable {
- continue
- }
- if err = ts.versionUpgrader.Upgrade(mngName); err != nil {
- return err
- }
- if err = ts.nodeWaiter.Wait(mngName, 3); err != nil {
- return err
- }
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) Delete() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnManagedNodeGroups() {
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnManagedNodeGroups.Created {
- ts.cfg.Logger.Info("ManagedNodeGroup is not created; skipping deletion")
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Delete", zap.String("tester", pkgName))
- deleteStart := time.Now()
- defer func() {
- deleteEnd := time.Now()
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.TimeFrameDelete = timeutil.NewTimeFrame(deleteStart, deleteEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- var errs []string
- var err error
-
- for name := range ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs {
- err = ts.revokeSecurityGroups(name)
- if err != nil {
- errs = append(errs, err.Error())
- }
- }
-
- failedMNGs := make(map[string]struct{})
- for name := range ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs {
- for i := 0; i < 5; i++ { // retry, leakly ENI may take awhile to be deleted
- derr := ts.deleteMNG(name)
- if derr == nil {
- ts.cfg.Logger.Info("successfully deleted mng", zap.String("name", name))
- delete(failedMNGs, name)
- break
- }
- ts.cfg.Logger.Warn("failed to delete mng; retrying", zap.String("name", name), zap.Error(derr))
- failedMNGs[name] = struct{}{}
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("aborted")
- return nil
- case <-time.After(time.Minute):
- }
- }
- }
-
- waitDur := time.Minute
- ts.cfg.Logger.Info("sleeping after MNG deletion", zap.Duration("wait", waitDur))
- time.Sleep(waitDur)
-
- for name := range ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs {
- time.Sleep(10 * time.Second)
- if ok := ts.deleteENIs(name); ok {
- time.Sleep(10 * time.Second)
- }
- }
-
- err = nil
- for name := range failedMNGs {
- ts.cfg.Logger.Warn("retrying mng delete after failure", zap.String("name", name))
- var derr error
- for i := 0; i < 5; i++ { // retry, leakly ENI may take awhile to be deleted
- derr = ts.deleteMNG(name)
- if derr == nil {
- ts.cfg.Logger.Info("successfully deleted mng (previously failed for delete)", zap.String("name", name))
- delete(failedMNGs, name)
- break
- }
- ts.cfg.Logger.Warn("failed to retry-delete mng; retrying", zap.String("name", name), zap.Error(derr))
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("aborted")
- return nil
- case <-time.After(time.Minute):
- }
- }
- if derr != nil {
- if err == nil {
- err = derr
- } else {
- err = fmt.Errorf("%v; %v", err, derr)
- }
- }
- }
- if err != nil {
- errs = append(errs, err.Error())
- }
-
- // must be run after deleting node group
- // otherwise, "Cannot delete entity, must remove roles from instance profile first. (Service: AmazonIdentityManagement; Status Code: 409; Error Code: DeleteConflict; Request ID: 197f795b-1003-4386-81cc-44a926c42be7)"
- if err := ts.deleteRole(); err != nil {
- ts.cfg.Logger.Warn("failed to delete mng role", zap.Error(err))
- errs = append(errs, err.Error())
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.Created = false
- ts.cfg.EKSConfig.Sync()
- return nil
-}
diff --git a/eks/mng/mng_asg.go b/eks/mng/mng_asg.go
deleted file mode 100644
index be6c94668..000000000
--- a/eks/mng/mng_asg.go
+++ /dev/null
@@ -1,290 +0,0 @@
-package mng
-
-import (
- "context"
- "errors"
- "fmt"
- "sort"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/eks/mng/wait"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-k8s-tester/pkg/user"
- "github.com/aws/aws-k8s-tester/version"
- aws_v2 "github.com/aws/aws-sdk-go-v2/aws"
- aws_eks "github.com/aws/aws-sdk-go/service/eks"
- smithy "github.com/aws/smithy-go"
- "github.com/dustin/go-humanize"
- "go.uber.org/zap"
-)
-
-func (ts *tester) createMNGs() error {
- now := time.Now()
- tss, err := ts._createMNGs()
- if err != nil {
- return err
- }
- if err = ts.waitForMNGs(now, tss); err != nil {
- return err
- }
- return nil
-}
-
-func (ts *tester) deleteMNG(mngName string) error {
- cur, ok := ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName]
- if !ok {
- return fmt.Errorf("MNGs[%q] not found; cannot delete managed node group", mngName)
- }
- if cur.RemoteAccessSecurityGroupID == "" {
- return fmt.Errorf("MNG[%q] security group ID not found; cannot delete managed node group", mngName)
- }
- ts.cfg.Logger.Info("deleting MNG/ASG", zap.String("mng-name", mngName))
-
- ts.cfg.Logger.Info("deleting managed node group using EKS API", zap.String("name", mngName))
- _, err := ts.cfg.EKSAPI.DeleteNodegroup(&aws_eks.DeleteNodegroupInput{
- ClusterName: aws_v2.String(ts.cfg.EKSConfig.Name),
- NodegroupName: aws_v2.String(mngName),
- })
- if err != nil {
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- ts.cfg.EKSConfig.Status.DeletedResources[mngName] = "AddOnManagedNodeGroups.Name"
- ts.cfg.EKSConfig.Sync()
- return nil
- }
- }
- return err
- }
-
- timeStart := time.Now()
-
- ts.cfg.Logger.Info("waiting for delete managed node group using EKS API", zap.String("name", mngName))
- initialWait, timeout := 2*time.Minute, 15*time.Minute
- if len(cur.Instances) > 50 {
- initialWait, timeout = 3*time.Minute, 20*time.Minute
- }
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- ch := wait.Poll(
- ctx,
- ts.cfg.Stopc,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.EKSAPI,
- ts.cfg.EKSConfig.Name,
- mngName,
- wait.ManagedNodeGroupStatusDELETEDORNOTEXIST,
- initialWait,
- 20*time.Second,
- )
- for sv := range ch {
- serr := ts.setStatus(sv)
- if serr != nil {
- cancel()
- return serr
- }
- err = sv.Error
- }
- cancel()
- if err != nil {
- return err
- }
-
- timeEnd := time.Now()
- cur.TimeFrameDelete = timeutil.NewTimeFrame(timeStart, timeEnd)
- if err != nil {
- cur.Status = fmt.Sprintf("MNGs[%q] failed to delete %v", mngName, err)
- ts.cfg.Logger.Warn("failed to delete managed node group", zap.String("name", mngName), zap.Error(err))
- } else {
- cur.Status = wait.ManagedNodeGroupStatusDELETEDORNOTEXIST
- ts.cfg.Logger.Info("deleted managed node group", zap.String("name", mngName))
- }
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName] = cur
- ts.cfg.EKSConfig.Sync()
-
- return nil
-}
-
-func (ts *tester) _createMNGs() (tss tupleTimes, err error) {
- ts.cfg.Logger.Info("creating MNGs")
-
- for mngName, cur := range ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs {
- ts.cfg.Logger.Info("requesting MNG creation", zap.String("mng-name", mngName))
-
- createInput := aws_eks.CreateNodegroupInput{
- ClusterName: aws_v2.String(ts.cfg.EKSConfig.Name),
- NodegroupName: aws_v2.String(cur.Name),
- NodeRole: aws_v2.String(ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.ARN),
- AmiType: aws_v2.String(cur.AMIType),
- DiskSize: aws_v2.Int64(int64(cur.VolumeSize)),
- InstanceTypes: aws_v2.StringSlice(cur.InstanceTypes),
- RemoteAccess: &aws_eks.RemoteAccessConfig{
- Ec2SshKey: aws_v2.String(ts.cfg.EKSConfig.RemoteAccessKeyName),
- },
- ScalingConfig: &aws_eks.NodegroupScalingConfig{
- MinSize: aws_v2.Int64(int64(cur.ASGMinSize)),
- DesiredSize: aws_v2.Int64(int64(cur.ASGDesiredCapacity)),
- MaxSize: aws_v2.Int64(int64(cur.ASGMaxSize)),
- },
- Subnets: aws_v2.StringSlice(ts.cfg.EKSConfig.VPC.PublicSubnetIDs),
- Tags: map[string]*string{
- "Kind": aws_v2.String("aws-k8s-tester"),
- "aws-k8s-tester-version": aws_v2.String(version.ReleaseVersion),
- "User": aws_v2.String(user.Get()),
- },
- Labels: map[string]*string{
- "NodeType": aws_v2.String("regular"),
- "AMIType": aws_v2.String(cur.AMIType),
- "NGType": aws_v2.String("managed"),
- "NGName": aws_v2.String(cur.Name),
- },
- }
- for k, v := range cur.Tags {
- createInput.Tags[k] = aws_v2.String(v)
- ts.cfg.Logger.Info("added EKS tag", zap.String("key", k), zap.String("value", v))
- }
- if cur.ReleaseVersion != "" {
- createInput.ReleaseVersion = aws_v2.String(cur.ReleaseVersion)
- ts.cfg.Logger.Info("added EKS release version", zap.String("version", cur.ReleaseVersion))
- }
- timeStart := time.Now()
- req, _ := ts.cfg.EKSAPI.CreateNodegroupRequest(&createInput)
- if ts.cfg.EKSConfig.AddOnManagedNodeGroups.RequestHeaderKey != "" && ts.cfg.EKSConfig.AddOnManagedNodeGroups.RequestHeaderValue != "" {
- req.HTTPRequest.Header[ts.cfg.EKSConfig.AddOnManagedNodeGroups.RequestHeaderKey] = []string{ts.cfg.EKSConfig.AddOnManagedNodeGroups.RequestHeaderValue}
- ts.cfg.Logger.Info("set request header for EKS managed node group create request",
- zap.String("key", ts.cfg.EKSConfig.AddOnManagedNodeGroups.RequestHeaderKey),
- zap.String("value", ts.cfg.EKSConfig.AddOnManagedNodeGroups.RequestHeaderValue),
- )
- }
-
- err := req.Send()
- if err != nil {
- ts.cfg.Logger.Warn("failed to created MNG", zap.Error(err))
- return nil, fmt.Errorf("MNGs[%q] create request failed (%v)", cur.Name, err)
- }
-
- cur.TimeFrameCreate = timeutil.NewTimeFrame(timeStart, time.Now())
- cur.CreateRequested = true
- cur.Status = aws_eks.NodegroupStatusCreating
- cur.Instances = make(map[string]ec2config.Instance)
- cur.Logs = make(map[string][]string)
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName] = cur
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.Created = true
- ts.cfg.EKSConfig.Sync()
- ts.cfg.Logger.Info("sent create managed node group request")
-
- tss = append(tss, tupleTime{ts: time.Now(), name: mngName})
- }
-
- sort.Sort(sort.Reverse(tss))
- ts.cfg.Logger.Info("created MNGs")
- return tss, nil
-}
-
-func (ts *tester) waitForMNGs(now time.Time, tss tupleTimes) (err error) {
- ts.cfg.Logger.Info("waiting for MNGs")
-
- for _, tv := range tss {
- mngName := tv.name
- cur, ok := ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName]
- if !ok {
- return fmt.Errorf("MNG name %q not found after creation", mngName)
- }
-
- select {
- case <-time.After(10 * time.Second):
- case <-ts.cfg.Stopc:
- return errors.New("stopped")
- }
-
- ts.cfg.Logger.Info("waiting for MNG", zap.String("mng-name", mngName))
-
- timeStart := time.Now()
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
- ch := wait.Poll(
- ctx,
- ts.cfg.Stopc,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.EKSAPI,
- ts.cfg.EKSConfig.Name,
- mngName,
- aws_eks.NodegroupStatusActive,
- time.Minute,
- 20*time.Second,
- )
- for sv := range ch {
- serr := ts.setStatus(sv)
- if serr != nil {
- cancel()
- return serr
- }
- err = sv.Error
- }
- cancel()
- if err != nil {
- return err
- }
-
- cur, ok = ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName]
- if !ok {
- return fmt.Errorf("MNGs[%q] not found after creation", mngName)
- }
- timeEnd := time.Now()
- cur.TimeFrameCreate = timeutil.NewTimeFrame(cur.TimeFrameCreate.StartUTC, cur.TimeFrameCreate.EndUTC.Add(timeEnd.Sub(timeStart)))
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName] = cur
- ts.cfg.EKSConfig.Sync()
-
- timeStart = time.Now()
- if err := ts.nodeWaiter.Wait(mngName, 10); err != nil {
- return err
- }
- timeEnd = time.Now()
-
- cur, ok = ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName]
- if !ok {
- return fmt.Errorf("MNGs[%q] not found after creation", mngName)
- }
- cur.TimeFrameCreate = timeutil.NewTimeFrame(timeStart, timeEnd)
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName] = cur
- ts.cfg.EKSConfig.Sync()
-
- ts.cfg.Logger.Info("created a managed node group",
- zap.String("mng-name", cur.Name),
- zap.String("started", humanize.RelTime(now, time.Now(), "ago", "from now")),
- )
- }
-
- ts.cfg.Logger.Info("waited for MNGs")
- return nil
-}
-
-func (ts *tester) setStatus(sv wait.ManagedNodeGroupStatus) (err error) {
- name := sv.NodeGroupName
- if name == "" {
- return errors.New("EKS Managed Node Group empty name")
- }
- cur, ok := ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[name]
- if !ok {
- return fmt.Errorf("EKS MNGs[%q] not found", name)
- }
-
- if sv.NodeGroup == nil {
- if sv.Error != nil {
- cur.Status = fmt.Sprintf("%q failed with error %v", sv.NodeGroupName, sv.Error)
- } else {
- cur.Status = wait.ManagedNodeGroupStatusDELETEDORNOTEXIST
- }
- } else {
- cur.Status = aws_v2.ToString(sv.NodeGroup.Status)
- if sv.NodeGroup.Resources != nil && cur.RemoteAccessSecurityGroupID == "" {
- cur.RemoteAccessSecurityGroupID = aws_v2.ToString(sv.NodeGroup.Resources.RemoteAccessSecurityGroup)
- }
- }
-
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[name] = cur
- ts.cfg.EKSConfig.Sync()
- return nil
-}
diff --git a/eks/mng/role.go b/eks/mng/role.go
deleted file mode 100644
index 4a081fdee..000000000
--- a/eks/mng/role.go
+++ /dev/null
@@ -1,537 +0,0 @@
-package mng
-
-import (
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "strings"
- "time"
-
- aws_iam "github.com/aws/aws-k8s-tester/pkg/aws/iam"
- aws_v2 "github.com/aws/aws-sdk-go-v2/aws"
- aws_iam_v2 "github.com/aws/aws-sdk-go-v2/service/iam"
- smithy "github.com/aws/smithy-go"
- "go.uber.org/zap"
-)
-
-// see https://github.com/aws/aws-k8s-tester/blob/v1.6.0/eks/mng/role.go for CloudFormation based workflow
-
-func (ts *tester) createRole() error {
- if !ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Create {
- ts.cfg.Logger.Info("AddOnManagedNodeGroups.Role.Create false; skipping creation")
- return aws_iam.ValidateV2(
- ts.cfg.Logger,
- ts.cfg.IAMAPIV2,
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name,
- []string{"ec2.amazonaws.com", "eks.amazonaws.com"},
- []string{
- "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
- "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
- "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
- },
- )
- }
- if ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.ARN != "" {
- ts.cfg.Logger.Info("role already created; no need to create a new one")
- return nil
- }
- if ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name == "" {
- return errors.New("cannot create a cluster role with an empty AddOnManagedNodeGroups.Role.Name")
- }
-
- if err := ts._createRole(); err != nil {
- return err
- }
- if err := ts.createPolicy(); err != nil {
- return err
- }
- if err := ts.attachPolicy(); err != nil {
- return err
- }
-
- ts.cfg.Logger.Info("created a new role and attached policy",
- zap.String("role-arn", ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.ARN),
- zap.String("role-name", ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name),
- )
- return nil
-}
-
-func (ts *tester) deleteRole() error {
- if !ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Create {
- ts.cfg.Logger.Info("Role.Create false; skipping deletion")
- return nil
- }
-
- var errs []string
-
- if err := ts.detachPolicy(); err != nil {
- ts.cfg.Logger.Warn("failed to detach policy", zap.Error(err))
- errs = append(errs, err.Error())
- }
- if err := ts.deletePolicy(); err != nil {
- ts.cfg.Logger.Warn("failed to delete policy", zap.Error(err))
- errs = append(errs, err.Error())
- }
- if err := ts._deleteRole(); err != nil {
- ts.cfg.Logger.Warn("failed to delete role", zap.Error(err))
- errs = append(errs, err.Error())
- }
-
- if len(errs) == 0 {
- ts.cfg.Logger.Info("successfully deleted role",
- zap.String("role-arn", ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.ARN),
- zap.String("role-name", ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name),
- )
- return nil
- }
- return errors.New(strings.Join(errs, ","))
-}
-
-func (ts *tester) _createRole() error {
- ts.cfg.Logger.Info("creating role", zap.String("name", ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name))
- out, err := ts.cfg.IAMAPIV2.CreateRole(
- context.Background(),
- &aws_iam_v2.CreateRoleInput{
- RoleName: aws_v2.String(ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name),
- Path: aws_v2.String("/"),
- AssumeRolePolicyDocument: aws_v2.String(createAssumeRolePolicyDocument(ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.ServicePrincipals)),
- },
- )
- if err != nil {
- return err
- }
-
- ts.cfg.Logger.Info("created role")
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.ARN = aws_v2.ToString(out.Role.Arn)
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) _deleteRole() error {
- ts.cfg.Logger.Info("deleting role", zap.String("name", ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name))
- if _, ok := ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name]; ok {
- return nil
- }
-
- _, err := ts.cfg.IAMAPIV2.DeleteRole(
- context.Background(),
- &aws_iam_v2.DeleteRoleInput{
- RoleName: aws_v2.String(ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name),
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to delete cluster role", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name] = "AddOnManagedNodeGroups.Role.Name"
- ts.cfg.EKSConfig.Sync()
- return nil
- }
- }
- return err
- }
-
- ts.cfg.Logger.Info("deleted role")
- ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name] = "AddOnManagedNodeGroups.Role.Name"
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html
-// https://github.com/kubernetes-sigs/aws-alb-ingress-controller/blob/master/docs/examples/iam-policy.json
-// https://github.com/aws/eks-charts/tree/master/stable/appmesh-controller
-func (ts *tester) createPolicy() error {
- if ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.PolicyName == "" {
- return errors.New("emtpy PolicyName")
- }
- ts.cfg.Logger.Info("creating policy", zap.String("name", ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.PolicyName))
- pout, err := ts.cfg.IAMAPIV2.CreatePolicy(
- context.Background(),
- &aws_iam_v2.CreatePolicyInput{
- PolicyName: aws_v2.String(ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.PolicyName),
- PolicyDocument: aws_v2.String(createRolePolicyDocument(ts.cfg.EKSConfig.Partition, ts.cfg.EKSConfig.S3.BucketName)),
- },
- )
- if err != nil {
- return err
- }
-
- ts.cfg.Logger.Info("created policy")
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.PolicyARN = aws_v2.ToString(pout.Policy.Arn)
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) deletePolicy() error {
- ts.cfg.Logger.Info("deleting policy")
- if _, ok := ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.PolicyARN]; ok {
- return nil
- }
-
- _, err := ts.cfg.IAMAPIV2.DeletePolicy(
- context.Background(),
- &aws_iam_v2.DeletePolicyInput{
- PolicyArn: aws_v2.String(ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.PolicyARN),
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to delete policy", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.PolicyARN] = "AddOnManagedNodeGroups.Role.PolicyARN"
- ts.cfg.EKSConfig.Sync()
- return nil
- }
- }
- return err
- }
-
- ts.cfg.Logger.Info("deleted policy")
- ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.PolicyARN] = "AddOnManagedNodeGroups.Role.PolicyARN"
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) attachPolicy() error {
- ts.cfg.Logger.Info("attaching policies")
-
- _, err := ts.cfg.IAMAPIV2.AttachRolePolicy(
- context.Background(),
- &aws_iam_v2.AttachRolePolicyInput{
- RoleName: aws_v2.String(ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name),
- PolicyArn: aws_v2.String(ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.PolicyARN),
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to attach policy", zap.String("arn", ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.PolicyARN), zap.Error(err))
- return err
- }
- ts.cfg.Logger.Info("attached policy arn", zap.String("policy-arn", ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.PolicyARN))
-
- for _, arn := range ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.ManagedPolicyARNs {
- time.Sleep(3 * time.Second)
- ts.cfg.Logger.Info("attaching managed policy arn", zap.String("arn", arn))
- _, err := ts.cfg.IAMAPIV2.AttachRolePolicy(
- context.Background(),
- &aws_iam_v2.AttachRolePolicyInput{
- RoleName: aws_v2.String(ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name),
- PolicyArn: aws_v2.String(arn),
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to attach policy", zap.String("arn", arn), zap.Error(err))
- return err
- }
- }
-
- ts.cfg.Logger.Info("attached policies")
- return nil
-}
-
-func (ts *tester) detachPolicy() error {
- ts.cfg.Logger.Info("detaching policies")
-
- _, err := ts.cfg.IAMAPIV2.DetachRolePolicy(
- context.Background(),
- &aws_iam_v2.DetachRolePolicyInput{
- RoleName: aws_v2.String(ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name),
- PolicyArn: aws_v2.String(ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.PolicyARN),
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to detach policy", zap.String("arn", ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.PolicyARN), zap.Error(err))
- return err
- }
- for _, arn := range ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.ManagedPolicyARNs {
- time.Sleep(3 * time.Second)
- _, err := ts.cfg.IAMAPIV2.DetachRolePolicy(
- context.Background(),
- &aws_iam_v2.DetachRolePolicyInput{
- RoleName: aws_v2.String(ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name),
- PolicyArn: aws_v2.String(arn),
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to detach policy", zap.String("arn", arn), zap.Error(err))
- return err
- }
- }
-
- ts.cfg.Logger.Info("detached policies")
- return nil
-}
-
-func createAssumeRolePolicyDocument(sps []string) string {
- p := aws_iam.PolicyDocument{
- Version: "2012-10-17",
- Statement: createStatementEntriesForAssumeRole(sps),
- }
- b, err := json.Marshal(p)
- if err != nil {
- panic(err)
- }
- return string(b)
-}
-
-func createRolePolicyDocument(partition string, bucketName string) string {
- p := aws_iam.PolicyDocument{
- Version: "2012-10-17",
- Statement: createStatementEntriesForRolePolicyDocument(partition, bucketName),
- }
- b, err := json.Marshal(p)
- if err != nil {
- panic(err)
- }
- return string(b)
-}
-
-func createStatementEntriesForAssumeRole(sps []string) []aws_iam.StatementEntry {
- return []aws_iam.StatementEntry{
- {
- Effect: "Allow",
- Principal: &aws_iam.PrincipalEntry{
- Service: sps,
- },
- Action: []string{
- "sts:AssumeRole",
- },
- },
- }
-}
-
-// TODO: update based on add-on setups
-// https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html
-// https://github.com/kubernetes-sigs/aws-alb-ingress-controller/blob/master/docs/examples/iam-policy.json
-// https://github.com/aws/eks-charts/tree/master/stable/appmesh-controller
-func createStatementEntriesForRolePolicyDocument(partition string, bucketName string) []aws_iam.StatementEntry {
- return []aws_iam.StatementEntry{
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "acm:DescribeCertificate",
- "acm:ListCertificates",
- "acm:GetCertificate",
- },
- },
- // arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "ec2:AttachVolume",
- "ec2:AuthorizeSecurityGroupIngress",
- "ec2:CreateSecurityGroup",
- "ec2:CreateSnapshot",
- "ec2:CreateTags",
- "ec2:CreateVolume",
- "ec2:DeleteSecurityGroup",
- "ec2:DeleteSnapshot",
- "ec2:DeleteTags",
- "ec2:DeleteVolume",
- "ec2:DescribeAccountAttributes",
- "ec2:DescribeAddresses",
- "ec2:DescribeInstanceStatus",
- "ec2:DescribeInstances",
- "ec2:DescribeInternetGateways",
- "ec2:DescribeNetworkInterfaces",
- "ec2:DescribeRouteTables",
- "ec2:DescribeSecurityGroups",
- "ec2:DescribeSnapshots",
- "ec2:DescribeSubnets",
- "ec2:DescribeTags",
- "ec2:DescribeVolumes",
- "ec2:DescribeVolumes",
- "ec2:DescribeVolumesModifications",
- "ec2:DescribeVpcs",
- "ec2:DetachVolume",
- "ec2:ModifyInstanceAttribute",
- "ec2:ModifyNetworkInterfaceAttribute",
- "ec2:RevokeSecurityGroupIngress",
- "eks:DescribeCluster",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "elasticloadbalancing:AddListenerCertificates",
- "elasticloadbalancing:AddTags",
- "elasticloadbalancing:CreateListener",
- "elasticloadbalancing:CreateLoadBalancer",
- "elasticloadbalancing:CreateRule",
- "elasticloadbalancing:CreateTargetGroup",
- "elasticloadbalancing:DeleteListener",
- "elasticloadbalancing:DeleteLoadBalancer",
- "elasticloadbalancing:DeleteRule",
- "elasticloadbalancing:DeleteTargetGroup",
- "elasticloadbalancing:DeregisterTargets",
- "elasticloadbalancing:DescribeListenerCertificates",
- "elasticloadbalancing:DescribeListeners",
- "elasticloadbalancing:DescribeLoadBalancers",
- "elasticloadbalancing:DescribeLoadBalancerAttributes",
- "elasticloadbalancing:DescribeRules",
- "elasticloadbalancing:DescribeSSLPolicies",
- "elasticloadbalancing:DescribeTags",
- "elasticloadbalancing:DescribeTargetGroups",
- "elasticloadbalancing:DescribeTargetGroupAttributes",
- "elasticloadbalancing:DescribeTargetHealth",
- "elasticloadbalancing:ModifyListener",
- "elasticloadbalancing:ModifyLoadBalancerAttributes",
- "elasticloadbalancing:ModifyRule",
- "elasticloadbalancing:ModifyTargetGroup",
- "elasticloadbalancing:ModifyTargetGroupAttributes",
- "elasticloadbalancing:RegisterTargets",
- "elasticloadbalancing:RemoveListenerCertificates",
- "elasticloadbalancing:RemoveTags",
- "elasticloadbalancing:SetIpAddressType",
- "elasticloadbalancing:SetSecurityGroups",
- "elasticloadbalancing:SetSubnets",
- "elasticloadbalancing:SetWebACL",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "iam:CreateServiceLinkedRole",
- "iam:GetServerCertificate",
- "iam:ListServerCertificates",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "cognito-idp:DescribeUserPoolClient",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "waf-regional:GetWebACLForResource",
- "waf-regional:GetWebACL",
- "waf-regional:AssociateWebACL",
- "waf-regional:DisassociateWebACL",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "tag:GetResources",
- "tag:TagResources",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "waf:GetWebACL",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "wafv2:GetWebACL",
- "wafv2:GetWebACLForResource",
- "wafv2:AssociateWebACL",
- "wafv2:DisassociateWebACL",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "shield:DescribeProtection",
- "shield:GetSubscriptionState",
- "shield:DeleteProtection",
- "shield:CreateProtection",
- "shield:DescribeSubscription",
- "shield:ListProtections",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "appmesh:*",
- "servicediscovery:CreateService",
- "servicediscovery:GetService",
- "servicediscovery:RegisterInstance",
- "servicediscovery:DeregisterInstance",
- "servicediscovery:ListInstances",
- "servicediscovery:ListNamespaces",
- "servicediscovery:ListServices",
- "route53:GetHealthCheck",
- "route53:CreateHealthCheck",
- "route53:UpdateHealthCheck",
- "route53:ChangeResourceRecordSets",
- "route53:DeleteHealthCheck",
- },
- },
- { // for fluentd add-on
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "logs:CreateLogGroup",
- "logs:CreateLogStream",
- "logs:DescribeLogGroups",
- "logs:DescribeLogStreams",
- "logs:PutLogEvents",
- },
- },
- { // for cluster autoscaler
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "autoscaling:DescribeAutoScalingGroups",
- "autoscaling:DescribeAutoScalingInstances",
- "autoscaling:DescribeLaunchConfigurations",
- "autoscaling:DescribeTags",
- "autoscaling:SetDesiredCapacity",
- "autoscaling:TerminateInstanceInAutoScalingGroup",
- "ec2:DescribeLaunchTemplateVersions",
- },
- },
- { // for artifact uploads from worker nodes
- Effect: "Allow",
- Resource: fmt.Sprintf("arn:%s:s3:::%s/*", partition, bucketName),
- Action: []string{
- "s3:ListBucket",
- "s3:GetObject",
- "s3:PutObject",
- },
- },
- { // arn:aws:iam::aws:policy/AmazonS3FullAccess
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "s3:*",
- },
- },
- { // arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "ecr:GetAuthorizationToken",
- "ecr:BatchCheckLayerAvailability",
- "ecr:GetDownloadUrlForLayer",
- "ecr:GetRepositoryPolicy",
- "ecr:DescribeRepositories",
- "ecr:ListImages",
- "ecr:DescribeImages",
- "ecr:BatchGetImage",
- "ecr:GetLifecyclePolicy",
- "ecr:GetLifecyclePolicyPreview",
- "ecr:ListTagsForResource",
- "ecr:DescribeImageScanFindings",
- },
- },
- }
-}
diff --git a/eks/mng/scale/scale.go b/eks/mng/scale/scale.go
deleted file mode 100644
index c608ea1f7..000000000
--- a/eks/mng/scale/scale.go
+++ /dev/null
@@ -1,207 +0,0 @@
-// Package scale implements EKS cluster scaler tester.
-// ref. https://docs.aws.amazon.com/cli/latest/reference/eks/update-nodegroup-config.html
-package scale
-
-import (
- "context"
- "errors"
- "fmt"
- "io"
- "reflect"
- "time"
-
- wait "github.com/aws/aws-k8s-tester/eks/mng/wait"
- "github.com/aws/aws-k8s-tester/eksconfig"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/service/eks"
- aws_eks "github.com/aws/aws-sdk-go/service/eks"
- "github.com/aws/aws-sdk-go/service/eks/eksiface"
- "go.uber.org/zap"
-)
-
-// Scaler defines MNG scaler interface.
-// ref. https://docs.aws.amazon.com/cli/latest/reference/eks/update-nodegroup-config.html
-type Scaler interface {
- // Update starts MNG scaler process, and waits for its completion.
- // ref. https://docs.aws.amazon.com/cli/latest/reference/eks/update-nodegroup-config.html
- Scale(mngName string) error
-}
-
-// Config defines scaler configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- EKSAPI eksiface.EKSAPI
-}
-
-// New creates a new Scaler.
-func New(cfg Config) Scaler {
- cfg.Logger.Info("creating tester", zap.String("tester", reflect.TypeOf(tester{}).PkgPath()))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
-}
-
-func (ts *tester) Scale(mngName string) (err error) {
- cur, ok := ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName]
- if !ok {
- ts.cfg.Logger.Warn("MNG not found; failing update", zap.String("mng-name", mngName))
- return fmt.Errorf("MNGs[%q] not found; failed to update", mngName)
- }
- if len(cur.ScaleUpdates) == 0 {
- ts.cfg.Logger.Info("MNG scaler is not enabled; skipping update", zap.String("mng-name", mngName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Update", zap.String("tester", reflect.TypeOf(tester{}).PkgPath()))
- for idx, update := range cur.ScaleUpdates {
- if !update.Enable {
- continue
- }
- if update.Created {
- ts.cfg.Logger.Info("scale update already created; skipping", zap.Int("index", idx))
- continue
- }
- fmt.Print(ts.cfg.EKSConfig.Colorize("\n\n[yellow]*********************************\n"))
- fmt.Printf(ts.cfg.EKSConfig.Colorize("[light_green]MNGs[%q].Scale[%d]\n"), mngName, idx)
- ts.cfg.Logger.Info("waiting before starting MNG update",
- zap.String("cluster-name", ts.cfg.EKSConfig.Name),
- zap.String("mng-name", mngName),
- zap.Int("asg-min-size", cur.ASGMinSize),
- zap.Int("asg-max-size", cur.ASGMaxSize),
- zap.Int("asg-desired-capacity", cur.ASGDesiredCapacity),
- zap.Int64("target-min-size", update.ASGMinSize),
- zap.Int64("target-max-size", update.ASGMaxSize),
- zap.Int64("target-desired-size", update.ASGDesiredCapacity),
- zap.String("update-id", update.ID),
- zap.Duration("initial-wait", update.InitialWait),
- )
- select {
- case <-time.After(update.InitialWait):
- ts.cfg.Logger.Info("waited, starting MNG scaler")
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("MNG scaler wait aborted; exiting", zap.String("mng-name", mngName))
- return errors.New("MNG scaler wait aborted")
- }
-
- createStart := time.Now()
- if err = ts.scaleMNG(mngName, update); err != nil {
- return err
- }
- createEnd := time.Now()
- update.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- cur.ScaleUpdates[idx] = update
-
- ts.cfg.Logger.Info("completed MNG scaler",
- zap.Int("from", cur.ASGDesiredCapacity),
- zap.Int64("to", update.ASGDesiredCapacity),
- )
- }
-
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName] = cur
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) scaleMNG(mngName string, update eksconfig.MNGScaleUpdate) (err error) {
- cur, ok := ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName]
- if !ok {
- return fmt.Errorf("MNGs[%q] not found", mngName)
- }
-
- var out *eks.UpdateNodegroupConfigOutput
- out, err = ts.cfg.EKSAPI.UpdateNodegroupConfig(&aws_eks.UpdateNodegroupConfigInput{
- ClusterName: aws.String(ts.cfg.EKSConfig.Name),
- NodegroupName: aws.String(mngName),
- ScalingConfig: &aws_eks.NodegroupScalingConfig{
- DesiredSize: aws.Int64(update.ASGDesiredCapacity),
- MaxSize: aws.Int64(update.ASGMaxSize),
- MinSize: aws.Int64(update.ASGMinSize),
- },
- })
- if err != nil {
- ts.cfg.Logger.Warn("MNG scaler request failed", zap.String("mng-name", mngName), zap.Error(err))
- return err
- }
-
- reqID := ""
- if out.Update != nil {
- reqID = aws.StringValue(out.Update.Id)
- }
- if reqID == "" {
- return fmt.Errorf("MNGs[%q] UpdateNodegroupConfigOutput.Update.Id empty", mngName)
- }
-
- initialWait := 3 * time.Minute
- totalWait := time.Hour + 10*time.Minute*time.Duration(update.ASGDesiredCapacity)
- ts.cfg.Logger.Info("sent MNG scaler request; polling",
- zap.String("cluster-name", ts.cfg.EKSConfig.Name),
- zap.String("mng-name", mngName),
- zap.Int("current-asg-min-size", cur.ASGMinSize),
- zap.Int("current-asg-desired-capacity", cur.ASGDesiredCapacity),
- zap.Int("current-asg-max-size", cur.ASGMaxSize),
- zap.Int64("target-asg-min-size", update.ASGMinSize),
- zap.Int64("target-asg-desired-capacity", update.ASGDesiredCapacity),
- zap.Int64("target-asg-max-size", update.ASGMaxSize),
- zap.String("update-id", update.ID),
- zap.String("request-id", reqID),
- zap.Duration("total-wait", totalWait),
- )
-
- // enough time for upgrade fail/rollback
- ctx, cancel := context.WithTimeout(context.Background(), totalWait)
- updateCh := wait.PollUpdate(
- ctx,
- ts.cfg.Stopc,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.EKSAPI,
- ts.cfg.EKSConfig.Name,
- mngName,
- reqID,
- eks.UpdateStatusSuccessful,
- initialWait,
- 30*time.Second,
- )
- for v := range updateCh {
- err = v.Error
- }
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("MNG scale failed when polling", zap.String("mng-name", mngName), zap.Error(err))
- return err
- }
-
- ctx, cancel = context.WithTimeout(context.Background(), totalWait)
- nodesCh := wait.Poll(
- ctx,
- ts.cfg.Stopc,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.EKSAPI,
- ts.cfg.EKSConfig.Name,
- mngName,
- aws_eks.NodegroupStatusActive,
- initialWait,
- 20*time.Second,
- )
- for sv := range nodesCh {
- err = sv.Error
- }
- cancel()
- if err != nil {
- cur.Status = fmt.Sprintf("scale failed %v", err)
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName] = cur
- ts.cfg.EKSConfig.Sync()
- return fmt.Errorf("MNGs[%q] scale failed %v", mngName, err)
- }
-
- ts.cfg.Logger.Info("successfully scale updated MNG", zap.String("update-id", update.ID), zap.String("mng-name", mngName))
- ts.cfg.EKSConfig.Sync()
- return nil
-}
diff --git a/eks/mng/security-groups.go b/eks/mng/security-groups.go
deleted file mode 100644
index 1f63ae305..000000000
--- a/eks/mng/security-groups.go
+++ /dev/null
@@ -1,766 +0,0 @@
-package mng
-
-import (
- "context"
- "errors"
- "fmt"
- "strings"
-
- aws_v2 "github.com/aws/aws-sdk-go-v2/aws"
- aws_ec2_v2 "github.com/aws/aws-sdk-go-v2/service/ec2"
- aws_ec2_v2_types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
- smithy "github.com/aws/smithy-go"
- "go.uber.org/zap"
-)
-
-// see https://github.com/aws/aws-k8s-tester/blob/v1.6.0/eks/mng/security-groups.go for CloudFormation based workflow
-
-// "[sig-network] Networking Granular Checks" in "test/e2e/network/dns.go"
-// requires "e2enetwork.EndpointUDPPort/EndpointHTTPPort", 8081 and 8080
-// just open all for now...
-// TODO: restrict ports
-
-// AWS::EC2::SecurityGroup
-func (ts *tester) authorizeSecurityGroups(name string) error {
- cur, ok := ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[name]
- if !ok {
- return fmt.Errorf("MNGs[%q] not found; cannot authorize ingress/egress security group", name)
- }
- if cur.RemoteAccessSecurityGroupID == "" {
- return fmt.Errorf("MNG[%q] security group ID not found; cannot authorize ingress/egress security group", name)
- }
- ts.cfg.Logger.Info("authorizing security group",
- zap.String("mng-name", name),
- zap.String("api-server-node-security-group-id", ts.cfg.EKSConfig.VPC.SecurityGroupID),
- zap.String("managed-node-group-security-group-id", cur.RemoteAccessSecurityGroupID),
- )
-
- // allow node to communicate with each other
- ts.cfg.Logger.Info("authorizing IngressWithinManagedNodeGroupSecurityGroup", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err := ts.cfg.EC2APIV2.AuthorizeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
-
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("-1"),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- Description: aws_v2.String("allow node to communicate with each other"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized IngressWithinManagedNodeGroupSecurityGroup")
-
- // allow pods to communicate with the cluster API Server
- ts.cfg.Logger.Info("authorizing Ingress443FromNGtoCP", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
-
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(443),
- ToPort: aws_v2.Int32(443),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- Description: aws_v2.String("allow pods to communicate with the cluster API Server"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized Ingress443FromNGtoCP")
-
- // allow pods running extension API servers on port 443
- // to receive communication from cluster control plane
- ts.cfg.Logger.Info("authorizing Ingress443FromCPtoNG", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupIngressInput{
- // egress target
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(443),
- ToPort: aws_v2.Int32(443),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- Description: aws_v2.String("receive communication from cluster control plane"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized Ingress443FromCPtoNG")
-
- // allow the cluster control plane to communicate with pods running extension API servers on port 443
- ts.cfg.Logger.Info("authorizing Egress443FromCPtoNG", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupEgress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupEgressInput{
- // egress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(443),
- ToPort: aws_v2.Int32(443),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- Description: aws_v2.String("communicate with pods running extension API servers on port 443"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize egress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized Egress443FromCPtoNG")
-
- // allow worker Kubelets and pods to receive communication from the cluster control plane
- ts.cfg.Logger.Info("authorizing IngressAllFromCPtoNG", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(0),
- ToPort: aws_v2.Int32(65535),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- Description: aws_v2.String("receive communication from the cluster control plane"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized IngressAllFromCPtoNG")
-
- // allow the cluster control plane to communicate with worker Kubelet and pods
- ts.cfg.Logger.Info("authorizing EgressAllFromCPtoNG", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupEgress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupEgressInput{
- // egress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(0),
- ToPort: aws_v2.Int32(65535),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- Description: aws_v2.String("communicate with worker Kubelet and pods"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize egress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized EgressAllFromCPtoNG")
-
- ts.cfg.Logger.Info("authorizing Ingress22ForSSH", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- IpRanges: []aws_ec2_v2_types.IpRange{
- {
- CidrIp: aws_v2.String("0.0.0.0/0"),
- },
- },
- FromPort: aws_v2.Int32(22),
- ToPort: aws_v2.Int32(22),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized Ingress22ForSSH")
-
- ts.cfg.Logger.Info("authorizing IngressForGuestBook", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- IpRanges: []aws_ec2_v2_types.IpRange{
- {
- CidrIp: aws_v2.String("0.0.0.0/0"),
- },
- },
- FromPort: aws_v2.Int32(1),
- ToPort: aws_v2.Int32(10000),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized IngressForGuestBook")
-
- ts.cfg.Logger.Info("authorizing EgressForGuestBook", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupEgress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupEgressInput{
- // egress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(1),
- ToPort: aws_v2.Int32(10000),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize egress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized EgressForGuestBook")
-
- ts.cfg.Logger.Info("authorizing IngressForNodePortConformance", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- IpRanges: []aws_ec2_v2_types.IpRange{
- {
- CidrIp: aws_v2.String("0.0.0.0/0"),
- },
- },
- FromPort: aws_v2.Int32(1),
- ToPort: aws_v2.Int32(32767),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized IngressForNodePortConformance")
-
- ts.cfg.Logger.Info("authorized security group")
- return nil
-}
-
-func (ts *tester) revokeSecurityGroups(name string) (err error) {
- cur, ok := ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[name]
- if !ok {
- return fmt.Errorf("MNGs[%q] not found; cannot revoke ingress/egress security group", name)
- }
- if cur.RemoteAccessSecurityGroupID == "" {
- return fmt.Errorf("MNG[%q] security group ID not found; cannot revoke ingress/egress security group", name)
- }
- ts.cfg.Logger.Info("revoking security group",
- zap.String("mng-name", name),
- zap.String("api-server-node-security-group-id", ts.cfg.EKSConfig.VPC.SecurityGroupID),
- zap.String("managed-node-group-security-group-id", cur.RemoteAccessSecurityGroupID),
- )
-
- // allow node to communicate with each other
- ts.cfg.Logger.Info("revoking IngressWithinNodeGroupSecurityGroup", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
-
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("-1"),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- Description: aws_v2.String("allow node to communicate with each other"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked IngressWithinNodeGroupSecurityGroup")
-
- // allow pods to communicate with the cluster API Server
- ts.cfg.Logger.Info("revoking Ingress443FromNGtoCP", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
-
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(443),
- ToPort: aws_v2.Int32(443),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- Description: aws_v2.String("allow pods to communicate with the cluster API Server"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked Ingress443FromNGtoCP")
-
- // allow pods running extension API servers on port 443
- // to receive communication from cluster control plane
- ts.cfg.Logger.Info("revoking Ingress443FromCPtoNG", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupIngressInput{
- // egress target
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(443),
- ToPort: aws_v2.Int32(443),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- Description: aws_v2.String("receive communication from cluster control plane"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked Ingress443FromCPtoNG")
-
- // allow the cluster control plane to communicate with pods running extension API servers on port 443
- ts.cfg.Logger.Info("revoking Egress443FromCPtoNG", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupEgress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupEgressInput{
- // egress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(443),
- ToPort: aws_v2.Int32(443),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- Description: aws_v2.String("communicate with pods running extension API servers on port 443"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke egress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked Egress443FromCPtoNG")
-
- // allow worker Kubelets and pods to receive communication from the cluster control plane
- ts.cfg.Logger.Info("revoking IngressAllFromCPtoNG", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(0),
- ToPort: aws_v2.Int32(65535),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- Description: aws_v2.String("receive communication from the cluster control plane"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked IngressAllFromCPtoNG")
-
- // allow the cluster control plane to communicate with worker Kubelet and pods
- ts.cfg.Logger.Info("revoking EgressAllFromCPtoNG", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupEgress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupEgressInput{
- // egress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(0),
- ToPort: aws_v2.Int32(65535),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- Description: aws_v2.String("communicate with worker Kubelet and pods"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke egress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked EgressAllFromCPtoNG")
-
- ts.cfg.Logger.Info("revoking Ingress22ForSSH", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- IpRanges: []aws_ec2_v2_types.IpRange{
- {
- CidrIp: aws_v2.String("0.0.0.0/0"),
- },
- },
- FromPort: aws_v2.Int32(22),
- ToPort: aws_v2.Int32(22),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked Ingress22ForSSH")
-
- ts.cfg.Logger.Info("revoking IngressForGuestBook", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- IpRanges: []aws_ec2_v2_types.IpRange{
- {
- CidrIp: aws_v2.String("0.0.0.0/0"),
- },
- },
- FromPort: aws_v2.Int32(1),
- ToPort: aws_v2.Int32(10000),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked IngressForGuestBook")
-
- ts.cfg.Logger.Info("revoking EgressForGuestBook", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupEgress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupEgressInput{
- // egress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(1),
- ToPort: aws_v2.Int32(10000),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke egress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked EgressForGuestBook")
-
- ts.cfg.Logger.Info("revoking IngressForNodePortConformance", zap.String("sg-id", cur.RemoteAccessSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(cur.RemoteAccessSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- IpRanges: []aws_ec2_v2_types.IpRange{
- {
- CidrIp: aws_v2.String("0.0.0.0/0"),
- },
- },
- FromPort: aws_v2.Int32(1),
- ToPort: aws_v2.Int32(32767),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked IngressForNodePortConformance")
-
- ts.cfg.Logger.Info("revoked security group")
- return nil
-}
diff --git a/eks/mng/tuple.go b/eks/mng/tuple.go
deleted file mode 100644
index 8f3b3de9e..000000000
--- a/eks/mng/tuple.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package mng
-
-import "time"
-
-type tupleTime struct {
- ts time.Time
- name string
-}
-
-type tupleTimes []tupleTime
-
-func (ts tupleTimes) Len() int { return len(ts) }
-
-func (ts tupleTimes) Less(i, j int) bool {
- return ts[j].ts.After(ts[i].ts)
-}
-
-func (ts tupleTimes) Swap(i, j int) {
- t := ts[i]
- ts[i] = ts[j]
- ts[j] = t
-}
diff --git a/eks/mng/tuple_test.go b/eks/mng/tuple_test.go
deleted file mode 100644
index 46e0b753a..000000000
--- a/eks/mng/tuple_test.go
+++ /dev/null
@@ -1,26 +0,0 @@
-package mng
-
-import (
- "reflect"
- "sort"
- "testing"
- "time"
-)
-
-func Test_byTime(t *testing.T) {
- ts := time.Time{}
- tss1 := []tupleTime{
- {ts: ts.Add(time.Second), name: "1"},
- {ts: ts.Add(2 * time.Second), name: "2"},
- {ts: ts.Add(3 * time.Second), name: "3"},
- }
- sort.Sort(sort.Reverse(tupleTimes(tss1)))
- tss2 := []tupleTime{
- {ts: ts.Add(3 * time.Second), name: "3"},
- {ts: ts.Add(2 * time.Second), name: "2"},
- {ts: ts.Add(time.Second), name: "1"},
- }
- if !reflect.DeepEqual(tss1, tss2) {
- t.Fatalf("expected %+v, got %+v", tss2, tss1)
- }
-}
diff --git a/eks/mng/version-upgrade/version-upgrade.go b/eks/mng/version-upgrade/version-upgrade.go
deleted file mode 100644
index be0502f26..000000000
--- a/eks/mng/version-upgrade/version-upgrade.go
+++ /dev/null
@@ -1,221 +0,0 @@
-// Package versionupgrade implements EKS cluster version upgrade tester.
-package versionupgrade
-
-import (
- "context"
- "errors"
- "fmt"
- "io"
- "reflect"
- "time"
-
- "github.com/aws/aws-k8s-tester/eks/mng/wait"
- "github.com/aws/aws-k8s-tester/eksconfig"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/spinner"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/service/eks"
- aws_eks "github.com/aws/aws-sdk-go/service/eks"
- "github.com/aws/aws-sdk-go/service/eks/eksiface"
- "go.uber.org/zap"
- v1 "k8s.io/api/core/v1"
-)
-
-// Upgrader defines MNG version upgrade interface.
-type Upgrader interface {
- // Upgrade starts MNG version upgrade process, and waits for its completion.
- // ref. https://docs.aws.amazon.com/cli/latest/reference/eks/update-nodegroup-version.html
- Upgrade(mngName string) error
-}
-
-// Config defines version upgrade configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
- EKSAPI eksiface.EKSAPI
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-// New creates a new Upgrader.
-func New(cfg Config) Upgrader {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
-}
-
-func (ts *tester) Upgrade(mngName string) (err error) {
- cur, ok := ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName]
- if !ok {
- ts.cfg.Logger.Warn("MNG not found; failing upgrade", zap.String("mng-name", mngName))
- return fmt.Errorf("MNGs[%q] not found; failed to upgrade", mngName)
- }
- if cur.VersionUpgrade == nil || !cur.VersionUpgrade.Enable {
- ts.cfg.Logger.Info("MNG version upgrade is not enabled; skipping upgrade", zap.String("mng-name", mngName))
- return nil
- }
- if cur.VersionUpgrade.Created {
- ts.cfg.Logger.Info("MNG version upgrade is already completed; skipping upgrade", zap.String("mng-name", mngName))
- return nil
- }
- fmt.Print(ts.cfg.EKSConfig.Colorize("\n\n[yellow]*********************************\n"))
- fmt.Printf(ts.cfg.EKSConfig.Colorize("[light_green]MNGs[%q].Upgrade\n"), mngName)
-
- ts.cfg.Logger.Info("starting tester.Upgrade", zap.String("tester", pkgName))
- cur.VersionUpgrade.Created = true
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName] = cur
- cur, _ = ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName]
- ts.cfg.EKSConfig.Sync()
-
- createStart := time.Now()
- defer func() {
- createEnd := time.Now()
- cur.VersionUpgrade.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName] = cur
- ts.cfg.EKSConfig.Sync()
- }()
-
- sp := spinner.New(ts.cfg.LogWriter, "Waiting for before starting MNG upgrade "+mngName)
- ts.cfg.Logger.Info("waiting before starting MNG upgrade",
- zap.String("cluster-name", ts.cfg.EKSConfig.Name),
- zap.String("mng-name", mngName),
- zap.Duration("initial-wait", cur.VersionUpgrade.InitialWait),
- )
- sp.Restart()
- select {
- case <-time.After(cur.VersionUpgrade.InitialWait):
- sp.Stop()
- ts.cfg.Logger.Info("waited, starting MNG version upgrade",
- zap.String("cluster-name", ts.cfg.EKSConfig.Name),
- zap.String("mng-name", mngName),
- zap.Float64("cluster-version", ts.cfg.EKSConfig.Status.ServerVersionInfo.VersionValue),
- zap.String("target-mng-version", cur.VersionUpgrade.Version),
- )
- case <-ts.cfg.Stopc:
- sp.Stop()
- ts.cfg.Logger.Warn("MNG version upgrade wait aborted; exiting", zap.String("mng-name", mngName))
- return errors.New("MNG veresion upgrade wait aborted")
- }
-
- // ref. https://docs.aws.amazon.com/cli/latest/reference/eks/update-nodegroup-version.html
- var updateOut *eks.UpdateNodegroupVersionOutput
- updateOut, err = ts.cfg.EKSAPI.UpdateNodegroupVersion(&eks.UpdateNodegroupVersionInput{
- ClusterName: aws.String(ts.cfg.EKSConfig.Name),
- NodegroupName: aws.String(mngName),
- Version: aws.String(cur.VersionUpgrade.Version),
- })
- if err != nil {
- ts.cfg.Logger.Warn("MNG version upgrade request failed", zap.String("mng-name", mngName), zap.Error(err))
- return err
- }
- reqID := ""
- if updateOut.Update != nil {
- reqID = aws.StringValue(updateOut.Update.Id)
- }
-
- initialWait := 5 * time.Minute
- checkN := time.Duration(cur.ASGDesiredCapacity)
- if checkN == 0 {
- checkN = time.Duration(cur.ASGMinSize)
- }
- totalWait := 2*time.Hour + 30*time.Minute + 3*time.Minute*checkN
- ts.cfg.Logger.Info("sent MNG upgrade request; polling",
- zap.String("cluster-name", ts.cfg.EKSConfig.Name),
- zap.String("mng-name", mngName),
- zap.String("request-id", reqID),
- zap.Int("asg-desired-capacity", cur.ASGDesiredCapacity),
- zap.Duration("total-wait", totalWait),
- )
-
- // enough time for upgrade fail/rollback
- ctx, cancel := context.WithTimeout(context.Background(), totalWait)
- updateCh := wait.PollUpdate(
- ctx,
- ts.cfg.Stopc,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.EKSAPI,
- ts.cfg.EKSConfig.Name,
- mngName,
- reqID,
- eks.UpdateStatusSuccessful,
- initialWait,
- 2*time.Minute,
- wait.WithQueryFunc(func() {
- fmt.Fprintf(ts.cfg.LogWriter, "\n")
- ts.cfg.Logger.Info("listing nodes while polling mng update status", zap.String("mng-name", mngName))
- nodes, err := ts.cfg.K8SClient.ListNodes(1000, 5*time.Second)
- if err != nil {
- ts.cfg.Logger.Warn("failed to list nodes while polling mng update status", zap.Error(err))
- return
- }
- cnt := 0
- for _, node := range nodes {
- labels := node.GetLabels()
- if labels["NGName"] != mngName {
- continue
- }
- cnt++
- for _, cond := range node.Status.Conditions {
- if cond.Status != v1.ConditionTrue {
- continue
- }
- ts.cfg.Logger.Info("node",
- zap.String("name", node.GetName()),
- zap.String("mng-name", mngName),
- zap.String("status-type", fmt.Sprintf("%s", cond.Type)),
- zap.String("status", fmt.Sprintf("%s", cond.Status)),
- )
- break
- }
- }
- ts.cfg.Logger.Info("listed nodes while polling mng update status", zap.String("mng-name", mngName), zap.Int("total-nodes", cnt))
- }),
- )
- for v := range updateCh {
- err = v.Error
- }
- cancel()
- if err != nil {
- cur.Status = fmt.Sprintf("update failed %v", err)
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName] = cur
- ts.cfg.EKSConfig.Sync()
- return fmt.Errorf("MNGs[%q] update failed %v", mngName, err)
- }
-
- ctx, cancel = context.WithTimeout(context.Background(), totalWait)
- nodesCh := wait.Poll(
- ctx,
- ts.cfg.Stopc,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.EKSAPI,
- ts.cfg.EKSConfig.Name,
- mngName,
- aws_eks.NodegroupStatusActive,
- initialWait,
- 20*time.Second,
- )
- for sv := range nodesCh {
- err = sv.Error
- }
- cancel()
- if err != nil {
- cur.Status = fmt.Sprintf("update failed %v", err)
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName] = cur
- ts.cfg.EKSConfig.Sync()
- return fmt.Errorf("MNGs[%q] update failed %v", mngName, err)
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
diff --git a/eks/mng/wait/poll.go b/eks/mng/wait/poll.go
deleted file mode 100644
index 52ffa742d..000000000
--- a/eks/mng/wait/poll.go
+++ /dev/null
@@ -1,401 +0,0 @@
-package wait
-
-import (
- "context"
- "errors"
- "fmt"
- "io"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/pkg/ctxutil"
- "github.com/aws/aws-k8s-tester/pkg/spinner"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/aws/awserr"
- "github.com/aws/aws-sdk-go/service/eks"
- aws_eks "github.com/aws/aws-sdk-go/service/eks"
- "github.com/aws/aws-sdk-go/service/eks/eksiface"
- "github.com/dustin/go-humanize"
- "go.uber.org/zap"
-)
-
-// ManagedNodeGroupStatusDELETEDORNOTEXIST defines the cluster status when the cluster is not found.
-//
-// ref. https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html
-//
-// CREATING
-// ACTIVE
-// DELETING
-// FAILED
-// UPDATING
-//
-const ManagedNodeGroupStatusDELETEDORNOTEXIST = "DELETED/NOT-EXIST"
-
-// ManagedNodeGroupStatus represents the CloudFormation status.
-type ManagedNodeGroupStatus struct {
- NodeGroupName string
- NodeGroup *aws_eks.Nodegroup
- Error error
-}
-
-// Poll periodically fetches the managed node group status
-// until the node group becomes the desired state.
-func Poll(
- ctx context.Context,
- stopc chan struct{},
- lg *zap.Logger,
- logWriter io.Writer,
- eksAPI eksiface.EKSAPI,
- clusterName string,
- mngName string,
- desiredNodeGroupStatus string,
- initialWait time.Duration,
- pollInterval time.Duration,
- opts ...OpOption) <-chan ManagedNodeGroupStatus {
-
- ret := Op{}
- ret.applyOpts(opts)
-
- now := time.Now()
- sp := spinner.New(logWriter, "Waiting for Managed Node Group status "+desiredNodeGroupStatus)
-
- lg.Info("polling mng",
- zap.String("cluster-name", clusterName),
- zap.String("mng-name", mngName),
- zap.String("desired-status", desiredNodeGroupStatus),
- zap.String("initial-wait", initialWait.String()),
- zap.String("poll-interval", pollInterval.String()),
- zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
- )
-
- ch := make(chan ManagedNodeGroupStatus, 10)
- go func() {
- // very first poll should be no-wait
- // in case stack has already reached desired status
- // wait from second interation
- waitDur := time.Duration(0)
-
- first := true
- for ctx.Err() == nil {
- select {
- case <-ctx.Done():
- lg.Warn("wait aborted, ctx done", zap.Error(ctx.Err()))
- ch <- ManagedNodeGroupStatus{NodeGroupName: mngName, NodeGroup: nil, Error: ctx.Err()}
- close(ch)
- return
-
- case <-stopc:
- lg.Warn("wait stopped, stopc closed", zap.Error(ctx.Err()))
- ch <- ManagedNodeGroupStatus{NodeGroupName: mngName, NodeGroup: nil, Error: errors.New("wait stopped")}
- close(ch)
- return
-
- case <-time.After(waitDur):
- // very first poll should be no-wait
- // in case stack has already reached desired status
- // wait from second interation
- if waitDur == time.Duration(0) {
- waitDur = pollInterval
- }
- }
-
- output, err := eksAPI.DescribeNodegroup(&aws_eks.DescribeNodegroupInput{
- ClusterName: aws.String(clusterName),
- NodegroupName: aws.String(mngName),
- })
- if err != nil {
- if IsDeleted(err) {
- if desiredNodeGroupStatus == ManagedNodeGroupStatusDELETEDORNOTEXIST {
- lg.Info("managed node group is already deleted as desired; exiting", zap.Error(err))
- ch <- ManagedNodeGroupStatus{NodeGroupName: mngName, NodeGroup: nil, Error: nil}
- close(ch)
- return
- }
- lg.Warn("managed node group does not exist", zap.Error(err))
- ch <- ManagedNodeGroupStatus{NodeGroupName: mngName, NodeGroup: nil, Error: err}
- close(ch)
- return
- }
- lg.Warn("describe managed node group failed; retrying", zap.Error(err))
- ch <- ManagedNodeGroupStatus{NodeGroupName: mngName, NodeGroup: nil, Error: err}
- continue
- }
-
- if output.Nodegroup == nil {
- lg.Warn("expected non-nil managed node group; retrying")
- ch <- ManagedNodeGroupStatus{NodeGroupName: mngName, NodeGroup: nil, Error: fmt.Errorf("unexpected empty response %+v", output.GoString())}
- continue
- }
-
- nodeGroup := output.Nodegroup
- currentStatus := aws.StringValue(nodeGroup.Status)
- lg.Info("poll",
- zap.String("cluster-name", clusterName),
- zap.String("mng-name", mngName),
- zap.String("status", currentStatus),
- zap.String("started", humanize.RelTime(now, time.Now(), "ago", "from now")),
- zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
- )
- switch currentStatus {
- case desiredNodeGroupStatus:
- ch <- ManagedNodeGroupStatus{NodeGroupName: mngName, NodeGroup: nodeGroup, Error: nil}
- lg.Info("desired managed node group status; done", zap.String("status", currentStatus))
- close(ch)
- return
-
- case aws_eks.NodegroupStatusCreateFailed,
- aws_eks.NodegroupStatusDeleteFailed,
- aws_eks.NodegroupStatusDegraded:
- lg.Warn("unexpected managed node group status; failed", zap.String("status", currentStatus))
- ch <- ManagedNodeGroupStatus{NodeGroupName: mngName, NodeGroup: nodeGroup, Error: fmt.Errorf("unexpected mng status %q", currentStatus)}
- close(ch)
- return
-
- default:
- ch <- ManagedNodeGroupStatus{NodeGroupName: mngName, NodeGroup: nodeGroup, Error: nil}
- }
-
- if ret.queryFunc != nil {
- ret.queryFunc()
- }
-
- if first {
- lg.Info("sleeping", zap.Duration("initial-wait", initialWait))
- sp.Restart()
- select {
- case <-ctx.Done():
- sp.Stop()
- lg.Warn("wait aborted, ctx done", zap.Error(ctx.Err()))
- ch <- ManagedNodeGroupStatus{NodeGroupName: mngName, NodeGroup: nil, Error: ctx.Err()}
- close(ch)
- return
- case <-stopc:
- sp.Stop()
- lg.Warn("wait stopped, stopc closed", zap.Error(ctx.Err()))
- ch <- ManagedNodeGroupStatus{NodeGroupName: mngName, NodeGroup: nil, Error: errors.New("wait stopped")}
- close(ch)
- return
- case <-time.After(initialWait):
- sp.Stop()
- }
- first = false
- }
- }
-
- lg.Warn("wait aborted, ctx done", zap.Error(ctx.Err()))
- ch <- ManagedNodeGroupStatus{NodeGroupName: mngName, NodeGroup: nil, Error: ctx.Err()}
- close(ch)
- return
- }()
- return ch
-}
-
-// IsDeleted returns true if error from EKS API indicates that
-// the EKS managed node group has already been deleted.
-func IsDeleted(err error) bool {
- if err == nil {
- return false
- }
- awsErr, ok := err.(awserr.Error)
- if ok && awsErr.Code() == "ResourceNotFoundException" {
- return true
- }
-
- // ResourceNotFoundException: nodeGroup eks-2019120505-pdx-us-west-2-tqy2d-managed-node-group not found for cluster eks-2019120505-pdx-us-west-2-tqy2d\n\tstatus code: 404, request id: 330998c1-22e9-4a8b-b180-420dadade090
- return strings.Contains(err.Error(), "No cluster found for") ||
- strings.Contains(err.Error(), " not found for cluster ")
-}
-
-// updateNotExists returns true if error from EKS API indicates that
-// the EKS cluster update does not exist.
-func updateNotExists(err error) bool {
- if err == nil {
- return false
- }
- awsErr, ok := err.(awserr.Error)
- if ok && awsErr.Code() == "ResourceNotFoundException" &&
- strings.HasPrefix(awsErr.Message(), "No update found for") {
- return true
- }
- // An error occurred (ResourceNotFoundException) when calling the DescribeUpdate operation: No update found for ID: 10bddb13-a71b-425a-b0a6-71cd03e59161
- return strings.Contains(err.Error(), "No update found")
-}
-
-// UpdateStatus represents the CloudFormation status.
-type UpdateStatus struct {
- Update *eks.Update
- Error error
-}
-
-// PollUpdate periodically fetches the MNG update status
-// until the MNG update becomes the desired state.
-// ref. https://docs.aws.amazon.com/eks/latest/APIReference/API_DescribeUpdate.html
-func PollUpdate(
- ctx context.Context,
- stopc chan struct{},
- lg *zap.Logger,
- logWriter io.Writer,
- eksAPI eksiface.EKSAPI,
- clusterName string,
- mngName string,
- requestID string,
- desiredUpdateStatus string,
- initialWait time.Duration,
- pollInterval time.Duration,
- opts ...OpOption) <-chan UpdateStatus {
-
- ret := Op{}
- ret.applyOpts(opts)
-
- now := time.Now()
- sp := spinner.New(logWriter, "Waiting for Managed Node Group update status "+desiredUpdateStatus)
-
- lg.Info("polling mng update",
- zap.String("cluster-name", clusterName),
- zap.String("mng-name", mngName),
- zap.String("request-id", requestID),
- zap.String("desired-update-status", desiredUpdateStatus),
- zap.String("initial-wait", initialWait.String()),
- zap.String("poll-interval", pollInterval.String()),
- zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
- )
-
- ch := make(chan UpdateStatus, 10)
- go func() {
- // very first poll should be no-wait
- // in case stack has already reached desired status
- // wait from second interation
- waitDur := time.Duration(0)
-
- first := true
- for ctx.Err() == nil {
- select {
- case <-ctx.Done():
- lg.Warn("wait aborted, ctx done", zap.Error(ctx.Err()))
- ch <- UpdateStatus{Update: nil, Error: ctx.Err()}
- close(ch)
- return
-
- case <-stopc:
- lg.Warn("wait stopped, stopc closed", zap.Error(ctx.Err()))
- ch <- UpdateStatus{Update: nil, Error: errors.New("wait stopped")}
- close(ch)
- return
-
- case <-time.After(waitDur):
- // very first poll should be no-wait
- // in case stack has already reached desired status
- // wait from second interation
- if waitDur == time.Duration(0) {
- waitDur = pollInterval
- }
- }
-
- output, err := eksAPI.DescribeUpdate(&eks.DescribeUpdateInput{
- Name: aws.String(clusterName),
- NodegroupName: aws.String(mngName),
- UpdateId: aws.String(requestID),
- })
- if err != nil {
- if updateNotExists(err) {
- lg.Warn("mng update does not exist; aborting", zap.Error(ctx.Err()))
- ch <- UpdateStatus{Update: nil, Error: err}
- close(ch)
- return
- }
-
- lg.Warn("describe mng update failed; retrying", zap.Error(err))
- ch <- UpdateStatus{Update: nil, Error: err}
- continue
- }
-
- if output.Update == nil {
- lg.Warn("expected non-nil mng update; retrying")
- ch <- UpdateStatus{Update: nil, Error: fmt.Errorf("unexpected empty response %+v", output.GoString())}
- continue
- }
-
- update := output.Update
- currentStatus := aws.StringValue(update.Status)
- updateType := aws.StringValue(update.Type)
- lg.Info("poll",
- zap.String("cluster-name", clusterName),
- zap.String("mng-name", mngName),
- zap.String("status", currentStatus),
- zap.String("update-type", updateType),
- zap.String("started", humanize.RelTime(now, time.Now(), "ago", "from now")),
- zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
- )
- switch currentStatus {
- case desiredUpdateStatus:
- ch <- UpdateStatus{Update: update, Error: nil}
- lg.Info("desired mng update status; done", zap.String("status", currentStatus))
- close(ch)
- return
- case eks.UpdateStatusCancelled:
- ch <- UpdateStatus{Update: update, Error: fmt.Errorf("unexpected mng update status %q", eks.UpdateStatusCancelled)}
- lg.Warn("mng update status cancelled", zap.String("status", currentStatus), zap.String("desired-status", desiredUpdateStatus))
- close(ch)
- return
- case eks.UpdateStatusFailed:
- ch <- UpdateStatus{Update: update, Error: fmt.Errorf("unexpected mng update status %q", eks.UpdateStatusFailed)}
- lg.Warn("mng update status failed", zap.String("status", currentStatus), zap.String("desired-status", desiredUpdateStatus))
- close(ch)
- return
- default:
- ch <- UpdateStatus{Update: update, Error: nil}
- }
-
- if ret.queryFunc != nil {
- ret.queryFunc()
- }
-
- if first {
- lg.Info("sleeping", zap.Duration("initial-wait", initialWait))
- sp.Restart()
- select {
- case <-ctx.Done():
- sp.Stop()
- lg.Warn("wait aborted, ctx done", zap.Error(ctx.Err()))
- ch <- UpdateStatus{Update: nil, Error: ctx.Err()}
- close(ch)
- return
- case <-stopc:
- sp.Stop()
- lg.Warn("wait stopped, stopc closed", zap.Error(ctx.Err()))
- ch <- UpdateStatus{Update: nil, Error: errors.New("wait stopped")}
- close(ch)
- return
- case <-time.After(initialWait):
- sp.Stop()
- }
- first = false
- }
- }
-
- lg.Warn("wait aborted, ctx done", zap.Error(ctx.Err()))
- ch <- UpdateStatus{Update: nil, Error: ctx.Err()}
- close(ch)
- return
- }()
- return ch
-}
-
-// Op represents a MNG operation.
-type Op struct {
- queryFunc func()
-}
-
-// OpOption configures archiver operations.
-type OpOption func(*Op)
-
-// WithQueryFunc configures query function to be called in retry func.
-func WithQueryFunc(f func()) OpOption {
- return func(op *Op) { op.queryFunc = f }
-}
-
-func (op *Op) applyOpts(opts []OpOption) {
- for _, opt := range opts {
- opt(op)
- }
-}
diff --git a/eks/mng/wait/wait.go b/eks/mng/wait/wait.go
deleted file mode 100644
index 92b5fe153..000000000
--- a/eks/mng/wait/wait.go
+++ /dev/null
@@ -1,330 +0,0 @@
-// Package wait implements node waiter.
-package wait
-
-import (
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "reflect"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/eksconfig"
- aws_ec2 "github.com/aws/aws-k8s-tester/pkg/aws/ec2"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- k8s_object "github.com/aws/aws-k8s-tester/pkg/k8s-object"
- aws_v2 "github.com/aws/aws-sdk-go-v2/aws"
- aws_asg_v2 "github.com/aws/aws-sdk-go-v2/service/autoscaling"
- aws_ec2_v2 "github.com/aws/aws-sdk-go-v2/service/ec2"
- "github.com/aws/aws-sdk-go/service/eks"
- "github.com/aws/aws-sdk-go/service/eks/eksiface"
- "go.uber.org/zap"
- certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
- v1 "k8s.io/api/core/v1"
-)
-
-// NodeWaiter defines node waiter operation.
-type NodeWaiter interface {
- // Wait waits until all MNG and Kubernetes nodes are ready.
- Wait(mngName string, retries int) error
-}
-
-// Config defines version upgrade configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
-
- EC2APIV2 *aws_ec2_v2.Client
- ASGAPIV2 *aws_asg_v2.Client
- EKSAPI eksiface.EKSAPI
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-// New creates a new node waiter.
-func New(cfg Config) NodeWaiter {
- cfg.Logger.Info("creating node waiter", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
-}
-
-func (ts *tester) Wait(mngName string, retries int) error {
- return ts.waitForNodes(mngName, retries)
-}
-
-func (ts *tester) waitForNodes(mngName string, retriesLeft int) error {
- cur, ok := ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName]
- if !ok {
- return fmt.Errorf("MNGs[%q] not found", mngName)
- }
-
- ts.cfg.Logger.Info("checking MNG using EKS API", zap.String("mng-name", cur.Name))
- dout, err := ts.cfg.EKSAPI.DescribeNodegroup(&eks.DescribeNodegroupInput{
- ClusterName: aws_v2.String(ts.cfg.EKSConfig.Name),
- NodegroupName: aws_v2.String(cur.Name),
- })
- if err != nil {
- return err
- }
- if dout.Nodegroup == nil {
- return fmt.Errorf("MNG %q not found", cur.Name)
- }
- if dout.Nodegroup.Resources == nil {
- return fmt.Errorf("MNG %q Resources not found", cur.Name)
- }
- if len(dout.Nodegroup.Resources.AutoScalingGroups) != 1 {
- return fmt.Errorf("expected 1 ASG for %q, got %d", mngName, len(dout.Nodegroup.Resources.AutoScalingGroups))
- }
- if cur.RemoteAccessSecurityGroupID == "" {
- cur.RemoteAccessSecurityGroupID = aws_v2.ToString(dout.Nodegroup.Resources.RemoteAccessSecurityGroup)
- ts.cfg.Logger.Info("checking MNG security group", zap.String("mng-name", cur.Name), zap.String("security-group-id", cur.RemoteAccessSecurityGroupID))
- }
- if cur.RemoteAccessSecurityGroupID == "" {
- if retriesLeft > 0 {
- ts.cfg.Logger.Warn("remote access security group ID not found; retrying", zap.String("mng-name", mngName), zap.Int("retries-left", retriesLeft))
- time.Sleep(5 * time.Second)
- return ts.waitForNodes(mngName, retriesLeft-1)
- }
- return fmt.Errorf("remote access security group ID not found for mng %q", mngName)
- }
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName] = cur
- ts.cfg.EKSConfig.Sync()
-
- cur, ok = ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName]
- if !ok {
- return fmt.Errorf("MNGs[%q] not found", mngName)
- }
- asg := dout.Nodegroup.Resources.AutoScalingGroups[0]
- cur.ASGName = aws_v2.ToString(asg.Name)
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName] = cur
- ts.cfg.EKSConfig.Sync()
- ts.cfg.Logger.Info("checking MNG ASG", zap.String("mng-name", cur.Name), zap.String("asg-name", cur.ASGName))
-
- checkN := time.Duration(cur.ASGDesiredCapacity)
- if checkN == 0 {
- checkN = time.Duration(cur.ASGMinSize)
- }
- waitDur := 10*time.Minute + 10*time.Second*checkN
- ctx, cancel := context.WithTimeout(context.Background(), waitDur)
- ec2Instances, err := aws_ec2.WaitUntilRunning(
- ctx,
- ts.cfg.Stopc,
- ts.cfg.EC2APIV2,
- ts.cfg.ASGAPIV2,
- cur.ASGName,
- )
- cancel()
- if err != nil {
- return err
- }
- cur, ok = ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName]
- if !ok {
- return fmt.Errorf("MNGs[%q] not found", mngName)
- }
- cur.Instances = make(map[string]ec2config.Instance)
- for id, vv := range ec2Instances {
- ivv := ec2config.ConvertInstance(vv)
- ivv.RemoteAccessUserName = cur.RemoteAccessUserName
- cur.Instances[id] = ivv
- }
- for _, inst := range cur.Instances {
- ts.cfg.EKSConfig.Status.PrivateDNSToNodeInfo[inst.PrivateDNSName] = eksconfig.NodeInfo{
- NodeGroupName: cur.Name,
- AMIType: cur.AMIType,
- PublicIP: inst.PublicIP,
- PublicDNSName: inst.PublicDNSName,
- UserName: cur.RemoteAccessUserName,
- }
- }
- ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName] = cur
- ts.cfg.EKSConfig.Sync()
-
- cur, ok = ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs[mngName]
- if !ok {
- return fmt.Errorf("MNGs[%q] not found", mngName)
- }
-
- // Hostname/InternalDNS == EC2 private DNS
- // TODO: handle DHCP option domain name
- ec2PrivateDNS := make(map[string]struct{})
- for _, v := range cur.Instances {
- ts.cfg.Logger.Debug("found private DNS for an EC2 instance", zap.String("instance-id", v.InstanceID), zap.String("private-dns-name", v.PrivateDNSName))
- ec2PrivateDNS[v.PrivateDNSName] = struct{}{}
- // "ip-192-168-81-186" from "ip-192-168-81-186.my-private-dns"
- ec2PrivateDNS[strings.Split(v.PrivateDNSName, ".")[0]] = struct{}{}
- }
-
- ts.cfg.Logger.Info("checking nodes readiness")
- retryStart := time.Now()
- ready := false
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("checking node aborted")
- case <-time.After(5 * time.Second):
- }
-
- nodes, err := ts.cfg.K8SClient.ListNodes(1000, 5*time.Second)
- if err != nil {
- ts.cfg.Logger.Warn("get nodes failed", zap.Error(err))
- continue
- }
-
- readies := 0
- for _, node := range nodes {
- labels := node.GetLabels()
- if labels["NGName"] != mngName {
- continue
- }
- nodeName := node.GetName()
- nodeInfo, _ := json.Marshal(k8s_object.ParseNodeInfo(node.Status.NodeInfo))
-
- // e.g. given node name ip-192-168-81-186.us-west-2.compute.internal + DHCP option my-private-dns
- // InternalIP == 192.168.81.186
- // ExternalIP == 52.38.118.149
- // Hostname == my-private-dns (without DHCP option, it's "ip-192-168-81-186.my-private-dns", private DNS, InternalDNS)
- // InternalDNS == ip-192-168-81-186.my-private-dns
- // ExternalDNS == ec2-52-38-118-149.us-west-2.compute.amazonaws.com
- ts.cfg.Logger.Debug("checking node address with EC2 Private DNS",
- zap.String("node-name", nodeName),
- zap.String("node-info", string(nodeInfo)),
- zap.String("labels", fmt.Sprintf("%v", labels)),
- )
-
- hostName := ""
- for _, av := range node.Status.Addresses {
- ts.cfg.Logger.Debug("node status address",
- zap.String("node-name", nodeName),
- zap.String("type", string(av.Type)),
- zap.String("address", string(av.Address)),
- )
- if av.Type != v1.NodeHostName && av.Type != v1.NodeInternalDNS {
- continue
- }
- // handle when node is configured DHCP
- hostName = av.Address
- _, ok := ec2PrivateDNS[hostName]
- if !ok {
- // "ip-192-168-81-186" from "ip-192-168-81-186.my-private-dns"
- _, ok = ec2PrivateDNS[strings.Split(hostName, ".")[0]]
- }
- if ok {
- break
- }
- }
- if hostName == "" {
- return fmt.Errorf("%q not found for node %q", v1.NodeHostName, nodeName)
- }
- _, ok := ec2PrivateDNS[hostName]
- if !ok {
- // "ip-192-168-81-186" from "ip-192-168-81-186.my-private-dns"
- _, ok = ec2PrivateDNS[strings.Split(hostName, ".")[0]]
- }
- if !ok {
- ts.cfg.Logger.Warn("node may not belong to this ASG", zap.String("host-name", hostName), zap.Int("ec2-private-dnss", len(ec2PrivateDNS)))
- continue
- }
- ts.cfg.Logger.Debug("checked node host name with EC2 Private DNS", zap.String("name", nodeName), zap.String("host-name", hostName))
-
- for _, cond := range node.Status.Conditions {
- if cond.Status != v1.ConditionTrue {
- continue
- }
- if cond.Type != v1.NodeReady {
- continue
- }
- ts.cfg.Logger.Debug("node is ready!",
- zap.String("name", nodeName),
- zap.String("status-type", fmt.Sprint(cond.Type)),
- zap.String("status", fmt.Sprint(cond.Status)),
- )
- readies++
- break
- }
- }
- /*
- e.g.
- "/tmp/kubectl-test-v1.16.9 --kubeconfig=/tmp/leegyuho-test-eks.kubeconfig.yaml get csr -o=wide":
- NAME AGE REQUESTOR CONDITION
- csr-4msk5 58s system:node:ip-192-168-65-124.us-west-2.compute.internal Approved,Issued
- csr-9dbs8 57s system:node:ip-192-168-208-6.us-west-2.compute.internal Approved,Issued
- */
- allCSRs := make(map[string]int)
- output, err := ts.cfg.K8SClient.ListCSRs(1000, 5*time.Second)
- if err != nil {
- ts.cfg.Logger.Warn("list CSRs failed", zap.Error(err))
- } else {
- for _, cv := range output {
- k := extractCSRStatus(cv)
- ts.cfg.Logger.Debug("current CSR",
- zap.String("name", cv.GetName()),
- zap.String("requester", cv.Spec.Username),
- zap.String("status", k),
- )
- v, ok := allCSRs[k]
- if !ok {
- allCSRs[k] = 1
- } else {
- allCSRs[k] = v + 1
- }
- }
- }
- ts.cfg.Logger.Info("polling nodes",
- zap.String("command", ts.cfg.EKSConfig.KubectlCommand()+" get nodes"),
- zap.String("mng-name", cur.Name),
- zap.Int("current-ready-nodes", readies),
- zap.Int("min-ready-nodes", cur.ASGMinSize),
- zap.Int("desired-ready-nodes", cur.ASGDesiredCapacity),
- zap.String("all-csrs", fmt.Sprintf("%+v", allCSRs)),
- )
- if readies >= cur.ASGMinSize {
- ready = true
- break
- }
- }
- if !ready {
- return fmt.Errorf("MNG %q not ready", mngName)
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// "pkg/printers/internalversion/printers.go"
-func extractCSRStatus(csr certificatesv1beta1.CertificateSigningRequest) string {
- var approved, denied bool
- for _, c := range csr.Status.Conditions {
- switch c.Type {
- case certificatesv1beta1.CertificateApproved:
- approved = true
- case certificatesv1beta1.CertificateDenied:
- denied = true
- default:
- return ""
- }
- }
- var status string
- // must be in order of presidence
- if denied {
- status += "Denied"
- } else if approved {
- status += "Approved"
- } else {
- status += "Pending"
- }
- if len(csr.Status.Certificate) > 0 {
- status += ",Issued"
- }
- return status
-}
diff --git a/eks/neuron/neuron.go b/eks/neuron/neuron.go
deleted file mode 100644
index 418211a20..000000000
--- a/eks/neuron/neuron.go
+++ /dev/null
@@ -1,567 +0,0 @@
-package neuron
-
-import (
- "bytes"
- "context"
- "errors"
- "fmt"
- "html/template"
- "io"
- "reflect"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/eksconfig"
- "github.com/aws/aws-k8s-tester/pkg/fileutil"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "go.uber.org/zap"
- "k8s.io/utils/exec"
-)
-
-// Config defines Neuron configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
-}
-
-// Tester defines Neuron tester.
-type Tester interface {
- // Name returns the name of the tester.
- Name() string
- InstallNeuronDriver() error
- InstallBertService() error
- CreateBertJob() error
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-// New creates a new Job tester.
-func New(cfg Config) Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
-}
-
-// neuron device plugin for Kubernetes from
-// https://github.com/aws/aws-neuron-sdk/blob/master/docs/neuron-container-tools/k8s-neuron-device-plugin.yml
-const neuronDriverTemplate = `
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
- name: neuron-device-plugin
-rules:
-- apiGroups:
- - ""
- resources:
- - nodes
- verbs:
- - get
- - list
- - watch
-- apiGroups:
- - ""
- resources:
- - events
- verbs:
- - create
- - patch
-- apiGroups:
- - ""
- resources:
- - pods
- verbs:
- - update
- - patch
- - get
- - list
- - watch
-- apiGroups:
- - ""
- resources:
- - nodes/status
- verbs:
- - patch
- - update
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: neuron-device-plugin
- namespace: kube-system
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
- name: neuron-device-plugin
- namespace: kube-system
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: neuron-device-plugin
-subjects:
-- kind: ServiceAccount
- name: neuron-device-plugin
- namespace: kube-system
----
-# https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
- name: neuron-device-plugin-daemonset
- namespace: kube-system
-spec:
- selector:
- matchLabels:
- name: neuron-device-plugin-ds
- updateStrategy:
- type: RollingUpdate
- template:
- metadata:
- annotations:
- scheduler.alpha.kubernetes.io/critical-pod: ""
- labels:
- name: neuron-device-plugin-ds
- spec:
- serviceAccount: neuron-device-plugin
- tolerations:
- - key: CriticalAddonsOnly
- operator: Exists
- - key: aws.amazon.com/neuron
- operator: Exists
- effect: NoSchedule
- # Mark this pod as a critical add-on; when enabled, the critical add-on
- # scheduler reserves resources for critical add-on pods so that they can
- # be rescheduled after a failure.
- # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
- priorityClassName: "system-node-critical"
- affinity:
- nodeAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- nodeSelectorTerms:
- - matchExpressions:
- - key: "beta.kubernetes.io/instance-type"
- operator: In
- values:
- - inf1.xlarge
- - inf1.2xlarge
- - inf1.6xlarge
- - inf1.24xlarge
- - trn1.2xlarge
- - trn1.32xlarge
- - trn2.48xlarge
- - matchExpressions:
- - key: "node.kubernetes.io/instance-type"
- operator: In
- values:
- - inf1.xlarge
- - inf1.2xlarge
- - inf1.6xlarge
- - inf1.24xlarge
- - trn1.2xlarge
- - trn1.32xlarge
- - trn2.48xlarge
- containers:
- - image: public.ecr.aws/neuron/neuron-device-plugin:1.9.3.0
- imagePullPolicy: IfNotPresent
- name: k8s-neuron-device-plugin-ctr
- env:
- - name: KUBECONFIG
- value: /etc/kubernetes/kubelet.conf
- - name: NODE_NAME
- valueFrom:
- fieldRef:
- fieldPath: spec.nodeName
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop: ["ALL"]
- volumeMounts:
- - name: device-plugin
- mountPath: /var/lib/kubelet/device-plugins
- volumes:
- - name: device-plugin
- hostPath:
- path: /var/lib/kubelet/device-plugins
-
-`
-
-func (ts *tester) InstallNeuronDriver() (err error) {
- ts.cfg.Logger.Info("starting tester.InstallNeuronDriver", zap.String("tester", pkgName))
- fpath, err := fileutil.WriteTempFile([]byte(neuronDriverTemplate))
- if err != nil {
- return err
- }
- applyArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "apply",
- "-f",
- fpath,
- }
- applyCmd := strings.Join(applyArgs, " ")
-
- applied := false
- retryStart, waitDur := time.Now(), 3*time.Minute
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("install neuron driver stopped")
- return nil
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
- output, err := exec.New().CommandContext(ctx, applyArgs[0], applyArgs[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", applyCmd, out)
- if err != nil {
- ts.cfg.Logger.Warn("failed to install Neuron driver", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
-
- applied = true
- ts.cfg.Logger.Info("installed neuron driver")
- break
- }
- if !applied {
- return errors.New("failed to install neuron driver")
- }
-
- ts.cfg.Logger.Info("checking neuron driver")
-
- descArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=kube-system",
- "describe",
- "daemonset.apps/neuron-device-plugin-daemonset",
- }
- descCmd := strings.Join(descArgs, " ")
-
- installed := false
- for time.Since(retryStart) < waitDur {
- ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
- out, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
- cancel()
- output := strings.TrimSpace(string(out))
- if err != nil {
- ts.cfg.Logger.Warn("failed to kubectl describe daemonset.apps/neuron-device-plugin-daemonset", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmd, output)
-
- if strings.Contains(output, "SuccessfulCreate") {
- installed = true
- break
- }
- }
-
- if installed {
- ts.cfg.Logger.Info("checked neuron driver")
- return ts.cfg.EKSConfig.Sync()
- }
- ts.cfg.Logger.Warn("failed to install neuron driver")
- return errors.New("neuron driver installation failed")
-}
-
-const bertServiceTemplate = `
----
-kind: Service
-apiVersion: v1
-metadata:
- name: eks-neuron-test-bert-service
- labels:
- app: eks-neuron-test-bert-service
-spec:
- type: ClusterIP
- ports:
- - name: grpc-tf-serving
- port: 8500
- targetPort: 8500
- - name: http-tf-serving
- port: 8501
- targetPort: 8501
- selector:
- app: eks-neuron-test-bert-service
- role: master
----
-kind: Deployment
-apiVersion: apps/v1
-metadata:
- name: eks-neuron-test-bert-service
- labels:
- app: eks-neuron-test-bert-service
- role: master
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: eks-neuron-test-bert-service
- role: master
- template:
- metadata:
- labels:
- app: eks-neuron-test-bert-service
- role: master
- spec:
- initContainers:
- - name: eks-neuron-bert-model
- image: {{ .Account }}.dkr.ecr.{{ .Region }}.amazonaws.com/tensorflow-neuron-bert-model:1.0
- volumeMounts:
- - name: models
- mountPath: /models
- command: ["mv", "/model", "/models/"]
- containers:
- - name: eks-neuron-test-bert-service
- image: {{ .Account }}.dkr.ecr.{{ .Region }}.amazonaws.com/neuron-test-images/tensorflow-inference-neuron:1.15.5-neuron-py37-sdk1.17.1-ubuntu18.04-v1.1
- ports:
- - containerPort: 8500
- - containerPort: 8501
- imagePullPolicy: IfNotPresent
- env:
- - name: AWS_REGION
- value: {{ .Region }}
- - name: S3_USE_HTTPS
- value: "1"
- - name: S3_VERIFY_SSL
- value: "0"
- - name: S3_ENDPOINT
- value: s3.{{ .Region }}.amazonaws.com
- - name: AWS_LOG_LEVEL
- value: "3"
- resources:
- limits:
- cpu: 4
- memory: 4Gi
- aws.amazon.com/neuron: 1
- requests:
- cpu: "1"
- memory: 1Gi
- volumeMounts:
- - name: models
- mountPath: /models
- volumes:
- - name: models
- emptyDir: {}
-`
-
-func (ts *tester) InstallBertService() error {
- ts.cfg.Logger.Info("starting tester.InstallBertService", zap.String("tester", pkgName))
- tpl := template.Must(template.New("tmplBertServiceTemplate").Parse(bertServiceTemplate))
- buf := bytes.NewBuffer(nil)
- if err := tpl.Execute(buf, struct {
- Account string
- Region string
- }{
- Account: ts.cfg.EKSConfig.Status.AWSAccountID,
- Region: ts.cfg.EKSConfig.Region,
- }); err != nil {
- return err
- }
- fpath, err := fileutil.WriteTempFile(buf.Bytes())
- if err != nil {
- return err
- }
- applyArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "apply",
- "-f",
- fpath,
- }
- applyCmd := strings.Join(applyArgs, " ")
-
- applied := false
- retryStart, waitDur := time.Now(), 20*time.Minute
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("install bert service stopped")
- return nil
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- output, err := exec.New().CommandContext(ctx, applyArgs[0], applyArgs[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", applyCmd, out)
- if err != nil {
- ts.cfg.Logger.Warn("failed to install bert service", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
-
- applied = true
- ts.cfg.Logger.Info("installed bert service")
- break
- }
- if !applied {
- return errors.New("failed to install bert service")
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
- _, err = k8s_client.WaitForDeploymentCompletes(
- ctx,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.Stopc,
- ts.cfg.K8SClient,
- time.Minute,
- 20*time.Second,
- "default",
- "eks-neuron-test-bert-service",
- 1,
- k8s_client.WithQueryFunc(func() {
- descArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "describe",
- "deployment",
- "eks-neuron-test-bert-service",
- }
- descCmd := strings.Join(descArgs, " ")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe deployment' failed", zap.Error(err))
- }
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", descCmd, out)
- }),
- )
- cancel()
- if err != nil {
- return errors.New("failed to deploy bert service")
- }
- return ts.cfg.EKSConfig.Sync()
-}
-
-const bertJobTemplate = `
-apiVersion: batch/v1
-kind: Job
-metadata:
- labels:
- k8s-app: bert-client-app
- # Unique key of the Job instance
- name: bert-client
-spec:
- template:
- metadata:
- name: bert-client
- spec:
- containers:
- - name: bert-client-container
- image: {{ .Account }}.dkr.ecr.{{ .Region }}.amazonaws.com/bert-client:2.0
- imagePullPolicy: IfNotPresent
- command: ["/bin/sh","-c"]
- args:
- - python3 bert_client.py eks-neuron-test-bert-service:8500 model;
-
- # Do not restart containers after they exit
- restartPolicy: Never
- # of retries before marking as failed.
- backoffLimit: 3
-`
-
-func (ts *tester) CreateBertJob() error {
- ts.cfg.Logger.Info("starting tester.CreateBertJob", zap.String("tester", pkgName))
- tpl := template.Must(template.New("tmplBertJobTemplate").Parse(bertJobTemplate))
- buf := bytes.NewBuffer(nil)
- if err := tpl.Execute(buf, struct {
- Account string
- Region string
- }{
- Account: ts.cfg.EKSConfig.Status.AWSAccountID,
- Region: ts.cfg.EKSConfig.Region,
- }); err != nil {
- return err
- }
- fpath, err := fileutil.WriteTempFile(buf.Bytes())
- if err != nil {
- return err
- }
- applyArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "apply",
- "-f",
- fpath,
- }
- applyCmd := strings.Join(applyArgs, " ")
-
- applied := false
- retryStart, waitDur := time.Now(), 10*time.Minute
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("create bert job stopped")
- return nil
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
- output, err := exec.New().CommandContext(ctx, applyArgs[0], applyArgs[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", applyCmd, out)
- if err != nil {
- ts.cfg.Logger.Warn("failed to create bert job", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
-
- applied = true
- ts.cfg.Logger.Info("created bert job")
- break
- }
- if !applied {
- return errors.New("failed to create bert job")
- }
-
- ts.cfg.Logger.Info("checking bert job")
-
- getArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "get",
- "pods",
- "--selector=job-name=bert-client",
- }
- getCmd := strings.Join(getArgs, " ")
-
- completed := false
- for time.Since(retryStart) < waitDur {
- ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
- out, err := exec.New().CommandContext(ctx, getArgs[0], getArgs[1:]...).CombinedOutput()
- cancel()
- output := strings.TrimSpace(string(out))
- if err != nil {
- ts.cfg.Logger.Warn("failed to kubectl get pods --selector=job-name=bert-client", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getCmd, output)
-
- if strings.Contains(output, "Completed") {
- completed = true
- break
- }
- }
-
- if completed {
- ts.cfg.Logger.Info("checked bert job")
- return ts.cfg.EKSConfig.Sync()
- }
- ts.cfg.Logger.Warn("failed to test bert job")
- return errors.New("bert job failed")
-}
diff --git a/eks/ng/asg.go b/eks/ng/asg.go
deleted file mode 100644
index 411dab506..000000000
--- a/eks/ng/asg.go
+++ /dev/null
@@ -1,446 +0,0 @@
-package ng
-
-import (
- "context"
- "encoding/base64"
- "errors"
- "fmt"
- "sort"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- aws_v2 "github.com/aws/aws-sdk-go-v2/aws"
- aws_asg_v2 "github.com/aws/aws-sdk-go-v2/service/autoscaling"
- aws_asg_v2_types "github.com/aws/aws-sdk-go-v2/service/autoscaling/types"
- aws_ec2_v2 "github.com/aws/aws-sdk-go-v2/service/ec2"
- aws_ec2_v2_types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
- aws_eks_v2_types "github.com/aws/aws-sdk-go-v2/service/eks/types"
- aws_ssm_v2 "github.com/aws/aws-sdk-go-v2/service/ssm"
- smithy "github.com/aws/smithy-go"
- "go.uber.org/zap"
-)
-
-func (ts *tester) createASGs() (err error) {
- tss, err := ts._createASGs()
- if err != nil {
- return err
- }
- if err = ts.waitForASGs(tss); err != nil {
- return err
- }
- return nil
-}
-
-func (ts *tester) deleteASGs() error {
- var errs []string
-
- if err := ts._deleteASGs(); err != nil {
- ts.cfg.Logger.Warn("failed to delete ASGs", zap.Error(err))
- errs = append(errs, err.Error())
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ","))
- }
- return nil
-}
-
-// track timestamps and check status in reverse order to minimize polling API calls
-func (ts *tester) _createASGs() (tss tupleTimes, err error) {
- ts.cfg.Logger.Info("creating ASGs")
-
- for asgName, cur := range ts.cfg.EKSConfig.AddOnNodeGroups.ASGs {
- imgID := cur.ImageID
- if imgID == "" {
- imgID, err = ts.fetchImageID(cur.ImageIDSSMParameter)
- if err != nil {
- return nil, err
- }
- }
- ts.cfg.Logger.Info("creating launch template",
- zap.String("launch-template-name", cur.LaunchTemplateName),
- zap.String("image-id", imgID),
- )
-
- userData, err := ts.generateUserData(asgName, cur.AMIType, cur.KubeletExtraArgs, cur.BootstrapArgs)
- if err != nil {
- return nil, fmt.Errorf("failed to create user data for %q (%v)", asgName, err)
- }
- userData = base64.StdEncoding.EncodeToString([]byte(userData))
-
- _, err = ts.cfg.EC2APIV2.CreateLaunchTemplate(
- context.Background(),
- &aws_ec2_v2.CreateLaunchTemplateInput{
- LaunchTemplateName: aws_v2.String(cur.LaunchTemplateName),
-
- LaunchTemplateData: &aws_ec2_v2_types.RequestLaunchTemplateData{
- IamInstanceProfile: &aws_ec2_v2_types.LaunchTemplateIamInstanceProfileSpecificationRequest{
- Arn: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.InstanceProfileARN),
- },
-
- KeyName: aws_v2.String(ts.cfg.EKSConfig.RemoteAccessKeyName),
-
- ImageId: aws_v2.String(imgID),
- InstanceType: aws_ec2_v2_types.InstanceType(cur.InstanceType),
-
- BlockDeviceMappings: []aws_ec2_v2_types.LaunchTemplateBlockDeviceMappingRequest{
- {
- DeviceName: aws_v2.String("/dev/xvda"),
- Ebs: &aws_ec2_v2_types.LaunchTemplateEbsBlockDeviceRequest{
- DeleteOnTermination: aws_v2.Bool(true),
- Encrypted: aws_v2.Bool(true),
- VolumeType: cur.VolumeType,
- VolumeSize: aws_v2.Int32(cur.VolumeSize),
- },
- },
- },
-
- // for public DNS + SSH access
- NetworkInterfaces: []aws_ec2_v2_types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{
- {
- AssociatePublicIpAddress: aws_v2.Bool(true),
- DeleteOnTermination: aws_v2.Bool(true),
- DeviceIndex: aws_v2.Int32(0),
- Groups: []string{ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID},
- },
- },
-
- UserData: aws_v2.String(userData),
-
- Monitoring: &aws_ec2_v2_types.LaunchTemplatesMonitoringRequest{Enabled: aws_v2.Bool(true)},
- InstanceInitiatedShutdownBehavior: aws_ec2_v2_types.ShutdownBehaviorTerminate,
- },
-
- TagSpecifications: []aws_ec2_v2_types.TagSpecification{
- {
- ResourceType: aws_ec2_v2_types.ResourceTypeLaunchTemplate,
- Tags: []aws_ec2_v2_types.Tag{
- {
- Key: aws_v2.String("Name"),
- Value: aws_v2.String(fmt.Sprintf("%s-instance-launch-template", cur.Name)),
- },
- },
- },
- },
- },
- )
- if err != nil {
- return nil, fmt.Errorf("failed to create launch template for %q (%v)", asgName, err)
- }
-
- select {
- case <-time.After(10 * time.Second):
- case <-ts.cfg.Stopc:
- return nil, errors.New("stopped")
- }
-
- ts.cfg.Logger.Info("creating ASG",
- zap.String("asg-name", asgName),
- zap.String("image-id", imgID),
- )
-
- // TOOD: tag instance and volume
- // Valid requests must contain either LaunchTemplate, LaunchConfigurationName, InstanceId or MixedInstancesPolicy parameter
- asgInput := &aws_asg_v2.CreateAutoScalingGroupInput{
- AutoScalingGroupName: aws_v2.String(asgName),
- MaxSize: aws_v2.Int32(cur.ASGMaxSize),
- MinSize: aws_v2.Int32(cur.ASGMinSize),
- VPCZoneIdentifier: aws_v2.String(strings.Join(ts.cfg.EKSConfig.VPC.PublicSubnetIDs, ",")),
- HealthCheckGracePeriod: aws_v2.Int32(300),
- HealthCheckType: aws_v2.String("EC2"),
- LaunchTemplate: &aws_asg_v2_types.LaunchTemplateSpecification{
- LaunchTemplateName: aws_v2.String(cur.LaunchTemplateName),
- Version: aws_v2.String("$Latest"),
- },
- Tags: []aws_asg_v2_types.Tag{
- {
- Key: aws_v2.String("Name"),
- Value: aws_v2.String(cur.Name),
- PropagateAtLaunch: aws_v2.Bool(true),
- },
- {
- Key: aws_v2.String(fmt.Sprintf("kubernetes.io/cluster/%s", ts.cfg.EKSConfig.Name)),
- Value: aws_v2.String("owned"),
- PropagateAtLaunch: aws_v2.Bool(true),
- },
- {
- Key: aws_v2.String(fmt.Sprintf("kubernetes.io/cluster-autoscaler/%s", ts.cfg.EKSConfig.Name)),
- Value: aws_v2.String("owned"),
- PropagateAtLaunch: aws_v2.Bool(true),
- },
- {
- Key: aws_v2.String("kubernetes.io/cluster-autoscaler/enabled"),
- Value: aws_v2.String("true"),
- PropagateAtLaunch: aws_v2.Bool(true),
- },
- },
- }
- if cur.ASGDesiredCapacity > 0 {
- asgInput.DesiredCapacity = aws_v2.Int32(cur.ASGDesiredCapacity)
- }
- _, err = ts.cfg.ASGAPIV2.CreateAutoScalingGroup(context.Background(), asgInput)
- if err != nil {
- return nil, fmt.Errorf("failed to create ASG for %q (%v)", asgName, err)
- }
-
- cur.Instances = make(map[string]ec2config.Instance)
- cur.Logs = make(map[string][]string)
- ts.cfg.EKSConfig.AddOnNodeGroups.ASGs[asgName] = cur
- ts.cfg.EKSConfig.AddOnNodeGroups.Created = true
- ts.cfg.EKSConfig.Sync()
-
- tss = append(tss, tupleTime{ts: time.Now(), name: asgName})
- }
-
- sort.Sort(sort.Reverse(tss))
- ts.cfg.Logger.Info("created ASGs")
- return tss, nil
-}
-
-func (ts *tester) _deleteASGs() (err error) {
- ts.cfg.Logger.Info("deleting ASGs")
-
- for asgName, cur := range ts.cfg.EKSConfig.AddOnNodeGroups.ASGs {
- _, err = ts.cfg.ASGAPIV2.DeleteAutoScalingGroup(
- context.Background(),
- &aws_asg_v2.DeleteAutoScalingGroupInput{
- AutoScalingGroupName: aws_v2.String(asgName),
- ForceDelete: aws_v2.Bool(true),
- },
- )
- if err != nil {
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- ts.cfg.EKSConfig.Status.DeletedResources[asgName] = "AddOnNodeGroups.ASGs"
- ts.cfg.EKSConfig.Sync()
- return nil
- }
- }
- return fmt.Errorf("failed to delete ASG for %q (%v)", asgName, err)
- }
-
- select {
- case <-time.After(30 * time.Second):
- case <-ts.cfg.Stopc:
- return errors.New("stopped")
- }
-
- for i := 0; i < int(cur.ASGDesiredCapacity); i++ {
- if _, ok := ts.cfg.EKSConfig.Status.DeletedResources[asgName]; ok {
- break
- }
- select {
- case <-time.After(5 * time.Second):
- case <-ts.cfg.Stopc:
- return errors.New("stopped")
- }
-
- ts.cfg.Logger.Info("polling ASG until deletion", zap.String("asg-name", asgName))
- aout, err := ts.cfg.ASGAPIV2.DescribeAutoScalingGroups(
- context.Background(),
- &aws_asg_v2.DescribeAutoScalingGroupsInput{
- AutoScalingGroupNames: []string{asgName},
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to describe ASG", zap.String("asg-name", asgName), zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- ts.cfg.EKSConfig.Status.DeletedResources[asgName] = "AddOnNodeGroups.ASGs"
- ts.cfg.EKSConfig.Sync()
- break
- }
- }
- continue
- }
- ts.cfg.Logger.Info("described ASG",
- zap.String("asg-name", asgName),
- zap.Int("results", len(aout.AutoScalingGroups)),
- )
- if len(aout.AutoScalingGroups) == 0 {
- ts.cfg.EKSConfig.Status.DeletedResources[asgName] = "AddOnNodeGroups.ASGs"
- ts.cfg.EKSConfig.Sync()
- break
- }
- if len(aout.AutoScalingGroups[0].Instances) == 0 {
- ts.cfg.EKSConfig.Status.DeletedResources[asgName] = "AddOnNodeGroups.ASGs"
- ts.cfg.EKSConfig.Sync()
- break
- }
- ts.cfg.Logger.Info("ASG still has instances; retrying",
- zap.String("asg-name", asgName),
- zap.Int("instances", len(aout.AutoScalingGroups[0].Instances)),
- )
- }
-
- for i := 0; i < int(cur.ASGDesiredCapacity); i++ {
- if _, ok := ts.cfg.EKSConfig.Status.DeletedResources[cur.LaunchTemplateName]; ok {
- break
- }
- select {
- case <-time.After(5 * time.Second):
- case <-ts.cfg.Stopc:
- return errors.New("stopped")
- }
-
- _, err = ts.cfg.EC2APIV2.DeleteLaunchTemplate(
- context.Background(),
- &aws_ec2_v2.DeleteLaunchTemplateInput{
- LaunchTemplateName: aws_v2.String(cur.LaunchTemplateName),
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to delete launch template", zap.String("name", cur.LaunchTemplateName), zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- ts.cfg.EKSConfig.Status.DeletedResources[cur.LaunchTemplateName] = "AddOnNodeGroups.ASGs.LaunchTemplateName"
- ts.cfg.EKSConfig.Sync()
- break
- }
- }
- continue
- }
-
- ts.cfg.EKSConfig.Status.DeletedResources[cur.LaunchTemplateName] = "AddOnNodeGroups.ASGs.LaunchTemplateName"
- ts.cfg.EKSConfig.Sync()
- break
- }
- }
-
- ts.cfg.Logger.Info("deleted ASGs")
- return nil
-}
-
-func (ts *tester) waitForASGs(tss tupleTimes) (err error) {
- ts.cfg.Logger.Info("waiting for ASGs")
-
- for _, tv := range tss {
- asgName := tv.name
- cur, ok := ts.cfg.EKSConfig.AddOnNodeGroups.ASGs[asgName]
- if !ok {
- return fmt.Errorf("ASG name %q not found after creation", asgName)
- }
-
- select {
- case <-time.After(10 * time.Second):
- case <-ts.cfg.Stopc:
- return errors.New("stopped")
- }
-
- ts.cfg.Logger.Info("waiting for ASG", zap.String("asg-name", asgName))
-
- timeStart := time.Now()
- if err := ts.nodeWaiter.Wait(asgName, 10); err != nil {
- return err
- }
- timeEnd := time.Now()
-
- cur, ok = ts.cfg.EKSConfig.AddOnNodeGroups.ASGs[asgName]
- if !ok {
- return fmt.Errorf("ASG name %q not found after creation", asgName)
- }
- cur.TimeFrameCreate = timeutil.NewTimeFrame(timeStart, timeEnd)
- ts.cfg.EKSConfig.AddOnNodeGroups.ASGs[asgName] = cur
- ts.cfg.EKSConfig.Sync()
- }
-
- ts.cfg.Logger.Info("waited for ASGs")
- return nil
-}
-
-func (ts *tester) fetchImageID(ssmParam string) (img string, err error) {
- if ssmParam == "" {
- return "", errors.New("empty SSM parameter")
- }
- out, err := ts.cfg.SSMAPIV2.GetParameter(
- context.Background(),
- &aws_ssm_v2.GetParameterInput{
- Name: aws_v2.String(ssmParam),
- },
- )
- if err != nil {
- return "", err
- }
- return aws_v2.ToString(out.Parameter.Value), nil
-}
-
-func (ts *tester) generateUserData(asgName string, amiType string, kubeletExtraArgs string, bootstrapArgs string) (d string, err error) {
- switch amiType {
- case ec2config.AMITypeWindowsServerCore2019X8664:
- d = fmt.Sprintf(`
-
-[string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS"
-[string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1'
-[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName"
-& $EKSBootstrapScriptFile -EKSClusterName %s -Base64ClusterCA %s -APIServerEndpoint %s -KubeletExtraArgs %s 3>&1 4>&1 5>&1 6>&1
-`,
- ts.cfg.EKSConfig.Name,
- ts.cfg.EKSConfig.Status.ClusterCA,
- ts.cfg.EKSConfig.Status.ClusterAPIServerEndpoint,
- fmt.Sprintf(`"--node-labels=NodeType=regular,AMIType=%s,NGType=custom,NGName=%s %s"`, amiType, asgName, kubeletExtraArgs))
-
- case ec2config.AMITypeBottleRocketCPU:
- d = fmt.Sprintf(`[settings.kubernetes]
-cluster-name = "%s"
-cluster-certificate = "%s"
-api-server = "%s"
-[settings.kubernetes.node-labels]
-NodeType = "regular"
-AMIType = "%s"
-NGType = "custom"
-NGName = "%s"`,
- ts.cfg.EKSConfig.Name,
- ts.cfg.EKSConfig.Status.ClusterCA,
- ts.cfg.EKSConfig.Status.ClusterAPIServerEndpoint,
- ec2config.AMITypeBottleRocketCPU,
- asgName,
- )
-
- case fmt.Sprint(aws_eks_v2_types.AMITypesAl2X8664),
- fmt.Sprint(aws_eks_v2_types.AMITypesAl2Arm64),
- fmt.Sprint(aws_eks_v2_types.AMITypesAl2X8664Gpu):
- d = fmt.Sprintf(`#!/bin/bash
-set -xeu
-
-/etc/eks/bootstrap.sh %s`, ts.cfg.EKSConfig.Name)
- if ts.cfg.EKSConfig.ResolverURL != "" {
- clusterVPCIP := ts.cfg.EKSConfig.VPC.CIDRs[0]
- dnsClusterIP := "10.100.0.10"
- if clusterVPCIP[:strings.IndexByte(clusterVPCIP, '.')] == "10" {
- dnsClusterIP = "172.20.0.10"
- }
- ts.cfg.Logger.Info("adding extra bootstrap arguments --b64-cluster-ca and --apiserver-endpoint to user data",
- zap.String("b64-cluster-ca", ts.cfg.EKSConfig.Status.ClusterCA),
- zap.String("apiserver-endpoint", ts.cfg.EKSConfig.Status.ClusterAPIServerEndpoint),
- zap.String("dns-cluster-ip", dnsClusterIP),
- )
- d += fmt.Sprintf(` --b64-cluster-ca %s --apiserver-endpoint %s --dns-cluster-ip %s`,
- ts.cfg.EKSConfig.Status.ClusterCA,
- ts.cfg.EKSConfig.Status.ClusterAPIServerEndpoint,
- dnsClusterIP,
- )
- }
- // https://aws.amazon.com/blogs/opensource/improvements-eks-worker-node-provisioning/
- d += fmt.Sprintf(` --kubelet-extra-args '--node-labels=NodeType=regular,AMIType=%s,NGType=custom,NGName=%s`, amiType, asgName)
- if kubeletExtraArgs != "" {
- ts.cfg.Logger.Info("adding extra bootstrap arguments --kubelet-extra-args to user data",
- zap.String("kubelet-extra-args", kubeletExtraArgs),
- )
- d += fmt.Sprintf(` %s`, kubeletExtraArgs)
- }
- d += "'"
- if bootstrapArgs != "" {
- ts.cfg.Logger.Info("adding further additional bootstrap arguments to user data",
- zap.String("bootstrap-args", bootstrapArgs),
- )
- d += fmt.Sprintf(` %s`, bootstrapArgs)
- }
- }
-
- return d, nil
-}
diff --git a/eks/ng/autoscaler/autoscaler.go b/eks/ng/autoscaler/autoscaler.go
deleted file mode 100644
index 4fa9763e4..000000000
--- a/eks/ng/autoscaler/autoscaler.go
+++ /dev/null
@@ -1,3 +0,0 @@
-// Package autoscaler implements various auto-scaler.
-// ref. https://github.com/kubernetes/autoscaler
-package autoscaler
diff --git a/eks/ng/autoscaler/cluster-autoscaler.go b/eks/ng/autoscaler/cluster-autoscaler.go
deleted file mode 100644
index 61f2a214c..000000000
--- a/eks/ng/autoscaler/cluster-autoscaler.go
+++ /dev/null
@@ -1,359 +0,0 @@
-package autoscaler
-
-import (
- "bytes"
- "context"
- "errors"
- "fmt"
- "io"
- "reflect"
- "strings"
- "text/template"
- "time"
-
- "github.com/aws/aws-k8s-tester/eksconfig"
- "github.com/aws/aws-k8s-tester/pkg/fileutil"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "go.uber.org/zap"
- "k8s.io/utils/exec"
-)
-
-// auto discover
-// ref. https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml
-const clusterAutoscalerYAML = `
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- labels:
- k8s-addon: cluster-autoscaler.addons.k8s.io
- k8s-app: cluster-autoscaler
- name: cluster-autoscaler
- namespace: kube-system
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- name: cluster-autoscaler
- labels:
- k8s-addon: cluster-autoscaler.addons.k8s.io
- k8s-app: cluster-autoscaler
-rules:
- - apiGroups: [""]
- resources: ["events", "endpoints"]
- verbs: ["create", "patch"]
- - apiGroups: [""]
- resources: ["pods/eviction"]
- verbs: ["create"]
- - apiGroups: [""]
- resources: ["pods/status"]
- verbs: ["update"]
- - apiGroups: [""]
- resources: ["endpoints"]
- resourceNames: ["cluster-autoscaler"]
- verbs: ["get", "update"]
- - apiGroups: [""]
- resources: ["nodes"]
- verbs: ["watch", "list", "get", "update"]
- - apiGroups: [""]
- resources:
- - "pods"
- - "services"
- - "replicationcontrollers"
- - "persistentvolumeclaims"
- - "persistentvolumes"
- verbs: ["watch", "list", "get"]
- - apiGroups: ["extensions"]
- resources: ["replicasets", "daemonsets"]
- verbs: ["watch", "list", "get"]
- - apiGroups: ["policy"]
- resources: ["poddisruptionbudgets"]
- verbs: ["watch", "list"]
- - apiGroups: ["apps"]
- resources: ["statefulsets", "replicasets", "daemonsets"]
- verbs: ["watch", "list", "get"]
- - apiGroups: ["storage.k8s.io"]
- resources: ["storageclasses", "csinodes"]
- verbs: ["watch", "list", "get"]
- - apiGroups: ["batch", "extensions"]
- resources: ["jobs"]
- verbs: ["get", "list", "watch", "patch"]
- - apiGroups: ["coordination.k8s.io"]
- resources: ["leases"]
- verbs: ["create"]
- - apiGroups: ["coordination.k8s.io"]
- resourceNames: ["cluster-autoscaler"]
- resources: ["leases"]
- verbs: ["get", "update"]
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
- name: cluster-autoscaler
- namespace: kube-system
- labels:
- k8s-addon: cluster-autoscaler.addons.k8s.io
- k8s-app: cluster-autoscaler
-rules:
- - apiGroups: [""]
- resources: ["configmaps"]
- verbs: ["create","list","watch"]
- - apiGroups: [""]
- resources: ["configmaps"]
- resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
- verbs: ["delete", "get", "update", "watch"]
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- name: cluster-autoscaler
- labels:
- k8s-addon: cluster-autoscaler.addons.k8s.io
- k8s-app: cluster-autoscaler
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: cluster-autoscaler
-subjects:
- - kind: ServiceAccount
- name: cluster-autoscaler
- namespace: kube-system
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
- name: cluster-autoscaler
- namespace: kube-system
- labels:
- k8s-addon: cluster-autoscaler.addons.k8s.io
- k8s-app: cluster-autoscaler
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: Role
- name: cluster-autoscaler
-subjects:
- - kind: ServiceAccount
- name: cluster-autoscaler
- namespace: kube-system
-
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: cluster-autoscaler
- namespace: kube-system
- labels:
- app: cluster-autoscaler
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: cluster-autoscaler
- template:
- metadata:
- labels:
- app: cluster-autoscaler
- annotations:
- prometheus.io/scrape: 'true'
- prometheus.io/port: '8085'
- cluster-autoscaler.kubernetes.io/safe-to-evict: 'false'
- spec:
- serviceAccountName: cluster-autoscaler
- containers:
-{{ if ne .ImageURI "" }}{{.ImageURI}}{{ end }}
- name: cluster-autoscaler
- resources:
- limits:
- cpu: 100m
- memory: 300Mi
- requests:
- cpu: 100m
- memory: 300Mi
- command:
- - ./cluster-autoscaler
- - --v=4
- - --stderrthreshold=info
- - --cloud-provider=aws
- - --skip-nodes-with-local-storage=false
- - --expander=least-waste
-{{ if ne .NodeGroupAutoDiscovery "" }}{{.NodeGroupAutoDiscovery}}{{ end }}
- - --balance-similar-node-groups
- - --skip-nodes-with-system-pods=false
- volumeMounts:
- - name: ssl-certs
- mountPath: /etc/ssl/certs/ca-certificates.crt
- readOnly: true
- imagePullPolicy: "Always"
- volumes:
- - name: ssl-certs
- hostPath:
- path: "/etc/ssl/certs/ca-bundle.crt"
-`
-
-// ref. https://github.com/kubernetes/autoscaler/releases
-var caImages = map[string]string{
- "1.16": ` - image: us.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler:v1.16.5`,
- "1.17": ` - image: us.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler:v1.17.2`,
- "1.18": ` - image: us.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler:v1.18.0`,
- "1.19": ` - image: us.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler:v1.19.0`,
- "1.20": ` - image: us.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler:v1.20.0`,
-}
-
-const (
- nodeGroupAuotDiscoveryData = ` - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/`
- clusterAutoscalerDeploymentName = "cluster-autoscaler"
-)
-
-type caSpecData struct {
- ImageURI string
- NodeGroupAutoDiscovery string
-}
-
-// Config defines version upgrade configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
-}
-
-// ClusterAutoscaler defines cluster autoscaler operation.
-type ClusterAutoscaler interface {
- Create() error
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-// New creates a new cluster autoscaler.
-func New(cfg Config) ClusterAutoscaler {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
-}
-
-func (ts *tester) Create() error {
- needInstall := false
- for _, cur := range ts.cfg.EKSConfig.AddOnNodeGroups.ASGs {
- if cur.ClusterAutoscaler != nil && cur.ClusterAutoscaler.Enable {
- needInstall = true
- break
- }
- }
- if !needInstall {
- ts.cfg.Logger.Info("no NG enables CA; skipping")
- return nil
- }
- return ts.installCA()
-}
-
-func (ts *tester) installCA() error {
- ts.cfg.Logger.Info("creating CA using kubectl", zap.String("name", ts.cfg.EKSConfig.Name))
- var ok bool
- var caData = caSpecData{}
- caData.ImageURI, ok = caImages[ts.cfg.EKSConfig.Version]
- if !ok {
- return fmt.Errorf("no CA found for %q", ts.cfg.EKSConfig.Version)
- }
- caData.NodeGroupAutoDiscovery = nodeGroupAuotDiscoveryData + ts.cfg.EKSConfig.Name
- tpl := template.Must(template.New("TemplateCA").Parse(clusterAutoscalerYAML))
- buf := bytes.NewBuffer(nil)
- if err := tpl.Execute(buf, caData); err != nil {
- return err
- }
- ts.cfg.Logger.Info("writing cluster autoscaler YAML")
- fpath, err := fileutil.WriteTempFile(buf.Bytes())
- if err != nil {
- ts.cfg.Logger.Warn("failed to write cluster-autoscaler YAML", zap.Error(err))
- return err
- }
-
- ts.cfg.Logger.Info("applying cluster-autoscaler YAML", zap.String("path", fpath))
- var output []byte
- waitDur := 5 * time.Minute
- retryStart := time.Now()
- applyArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "apply",
- "-f",
- fpath,
- }
- applyCmd := strings.Join(applyArgs, " ")
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("create CA aborted")
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, applyArgs[0], applyArgs[1:]...).CombinedOutput()
- cancel()
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", applyCmd, out)
- if err == nil {
- break
- }
- if strings.Contains(out, " created") || strings.Contains(out, " unchanged") {
- err = nil
- break
- }
-
- ts.cfg.Logger.Warn("create CA failed", zap.Error(err))
- ts.cfg.EKSConfig.RecordStatus(fmt.Sprintf("create CA failed (%v)", err))
- }
- if err != nil {
- return fmt.Errorf("'kubectl apply' failed %v (output %q)", err, string(output))
- }
- ts.cfg.Logger.Info("created cluster autoscaler")
-
- return ts.waitDeploymentCA()
-}
-
-func (ts *tester) waitDeploymentCA() (err error) {
- timeout := 7 * time.Minute
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- _, err = k8s_client.WaitForDeploymentCompletes(
- ctx,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.Stopc,
- ts.cfg.K8SClient,
- time.Minute,
- 20*time.Second,
- "kube-system",
- clusterAutoscalerDeploymentName,
- 1,
- k8s_client.WithQueryFunc(func() {
- descArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=kube-system",
- "describe",
- "deployment",
- clusterAutoscalerDeploymentName,
- }
- descCmd := strings.Join(descArgs, " ")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe deployment' failed", zap.Error(err))
- }
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", descCmd, out)
- }),
- )
- cancel()
- return err
-}
diff --git a/eks/ng/configmap.go b/eks/ng/configmap.go
deleted file mode 100644
index c21442d4d..000000000
--- a/eks/ng/configmap.go
+++ /dev/null
@@ -1,127 +0,0 @@
-package ng
-
-import (
- "bytes"
- "context"
- "errors"
- "fmt"
- "strings"
- "text/template"
- "time"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/pkg/fileutil"
- "go.uber.org/zap"
- "k8s.io/utils/exec"
-)
-
-func (ts *tester) createConfigMap() error {
- if ts.cfg.EKSConfig.AddOnNodeGroups.Role.ARN == "" {
- return errors.New("empty AddOnNodeGroups.Role.ARN")
- }
-
- ts.cfg.Logger.Info("writing ConfigMap", zap.String("instance-role-arn", ts.cfg.EKSConfig.AddOnNodeGroups.Role.ARN))
- body, p, err := ts.writeConfigMapAuth(ts.cfg.EKSConfig.AddOnNodeGroups.Role.ARN)
- if err != nil {
- return err
- }
- ts.cfg.Logger.Info("applying ConfigMap")
- fmt.Fprintf(ts.cfg.LogWriter, "\naws-auth ConfigMap:\n\n%s\n", body)
-
- var output []byte
- // might take several minutes for DNS to propagate
- waitDur := 5 * time.Minute
- retryStart := time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("create ConfigMap aborted")
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(
- ctx,
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig="+ts.cfg.EKSConfig.KubeConfigPath,
- "apply",
- "--filename="+p,
- ).CombinedOutput()
- cancel()
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\"kubectl apply\" output:\n%s\n", out)
- if err == nil {
- break
- }
- // "configmap/aws-auth created" or "configmap/aws-auth unchanged"
- if strings.Contains(out, " created") || strings.Contains(out, " unchanged") {
- err = nil
- break
- }
-
- ts.cfg.Logger.Warn("create ConfigMap failed", zap.Error(err))
- ts.cfg.EKSConfig.RecordStatus(fmt.Sprintf("create ConfigMap failed (%v)", err))
- }
- if err != nil {
- return fmt.Errorf("'kubectl apply' failed %v (output %q)", err, string(output))
- }
-
- ts.cfg.Logger.Info("created ConfigMap")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// TODO: use client-go
-// https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html
-const configMapAuthTempl = `---
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: aws-auth
- namespace: kube-system
-data:
- mapRoles: |
- - rolearn: {{.NGInstanceRoleARN}}
- %s
- groups:
- %s
-`
-
-const LinuxInstanceRBACGroups = `- system:bootstrappers
- - system:nodes
-`
-
-const WindowsInstanceAdditionalRBACGroups = ` - eks:kube-proxy-windows`
-
-type configMapAuth struct {
- NGInstanceRoleARN string
-}
-
-func (ts *tester) writeConfigMapAuth(instanceRoleARN string) (body string, fpath string, err error) {
- kc := configMapAuth{NGInstanceRoleARN: instanceRoleARN}
- nodeRoleRBACGroupList := LinuxInstanceRBACGroups
- // If Windows nodes are present then add additional kube-proxy role
- if ts.hasWindowsNode() {
- nodeRoleRBACGroupList += WindowsInstanceAdditionalRBACGroups
- }
-
- tpl := template.Must(template.New("configMapAuthTempl").Parse(configMapAuthTempl))
- buf := bytes.NewBuffer(nil)
- if err = tpl.Execute(buf, kc); err != nil {
- return "", "", err
- }
- // avoid '{{' conflicts with Go
- body = fmt.Sprintf(buf.String(), `username: system:node:{{EC2PrivateDNSName}}`, nodeRoleRBACGroupList)
- fpath, err = fileutil.WriteTempFile([]byte(body))
- return body, fpath, err
-}
-
-// hasWindowsNode returns true if any Windows AMI is present in the the ASG to be created
-func (ts *tester) hasWindowsNode() bool {
- for _, cur := range ts.cfg.EKSConfig.AddOnNodeGroups.ASGs {
- if cur.AMIType == ec2config.AMITypeWindowsServerCore2019X8664 {
- return true
- }
- }
- return false
-}
diff --git a/eks/ng/enis.go b/eks/ng/enis.go
deleted file mode 100644
index 8cfb016d0..000000000
--- a/eks/ng/enis.go
+++ /dev/null
@@ -1,142 +0,0 @@
-package ng
-
-import (
- "context"
- "errors"
- "fmt"
- "strings"
- "time"
-
- aws_v2 "github.com/aws/aws-sdk-go-v2/aws"
- aws_ec2_v2 "github.com/aws/aws-sdk-go-v2/service/ec2"
- aws_ec2_v2_types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
- smithy "github.com/aws/smithy-go"
- "go.uber.org/zap"
-)
-
-func (ts *tester) deleteENIs() bool {
- if ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID == "" {
- ts.cfg.Logger.Warn("empty security group ID; returning")
- return false
- }
-
- ts.cfg.Logger.Info("deleting ENIs for security group", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- out, err := ts.cfg.EC2APIV2.DescribeNetworkInterfaces(
- context.Background(),
- &aws_ec2_v2.DescribeNetworkInterfacesInput{
- Filters: []aws_ec2_v2_types.Filter{
- {
- Name: aws_v2.String("group-id"),
- Values: []string{ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID},
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to describe ENIs", zap.Error(err))
- return false
- }
-
- enis := make([]aws_ec2_v2_types.NetworkInterface, 0)
- for _, eni := range out.NetworkInterfaces {
- enis = append(enis, eni)
- ts.cfg.Logger.Info("found ENI", zap.String("eni", aws_v2.ToString(eni.NetworkInterfaceId)))
- }
-
- // detacth and delete ENIs
- deleted := false
- for _, eni := range enis {
- eniID := aws_v2.ToString(eni.NetworkInterfaceId)
- if _, ok := ts.cfg.EKSConfig.Status.DeletedResources[eniID]; ok {
- continue
- }
-
- ts.cfg.Logger.Warn("detaching ENI", zap.String("eni", eniID))
- out, err := ts.cfg.EC2APIV2.DescribeNetworkInterfaces(
- context.Background(),
- &aws_ec2_v2.DescribeNetworkInterfacesInput{
- NetworkInterfaceIds: []string{eniID},
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to describe ENI", zap.Error(err))
- continue
- }
- if len(out.NetworkInterfaces) != 1 {
- ts.cfg.Logger.Warn("expected 1 ENI", zap.String("eni", eniID), zap.Int("enis", len(out.NetworkInterfaces)))
- continue
- }
- if out.NetworkInterfaces[0].Attachment == nil {
- ts.cfg.Logger.Warn("no attachment found for ENI", zap.String("eni", eniID))
- } else {
- for i := 0; i < 5; i++ {
- time.Sleep(5 * time.Second)
- _, err = ts.cfg.EC2APIV2.DetachNetworkInterface(
- context.Background(),
- &aws_ec2_v2.DetachNetworkInterfaceInput{
- AttachmentId: out.NetworkInterfaces[0].Attachment.AttachmentId,
- Force: aws_v2.Bool(true),
- })
- if err == nil {
- ts.cfg.Logger.Info("successfully detached ENI", zap.String("eni", eniID))
- break
- }
- ts.cfg.Logger.Warn("failed to detach ENI", zap.String("eni", eniID), zap.Error(err))
- }
- }
-
- for i := 0; i < 5; i++ {
- // may take awhile for delete to success upon detach
- time.Sleep(10 * time.Second)
- ts.cfg.Logger.Info("deleting ENI", zap.String("eni", eniID))
- _, err = ts.cfg.EC2APIV2.DeleteNetworkInterface(
- context.Background(),
- &aws_ec2_v2.DeleteNetworkInterfaceInput{
- NetworkInterfaceId: aws_v2.String(eniID),
- })
- if err == nil {
- ts.cfg.Logger.Info("successfully deleted ENI", zap.String("eni", eniID))
- deleted = true
- break
- }
- ts.cfg.Logger.Warn("failed to delete ENI", zap.String("eni", eniID), zap.Error(err))
- }
-
- // confirm ENI deletion
- retryStart := time.Now()
- for time.Since(retryStart) < 5*time.Minute {
- time.Sleep(5 * time.Second)
- _, err = ts.cfg.EC2APIV2.DescribeNetworkInterfaces(
- context.Background(),
- &aws_ec2_v2.DescribeNetworkInterfacesInput{
- NetworkInterfaceIds: []string{eniID},
- })
- if err == nil {
- _, derr := ts.cfg.EC2APIV2.DeleteNetworkInterface(
- context.Background(),
- &aws_ec2_v2.DeleteNetworkInterfaceInput{
- NetworkInterfaceId: aws_v2.String(eniID),
- })
- ts.cfg.Logger.Warn("ENI still exists", zap.String("eni", eniID), zap.Error(derr))
- continue
- }
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- ts.cfg.EKSConfig.Status.DeletedResources[eniID] = "AddOnNodeGroups.ENI"
- ts.cfg.EKSConfig.Sync()
- deleted = true
- break
- }
- }
-
- _, derr := ts.cfg.EC2APIV2.DeleteNetworkInterface(
- context.Background(),
- &aws_ec2_v2.DeleteNetworkInterfaceInput{
- NetworkInterfaceId: aws_v2.String(eniID),
- })
- ts.cfg.Logger.Warn("ENI still exists", zap.String("eni", eniID), zap.String("errors", fmt.Sprintf("%v, %v", err, derr)))
- }
- }
- return deleted
-}
diff --git a/eks/ng/logs.go b/eks/ng/logs.go
deleted file mode 100644
index 10e8a33a7..000000000
--- a/eks/ng/logs.go
+++ /dev/null
@@ -1,507 +0,0 @@
-package ng
-
-import (
- "context"
- "fmt"
- "os"
- "path/filepath"
- "sort"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/pkg/fileutil"
- "github.com/aws/aws-k8s-tester/pkg/randutil"
- "github.com/aws/aws-k8s-tester/ssh"
- "github.com/dustin/go-humanize"
- "github.com/mholt/archiver/v3"
- "go.uber.org/zap"
- "golang.org/x/time/rate"
-)
-
-// TODO: fetch logs via SSM + S3
-
-var defaultLogs = map[string]string{
- // kernel logs
- "sudo journalctl --no-pager --output=short-precise -k": "kernel.out.log",
-
- // full journal logs (e.g. disk mounts)
- "sudo journalctl --no-pager --output=short-precise": "journal.out.log",
-
- // other systemd services
- "sudo systemctl list-units -t service --no-pager --no-legend --all": "list-units-systemctl.out.log",
-}
-
-// FetchLogs downloads logs from managed node group instances.
-func (ts *tester) FetchLogs() (err error) {
- if !ts.cfg.EKSConfig.IsEnabledAddOnNodeGroups() {
- ts.cfg.Logger.Info("skipping fetch logs for node groups")
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnNodeGroups.FetchLogs {
- ts.cfg.Logger.Info("skipping fetch logs for node groups")
- return nil
- }
-
- ts.logsMu.Lock()
- defer ts.logsMu.Unlock()
-
- err = os.MkdirAll(ts.cfg.EKSConfig.AddOnNodeGroups.LogsDir, 0700)
- if err != nil {
- ts.cfg.Logger.Warn("failed to mkdir", zap.Error(err))
- return err
- }
-
- err = ts.fetchLogs(250, 10)
- if err != nil {
- ts.cfg.Logger.Warn("failed to fetch logs", zap.Error(err))
- return err
- }
-
- ts.cfg.Logger.Info("gzipping logs dir", zap.String("logs-dir", ts.cfg.EKSConfig.AddOnNodeGroups.LogsDir), zap.String("file-path", ts.cfg.EKSConfig.AddOnNodeGroups.LogsTarGzPath))
- err = os.RemoveAll(ts.cfg.EKSConfig.AddOnNodeGroups.LogsTarGzPath)
- if err != nil {
- ts.cfg.Logger.Warn("failed to remove temp file", zap.Error(err))
- return err
- }
- err = archiver.Archive([]string{ts.cfg.EKSConfig.AddOnNodeGroups.LogsDir}, ts.cfg.EKSConfig.AddOnNodeGroups.LogsTarGzPath)
- if err != nil {
- ts.cfg.Logger.Warn("archive failed", zap.Error(err))
- return err
- }
- stat, err := os.Stat(ts.cfg.EKSConfig.AddOnNodeGroups.LogsTarGzPath)
- if err != nil {
- ts.cfg.Logger.Warn("failed to os stat", zap.Error(err))
- return err
- }
- sz := humanize.Bytes(uint64(stat.Size()))
- ts.cfg.Logger.Info("gzipped logs dir", zap.String("logs-dir", ts.cfg.EKSConfig.AddOnNodeGroups.LogsDir), zap.String("file-path", ts.cfg.EKSConfig.AddOnNodeGroups.LogsTarGzPath), zap.String("file-size", sz))
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) fetchLogs(qps float32, burst int) error {
- logsDir := ts.cfg.EKSConfig.AddOnNodeGroups.LogsDir
- sshOptLog := ssh.WithVerbose(ts.cfg.EKSConfig.LogLevel == "debug")
- rateLimiter := rate.NewLimiter(rate.Limit(qps), burst)
- rch, waits := make(chan instanceLogs, 10), 0
-
- for name, nodeGroup := range ts.cfg.EKSConfig.AddOnNodeGroups.ASGs {
- ts.cfg.Logger.Info("fetching logs from node group",
- zap.String("asg-name", name),
- zap.Int("nodes", len(nodeGroup.Instances)),
- )
- waits += len(nodeGroup.Instances)
-
- for instID, cur := range nodeGroup.Instances {
- pfx := instID + "-"
-
- go func(instID, logsDir, pfx string, cur ec2config.Instance) {
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("exiting fetch logger", zap.String("prefix", pfx))
- return
- default:
- }
-
- if !rateLimiter.Allow() {
- ts.cfg.Logger.Debug("waiting for rate limiter before SSH into the machine",
- zap.Float32("qps", qps),
- zap.Int("burst", burst),
- zap.String("instance-id", instID),
- )
- werr := rateLimiter.Wait(context.Background())
- ts.cfg.Logger.Debug("waited for rate limiter",
- zap.Float32("qps", qps),
- zap.Int("burst", burst),
- zap.Error(werr),
- )
- }
-
- sh, err := ssh.New(ssh.Config{
- Logger: ts.cfg.Logger,
- KeyPath: ts.cfg.EKSConfig.RemoteAccessPrivateKeyPath,
- PublicIP: cur.PublicIP,
- PublicDNSName: cur.PublicDNSName,
- UserName: cur.RemoteAccessUserName,
- })
- if err != nil {
- rch <- instanceLogs{asgName: name, errs: []string{err.Error()}}
- return
- }
- defer sh.Close()
- if err = sh.Connect(); err != nil {
- rch <- instanceLogs{asgName: name, errs: []string{err.Error()}}
- return
- }
-
- data := instanceLogs{asgName: name, instanceID: instID}
- // fetch default logs
- for cmd, fileName := range defaultLogs {
- if !rateLimiter.Allow() {
- ts.cfg.Logger.Debug("waiting for rate limiter before fetching file")
- werr := rateLimiter.Wait(context.Background())
- ts.cfg.Logger.Debug("waited for rate limiter", zap.Error(werr))
- }
- out, oerr := sh.Run(cmd, sshOptLog)
- if oerr != nil {
- data.errs = append(data.errs, fmt.Sprintf("failed to run command %q for %q (error %v)", cmd, instID, oerr))
- continue
- }
-
- fpath := filepath.Join(logsDir, shorten(ts.cfg.Logger, pfx+fileName))
- f, err := os.Create(fpath)
- if err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to create a file %q for %q (error %v)",
- fpath,
- instID,
- err,
- ))
- continue
- }
- if _, err = f.Write(out); err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to write to a file %q for %q (error %v)",
- fpath,
- instID,
- err,
- ))
- f.Close()
- continue
- }
- f.Close()
- ts.cfg.Logger.Debug("wrote", zap.String("file-path", fpath))
- data.paths = append(data.paths, fpath)
- }
-
- if !rateLimiter.Allow() {
- ts.cfg.Logger.Debug("waiting for rate limiter before fetching file")
- werr := rateLimiter.Wait(context.Background())
- ts.cfg.Logger.Debug("waited for rate limiter", zap.Error(werr))
- }
- ts.cfg.Logger.Info("listing systemd service units", zap.String("instance-id", instID))
- listCmd := "sudo systemctl list-units -t service --no-pager --no-legend --all"
- out, oerr := sh.Run(listCmd, sshOptLog)
- if oerr != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to run command %q for %q (error %v)",
- listCmd,
- instID,
- oerr,
- ))
- } else {
- /*
- auditd.service loaded active running Security Auditing Service
- auth-rpcgss-module.service loaded inactive dead Kernel Module supporting RPCSEC_GSS
- */
- svcCmdToFileName := make(map[string]string)
- for _, line := range strings.Split(string(out), "\n") {
- fields := strings.Fields(line)
- if len(fields) == 0 || fields[0] == "" || len(fields) < 5 {
- continue
- }
- if fields[1] == "not-found" {
- continue
- }
- if fields[2] == "inactive" {
- continue
- }
- svc := fields[0]
- svcCmd := "sudo journalctl --no-pager --output=cat -u " + svc
- svcFileName := svc + ".out.log"
- svcCmdToFileName[svcCmd] = svcFileName
- }
- for cmd, fileName := range svcCmdToFileName {
- if !rateLimiter.Allow() {
- ts.cfg.Logger.Debug("waiting for rate limiter before fetching file")
- werr := rateLimiter.Wait(context.Background())
- ts.cfg.Logger.Debug("waited for rate limiter", zap.Error(werr))
- }
- out, oerr := sh.Run(cmd, sshOptLog)
- if oerr != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to run command %q for %q (error %v)",
- listCmd,
- instID,
- oerr,
- ))
- continue
- }
-
- fpath := filepath.Join(logsDir, shorten(ts.cfg.Logger, pfx+fileName))
- f, err := os.Create(fpath)
- if err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to create a file %q for %q (error %v)",
- fpath,
- instID,
- err,
- ))
- continue
- }
- if _, err = f.Write(out); err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to write to a file %q for %q (error %v)",
- fpath,
- instID,
- err,
- ))
- f.Close()
- continue
- }
- f.Close()
- ts.cfg.Logger.Debug("wrote", zap.String("file-path", fpath))
- data.paths = append(data.paths, fpath)
- }
- }
-
- if !rateLimiter.Allow() {
- ts.cfg.Logger.Debug("waiting for rate limiter before fetching file")
- werr := rateLimiter.Wait(context.Background())
- ts.cfg.Logger.Debug("waited for rate limiter", zap.Error(werr))
- }
- // https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/troubleshooting.md#ipamd-debugging-commands
- // https://github.com/aws/amazon-vpc-cni-k8s/blob/master/scripts/aws-cni-support.sh
- ts.cfg.Logger.Info("fetching ENI information", zap.String("instance-id", instID))
- eniCmd := "curl -s http://localhost:61679/v1/enis"
- out, oerr = sh.Run(eniCmd, sshOptLog)
- if oerr != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to run command %q for %q (error %v)",
- eniCmd,
- instID,
- oerr,
- ))
- } else {
- v1ENIOutputPath := filepath.Join(logsDir, shorten(ts.cfg.Logger, pfx+"v1-enis.out.log"))
- f, err := os.Create(v1ENIOutputPath)
- if err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to create a file %q for %q (error %v)",
- v1ENIOutputPath,
- instID,
- err,
- ))
- } else {
- if _, err = f.Write(out); err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to write to a file %q for %q (error %v)",
- v1ENIOutputPath,
- instID,
- err,
- ))
- } else {
- ts.cfg.Logger.Debug("wrote", zap.String("file-path", v1ENIOutputPath))
- data.paths = append(data.paths, v1ENIOutputPath)
- }
- f.Close()
- }
- }
-
- ts.cfg.Logger.Info("running /opt/cni/bin/aws-cni-support.sh", zap.String("instance-id", instID))
- cniCmd := "sudo /opt/cni/bin/aws-cni-support.sh || true"
- out, oerr = sh.Run(cniCmd, sshOptLog)
- if oerr != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to run command %q for %q (error %v)",
- cniCmd,
- instID,
- oerr,
- ))
- } else {
- ts.cfg.Logger.Info("ran /opt/cni/bin/aws-cni-support.sh", zap.String("instance-id", instID), zap.String("output", string(out)))
- }
-
- if !rateLimiter.Allow() {
- ts.cfg.Logger.Debug("waiting for rate limiter before fetching file")
- werr := rateLimiter.Wait(context.Background())
- ts.cfg.Logger.Debug("waited for rate limiter", zap.Error(werr))
- }
- ts.cfg.Logger.Info("listing /var/log", zap.String("instance-id", instID))
- findCmd := "sudo find /var/log ! -type d"
- out, oerr = sh.Run(findCmd, sshOptLog, ssh.WithRetry(5, 3*time.Second))
- if oerr != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to run command %q for %q (error %v)",
- findCmd,
- instID,
- oerr,
- ))
- } else {
- varLogPaths := make(map[string]string)
- for _, line := range strings.Split(string(out), "\n") {
- if len(line) == 0 {
- // last value
- continue
- }
- logCmd := "sudo cat " + line
- logPath := filepath.Base(line)
- varLogPaths[logCmd] = logPath
- }
- for cmd, logPath := range varLogPaths {
- if !rateLimiter.Allow() {
- ts.cfg.Logger.Debug("waiting for rate limiter before fetching file")
- werr := rateLimiter.Wait(context.Background())
- ts.cfg.Logger.Debug("waited for rate limiter", zap.Error(werr))
- }
- // e.g. "read tcp 10.119.223.210:58688->54.184.39.156:22: read: connection timed out"
- out, oerr := sh.Run(cmd, sshOptLog, ssh.WithRetry(2, 3*time.Second))
- if oerr != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to run command %q for %q (error %v)",
- cmd,
- instID,
- oerr,
- ))
- continue
- }
-
- fpath := filepath.Join(logsDir, shorten(ts.cfg.Logger, pfx+logPath))
- f, err := os.Create(fpath)
- if err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to create a file %q for %q (error %v)",
- fpath,
- instID,
- err,
- ))
- continue
- }
- if _, err = f.Write(out); err != nil {
- data.errs = append(data.errs, fmt.Sprintf(
- "failed to write to a file %q for %q (error %v)",
- fpath,
- instID,
- err,
- ))
- f.Close()
- continue
- }
- f.Close()
- ts.cfg.Logger.Debug("wrote", zap.String("file-path", fpath))
- data.paths = append(data.paths, fpath)
- }
- }
- rch <- data
- }(instID, logsDir, pfx, cur)
- }
- }
-
- ts.cfg.Logger.Info("waiting for log fetcher goroutines", zap.Int("waits", waits))
- total := 0
- for i := 0; i < waits; i++ {
- var data instanceLogs
- select {
- case data = <-rch:
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("exiting fetch logger")
- ts.cfg.EKSConfig.Sync()
- return nil
- }
- if len(data.errs) > 0 {
- ts.cfg.Logger.Warn("failed to fetch logs, but keeping whatever available",
- zap.String("mng-name", data.asgName),
- zap.String("instance-id", data.instanceID),
- zap.Strings("errors", data.errs),
- )
- }
- cur, ok := ts.cfg.EKSConfig.AddOnNodeGroups.ASGs[data.asgName]
- if !ok {
- return fmt.Errorf("EKS Node Group name %q is unknown", data.asgName)
- }
- if cur.Logs == nil {
- cur.Logs = make(map[string][]string)
- }
-
- // existing logs are already written out to disk, merge/list them all
- var logs []string
- logs, ok = cur.Logs[data.instanceID]
- if ok {
- ts.cfg.Logger.Warn("node group already has existing logs; merging",
- zap.String("asg-name", data.asgName),
- zap.String("instance-id", data.instanceID),
- )
- }
- all := make(map[string]struct{})
- for _, v := range logs {
- all[v] = struct{}{}
- }
- for _, v := range data.paths {
- all[v] = struct{}{}
- }
- logs = make([]string, 0, len(all))
- for k := range all {
- logs = append(logs, k)
- }
- sort.Strings(logs)
- cur.Logs[data.instanceID] = logs
- files := len(logs)
-
- ts.cfg.EKSConfig.AddOnNodeGroups.ASGs[data.asgName] = cur
- ts.cfg.EKSConfig.Sync()
-
- total += files
- ts.cfg.Logger.Info("wrote log files",
- zap.String("instance-id", data.instanceID),
- zap.Int("files", files),
- zap.Int("total-downloaded-files", total),
- zap.Int("total-goroutines-to-wait", waits),
- zap.Int("current-waited-goroutines", i),
- )
- }
-
- ts.cfg.Logger.Info("wrote all log files",
- zap.String("log-dir", logsDir),
- zap.Int("total-downloaded-files", total),
- )
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-type instanceLogs struct {
- asgName string
- instanceID string
- paths []string
- errs []string
-}
-
-func (ts *tester) DownloadClusterLogs(artifactDir string) error {
- err := ts.FetchLogs()
- if err != nil {
- return err
- }
-
- ts.logsMu.RLock()
- defer ts.logsMu.RUnlock()
-
- for _, v := range ts.cfg.EKSConfig.AddOnNodeGroups.ASGs {
- for _, fpaths := range v.Logs {
- for _, fpath := range fpaths {
- newPath := filepath.Join(artifactDir, filepath.Base(fpath))
- if err := fileutil.Copy(fpath, newPath); err != nil {
- return err
- }
- }
- }
- }
-
- return fileutil.Copy(
- ts.cfg.EKSConfig.ConfigPath,
- filepath.Join(artifactDir, filepath.Base(ts.cfg.EKSConfig.ConfigPath)),
- )
-}
-
-func shorten(lg *zap.Logger, name string) string {
- if len(name) < 240 {
- return name
- }
-
- ext := filepath.Ext(name)
- oldName := name
-
- name = name[:230] + randutil.String(5) + ext
- lg.Info("file name too long; renamed", zap.String("old", oldName), zap.String("new", name))
- return name
-}
diff --git a/eks/ng/ng.go b/eks/ng/ng.go
deleted file mode 100644
index 4853b389a..000000000
--- a/eks/ng/ng.go
+++ /dev/null
@@ -1,223 +0,0 @@
-// Package ng implements EKS worker nodes with a custom AMI.
-package ng
-
-import (
- "errors"
- "io"
- "reflect"
- "strings"
- "sync"
- "time"
-
- "github.com/aws/aws-k8s-tester/eks/ng/autoscaler"
- "github.com/aws/aws-k8s-tester/eks/ng/wait"
- "github.com/aws/aws-k8s-tester/eksconfig"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- aws_asg_v2 "github.com/aws/aws-sdk-go-v2/service/autoscaling"
- aws_ec2_v2 "github.com/aws/aws-sdk-go-v2/service/ec2"
- aws_iam_v2 "github.com/aws/aws-sdk-go-v2/service/iam"
- aws_ssm_v2 "github.com/aws/aws-sdk-go-v2/service/ssm"
- "go.uber.org/zap"
-)
-
-// Config defines Node Group configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
-
- IAMAPIV2 *aws_iam_v2.Client
- EC2APIV2 *aws_ec2_v2.Client
- SSMAPIV2 *aws_ssm_v2.Client
- ASGAPIV2 *aws_asg_v2.Client
-}
-
-// Tester implements EKS "Node Group" for "kubetest2" Deployer.
-// ref. https://github.com/kubernetes/test-infra/blob/master/kubetest2/pkg/types/types.go
-// ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
-// ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html
-type Tester interface {
- // Name returns the name of the tester.
- Name() string
- // Create creates EKS "Node Group", and waits for completion.
- Create() error
- // Delete deletes all EKS "Node Group" resources.
- Delete() error
-
- // FetchLogs fetches logs from all worker nodes.
- FetchLogs() error
- // DownloadClusterLogs dumps all logs to artifact directory.
- // Let default kubetest log dumper handle all artifact uploads.
- // See https://github.com/kubernetes/test-infra/pull/9811/files#r225776067.
- DownloadClusterLogs(artifactDir string) error
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-// New creates a new Job tester.
-func New(cfg Config) Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{
- cfg: cfg,
- nodeWaiter: wait.New(wait.Config{
- Logger: cfg.Logger,
- LogWriter: cfg.LogWriter,
- Stopc: cfg.Stopc,
- EKSConfig: cfg.EKSConfig,
- K8SClient: cfg.K8SClient,
- EC2APIV2: cfg.EC2APIV2,
- ASGAPIV2: cfg.ASGAPIV2,
- }),
- logsMu: new(sync.RWMutex),
- failedOnce: false,
- clusterAutoscaler: autoscaler.New(autoscaler.Config{
- Logger: cfg.Logger,
- LogWriter: cfg.LogWriter,
- Stopc: cfg.Stopc,
- EKSConfig: cfg.EKSConfig,
- K8SClient: cfg.K8SClient,
- }),
- }
-}
-
-type tester struct {
- cfg Config
- nodeWaiter wait.NodeWaiter
- logsMu *sync.RWMutex
- failedOnce bool
- clusterAutoscaler autoscaler.ClusterAutoscaler
-}
-
-func (ts *tester) Create() (err error) {
- if !ts.cfg.EKSConfig.IsEnabledAddOnNodeGroups() {
- ts.cfg.Logger.Info("node group is disabled; skipping creation")
- return nil
- }
- if ts.cfg.EKSConfig.AddOnNodeGroups.Created {
- ts.cfg.Logger.Info("node group is already created; skipping creation")
- return nil
- }
- if len(ts.cfg.EKSConfig.VPC.PublicSubnetIDs) == 0 {
- return errors.New("empty EKSConfig.VPC.PublicSubnetIDs")
- }
-
- ts.cfg.Logger.Info("starting tester.Create", zap.String("tester", pkgName))
- createStart := time.Now()
- defer func() {
- createEnd := time.Now()
- ts.cfg.EKSConfig.AddOnNodeGroups.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- if err = ts.createSecurityGroups(); err != nil {
- return err
- }
- if err = ts.authorizeSecurityGroup(); err != nil {
- return err
- }
- if err = ts.createRole(); err != nil {
- return err
- }
- if err = ts.createConfigMap(); err != nil {
- return err
- }
- if err = ts.createASGs(); err != nil {
- return err
- }
- if err = ts.createSSM(); err != nil {
- return err
- }
- if err = ts.clusterAutoscaler.Create(); err != nil {
- return err
- }
-
- ts.cfg.EKSConfig.AddOnNodeGroups.Created = true
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) Delete() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnNodeGroups() {
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnNodeGroups.Created {
- ts.cfg.Logger.Info("ManagedNodeGroup is not created; skipping deletion")
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Delete", zap.String("tester", pkgName))
- deleteStart := time.Now()
- defer func() {
- deleteEnd := time.Now()
- ts.cfg.EKSConfig.AddOnNodeGroups.TimeFrameDelete = timeutil.NewTimeFrame(deleteStart, deleteEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- var errs []string
- if err := ts.deleteSSM(); err != nil {
- ts.cfg.Logger.Warn("failed to delete SSM", zap.Error(err))
- errs = append(errs, err.Error())
- }
-
- if err := ts.deleteRole(); err != nil {
- ts.cfg.Logger.Warn("failed to delete role", zap.Error(err))
- errs = append(errs, err.Error())
- }
-
- select {
- case <-time.After(10 * time.Second):
- case <-ts.cfg.Stopc:
- return errors.New("stopped")
- }
-
- if err := ts.deleteASGs(); err != nil {
- ts.cfg.Logger.Warn("failed to delete ASGs", zap.Error(err))
- errs = append(errs, err.Error())
- }
-
- if ok := ts.deleteENIs(); ok {
- time.Sleep(10 * time.Second)
- }
-
- select {
- case <-time.After(10 * time.Second):
- case <-ts.cfg.Stopc:
- return errors.New("stopped")
- }
-
- if ok := ts.deleteENIs(); ok {
- time.Sleep(10 * time.Second)
- }
-
- select {
- case <-time.After(10 * time.Second):
- case <-ts.cfg.Stopc:
- return errors.New("stopped")
- }
-
- if err := ts.revokeSecurityGroups(); err != nil {
- ts.cfg.Logger.Warn("failed to revoke SG", zap.Error(err))
- errs = append(errs, err.Error())
- }
- if err := ts.deleteSecurityGroups(); err != nil {
- ts.cfg.Logger.Warn("failed to delete SG", zap.Error(err))
- errs = append(errs, err.Error())
- }
-
- if ok := ts.deleteENIs(); ok {
- time.Sleep(10 * time.Second)
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- ts.cfg.EKSConfig.AddOnNodeGroups.Created = false
- ts.cfg.EKSConfig.Sync()
- return nil
-}
diff --git a/eks/ng/role.go b/eks/ng/role.go
deleted file mode 100644
index 9397087ca..000000000
--- a/eks/ng/role.go
+++ /dev/null
@@ -1,626 +0,0 @@
-package ng
-
-import (
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "strings"
- "time"
-
- aws_iam "github.com/aws/aws-k8s-tester/pkg/aws/iam"
- aws_v2 "github.com/aws/aws-sdk-go-v2/aws"
- aws_iam_v2 "github.com/aws/aws-sdk-go-v2/service/iam"
- smithy "github.com/aws/smithy-go"
- "go.uber.org/zap"
-)
-
-// see https://github.com/aws/aws-k8s-tester/blob/v1.6.0/eks/ng/role.go for CloudFormation based workflow
-
-func (ts *tester) createRole() error {
- if !ts.cfg.EKSConfig.AddOnNodeGroups.Role.Create {
- ts.cfg.Logger.Info("AddOnNodeGroups.Role.Create false; skipping creation")
- return aws_iam.ValidateV2(
- ts.cfg.Logger,
- ts.cfg.IAMAPIV2,
- ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name,
- []string{"ec2.amazonaws.com"},
- []string{
- "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
- },
- )
- }
- if ts.cfg.EKSConfig.AddOnNodeGroups.Role.ARN != "" {
- ts.cfg.Logger.Info("role already created; no need to create a new one")
- return nil
- }
- if ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name == "" {
- return errors.New("cannot create a cluster role with an empty AddOnNodeGroups.Role.Name")
- }
-
- if err := ts._createRole(); err != nil {
- return err
- }
- if err := ts.createPolicy(); err != nil {
- return err
- }
- if err := ts.attachPolicy(); err != nil {
- return err
- }
- if err := ts.createInstanceProfile(); err != nil {
- return err
- }
-
- ts.cfg.Logger.Info("created a new role and attached policy",
- zap.String("role-arn", ts.cfg.EKSConfig.AddOnNodeGroups.Role.ARN),
- zap.String("role-name", ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name),
- )
- return nil
-}
-
-func (ts *tester) deleteRole() error {
- if !ts.cfg.EKSConfig.AddOnNodeGroups.Role.Create {
- ts.cfg.Logger.Info("Role.Create false; skipping deletion")
- return nil
- }
-
- var errs []string
-
- // "Cannot delete entity, must remove roles from instance profile first"
- if err := ts.deleteInstanceProfile(); err != nil {
- ts.cfg.Logger.Warn("failed to delete instance profile", zap.Error(err))
- }
- if err := ts.detachPolicy(); err != nil {
- ts.cfg.Logger.Warn("failed to detach policy", zap.Error(err))
- errs = append(errs, err.Error())
- }
- if err := ts.deletePolicy(); err != nil {
- ts.cfg.Logger.Warn("failed to delete policy", zap.Error(err))
- errs = append(errs, err.Error())
- }
- if err := ts._deleteRole(); err != nil {
- ts.cfg.Logger.Warn("failed to delete role", zap.Error(err))
- errs = append(errs, err.Error())
- }
- if err := ts.deleteInstanceProfile(); err != nil { // retry
- ts.cfg.Logger.Warn("failed to delete instance profile", zap.Error(err))
- errs = append(errs, err.Error())
- }
-
- if len(errs) == 0 {
- ts.cfg.Logger.Info("successfully deleted role",
- zap.String("role-arn", ts.cfg.EKSConfig.AddOnNodeGroups.Role.ARN),
- zap.String("role-name", ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name),
- )
- return nil
- }
- return errors.New(strings.Join(errs, ","))
-}
-
-func (ts *tester) _createRole() error {
- ts.cfg.Logger.Info("creating role", zap.String("name", ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name))
- out, err := ts.cfg.IAMAPIV2.CreateRole(
- context.Background(),
- &aws_iam_v2.CreateRoleInput{
- RoleName: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name),
- Path: aws_v2.String("/"),
- AssumeRolePolicyDocument: aws_v2.String(createAssumeRolePolicyDocument(ts.cfg.EKSConfig.AddOnNodeGroups.Role.ServicePrincipals)),
- },
- )
- if err != nil {
- return err
- }
-
- ts.cfg.Logger.Info("created role")
- ts.cfg.EKSConfig.AddOnNodeGroups.Role.ARN = aws_v2.ToString(out.Role.Arn)
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) _deleteRole() error {
- ts.cfg.Logger.Info("deleting role", zap.String("name", ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name))
- if _, ok := ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name]; ok {
- return nil
- }
-
- _, err := ts.cfg.IAMAPIV2.DeleteRole(
- context.Background(),
- &aws_iam_v2.DeleteRoleInput{
- RoleName: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name),
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to delete cluster role", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name] = "AddOnNodeGroups.Role.Name"
- ts.cfg.EKSConfig.Sync()
- return nil
- }
- }
- return err
- }
-
- ts.cfg.Logger.Info("deleted role")
- ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name] = "AddOnNodeGroups.Role.Name"
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html
-// https://github.com/kubernetes-sigs/aws-alb-ingress-controller/blob/master/docs/examples/iam-policy.json
-// https://github.com/aws/eks-charts/tree/master/stable/appmesh-controller
-func (ts *tester) createPolicy() error {
- if ts.cfg.EKSConfig.AddOnNodeGroups.Role.PolicyName == "" {
- return errors.New("emtpy PolicyName")
- }
- ts.cfg.Logger.Info("creating policy", zap.String("name", ts.cfg.EKSConfig.AddOnNodeGroups.Role.PolicyName))
- pout, err := ts.cfg.IAMAPIV2.CreatePolicy(
- context.Background(),
- &aws_iam_v2.CreatePolicyInput{
- PolicyName: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.PolicyName),
- PolicyDocument: aws_v2.String(createRolePolicyDocument(ts.cfg.EKSConfig.Partition, ts.cfg.EKSConfig.S3.BucketName)),
- },
- )
- if err != nil {
- return err
- }
-
- ts.cfg.Logger.Info("created policy")
- ts.cfg.EKSConfig.AddOnNodeGroups.Role.PolicyARN = aws_v2.ToString(pout.Policy.Arn)
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) deletePolicy() error {
- ts.cfg.Logger.Info("deleting policy")
- if _, ok := ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnNodeGroups.Role.PolicyARN]; ok {
- return nil
- }
-
- _, err := ts.cfg.IAMAPIV2.DeletePolicy(
- context.Background(),
- &aws_iam_v2.DeletePolicyInput{
- PolicyArn: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.PolicyARN),
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to delete policy", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnNodeGroups.Role.PolicyARN] = "AddOnNodeGroups.Role.PolicyARN"
- ts.cfg.EKSConfig.Sync()
- return nil
- }
- }
- return err
- }
-
- ts.cfg.Logger.Info("deleted policy")
- ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnNodeGroups.Role.PolicyARN] = "AddOnNodeGroups.Role.PolicyARN"
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) attachPolicy() error {
- ts.cfg.Logger.Info("attaching policies")
-
- _, err := ts.cfg.IAMAPIV2.AttachRolePolicy(
- context.Background(),
- &aws_iam_v2.AttachRolePolicyInput{
- RoleName: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name),
- PolicyArn: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.PolicyARN),
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to attach policy", zap.String("arn", ts.cfg.EKSConfig.AddOnNodeGroups.Role.PolicyARN), zap.Error(err))
- return err
- }
- ts.cfg.Logger.Info("attached policy arn", zap.String("policy-arn", ts.cfg.EKSConfig.AddOnNodeGroups.Role.PolicyARN))
-
- for _, arn := range ts.cfg.EKSConfig.AddOnNodeGroups.Role.ManagedPolicyARNs {
- time.Sleep(3 * time.Second)
- ts.cfg.Logger.Info("attaching managed policy arn", zap.String("arn", arn))
- _, err := ts.cfg.IAMAPIV2.AttachRolePolicy(
- context.Background(),
- &aws_iam_v2.AttachRolePolicyInput{
- RoleName: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name),
- PolicyArn: aws_v2.String(arn),
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to attach policy", zap.String("arn", arn), zap.Error(err))
- return err
- }
- }
-
- ts.cfg.Logger.Info("attached policies")
- return nil
-}
-
-func (ts *tester) detachPolicy() error {
- ts.cfg.Logger.Info("detaching policies")
-
- _, err := ts.cfg.IAMAPIV2.DetachRolePolicy(
- context.Background(),
- &aws_iam_v2.DetachRolePolicyInput{
- RoleName: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name),
- PolicyArn: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.PolicyARN),
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to detach policy", zap.String("arn", ts.cfg.EKSConfig.AddOnNodeGroups.Role.PolicyARN), zap.Error(err))
- return err
- }
- for _, arn := range ts.cfg.EKSConfig.AddOnNodeGroups.Role.ManagedPolicyARNs {
- time.Sleep(3 * time.Second)
- _, err := ts.cfg.IAMAPIV2.DetachRolePolicy(
- context.Background(),
- &aws_iam_v2.DetachRolePolicyInput{
- RoleName: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name),
- PolicyArn: aws_v2.String(arn),
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to detach policy", zap.String("arn", arn), zap.Error(err))
- return err
- }
- }
-
- ts.cfg.Logger.Info("detached policies")
- return nil
-}
-
-func createAssumeRolePolicyDocument(sps []string) string {
- p := aws_iam.PolicyDocument{
- Version: "2012-10-17",
- Statement: createStatementEntriesForAssumeRole(sps),
- }
- b, err := json.Marshal(p)
- if err != nil {
- panic(err)
- }
- return string(b)
-}
-
-func createRolePolicyDocument(partition string, bucketName string) string {
- p := aws_iam.PolicyDocument{
- Version: "2012-10-17",
- Statement: createStatementEntriesForRolePolicyDocument(partition, bucketName),
- }
- b, err := json.Marshal(p)
- if err != nil {
- panic(err)
- }
- return string(b)
-}
-
-func createStatementEntriesForAssumeRole(sps []string) []aws_iam.StatementEntry {
- return []aws_iam.StatementEntry{
- {
- Effect: "Allow",
- Principal: &aws_iam.PrincipalEntry{
- Service: sps,
- },
- Action: []string{
- "sts:AssumeRole",
- },
- },
- }
-}
-
-// TODO: update based on add-on setups
-// https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html
-// https://github.com/kubernetes-sigs/aws-alb-ingress-controller/blob/master/docs/examples/iam-policy.json
-// https://github.com/aws/eks-charts/tree/master/stable/appmesh-controller
-func createStatementEntriesForRolePolicyDocument(partition string, bucketName string) []aws_iam.StatementEntry {
- return []aws_iam.StatementEntry{
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "acm:DescribeCertificate",
- "acm:ListCertificates",
- "acm:GetCertificate",
- },
- },
- // arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "ec2:AttachVolume",
- "ec2:AuthorizeSecurityGroupIngress",
- "ec2:CreateSecurityGroup",
- "ec2:CreateSnapshot",
- "ec2:CreateTags",
- "ec2:CreateVolume",
- "ec2:DeleteSecurityGroup",
- "ec2:DeleteSnapshot",
- "ec2:DeleteTags",
- "ec2:DeleteVolume",
- "ec2:DescribeAccountAttributes",
- "ec2:DescribeAddresses",
- "ec2:DescribeInstanceStatus",
- "ec2:DescribeInstances",
- "ec2:DescribeInternetGateways",
- "ec2:DescribeNetworkInterfaces",
- "ec2:DescribeRouteTables",
- "ec2:DescribeSecurityGroups",
- "ec2:DescribeSnapshots",
- "ec2:DescribeSubnets",
- "ec2:DescribeTags",
- "ec2:DescribeVolumes",
- "ec2:DescribeVolumes",
- "ec2:DescribeVolumesModifications",
- "ec2:DescribeVpcs",
- "ec2:DetachVolume",
- "ec2:ModifyInstanceAttribute",
- "ec2:ModifyNetworkInterfaceAttribute",
- "ec2:RevokeSecurityGroupIngress",
- "eks:DescribeCluster",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "elasticloadbalancing:AddListenerCertificates",
- "elasticloadbalancing:AddTags",
- "elasticloadbalancing:CreateListener",
- "elasticloadbalancing:CreateLoadBalancer",
- "elasticloadbalancing:CreateRule",
- "elasticloadbalancing:CreateTargetGroup",
- "elasticloadbalancing:DeleteListener",
- "elasticloadbalancing:DeleteLoadBalancer",
- "elasticloadbalancing:DeleteRule",
- "elasticloadbalancing:DeleteTargetGroup",
- "elasticloadbalancing:DeregisterTargets",
- "elasticloadbalancing:DescribeListenerCertificates",
- "elasticloadbalancing:DescribeListeners",
- "elasticloadbalancing:DescribeLoadBalancers",
- "elasticloadbalancing:DescribeLoadBalancerAttributes",
- "elasticloadbalancing:DescribeRules",
- "elasticloadbalancing:DescribeSSLPolicies",
- "elasticloadbalancing:DescribeTags",
- "elasticloadbalancing:DescribeTargetGroups",
- "elasticloadbalancing:DescribeTargetGroupAttributes",
- "elasticloadbalancing:DescribeTargetHealth",
- "elasticloadbalancing:ModifyListener",
- "elasticloadbalancing:ModifyLoadBalancerAttributes",
- "elasticloadbalancing:ModifyRule",
- "elasticloadbalancing:ModifyTargetGroup",
- "elasticloadbalancing:ModifyTargetGroupAttributes",
- "elasticloadbalancing:RegisterTargets",
- "elasticloadbalancing:RemoveListenerCertificates",
- "elasticloadbalancing:RemoveTags",
- "elasticloadbalancing:SetIpAddressType",
- "elasticloadbalancing:SetSecurityGroups",
- "elasticloadbalancing:SetSubnets",
- "elasticloadbalancing:SetWebACL",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "iam:CreateServiceLinkedRole",
- "iam:GetServerCertificate",
- "iam:ListServerCertificates",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "cognito-idp:DescribeUserPoolClient",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "waf-regional:GetWebACLForResource",
- "waf-regional:GetWebACL",
- "waf-regional:AssociateWebACL",
- "waf-regional:DisassociateWebACL",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "tag:GetResources",
- "tag:TagResources",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "waf:GetWebACL",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "wafv2:GetWebACL",
- "wafv2:GetWebACLForResource",
- "wafv2:AssociateWebACL",
- "wafv2:DisassociateWebACL",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "shield:DescribeProtection",
- "shield:GetSubscriptionState",
- "shield:DeleteProtection",
- "shield:CreateProtection",
- "shield:DescribeSubscription",
- "shield:ListProtections",
- },
- },
- {
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "appmesh:*",
- "servicediscovery:CreateService",
- "servicediscovery:GetService",
- "servicediscovery:RegisterInstance",
- "servicediscovery:DeregisterInstance",
- "servicediscovery:ListInstances",
- "servicediscovery:ListNamespaces",
- "servicediscovery:ListServices",
- "route53:GetHealthCheck",
- "route53:CreateHealthCheck",
- "route53:UpdateHealthCheck",
- "route53:ChangeResourceRecordSets",
- "route53:DeleteHealthCheck",
- },
- },
- { // for fluentd add-on
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "logs:CreateLogGroup",
- "logs:CreateLogStream",
- "logs:DescribeLogGroups",
- "logs:DescribeLogStreams",
- "logs:PutLogEvents",
- },
- },
- { // for cluster autoscaler
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "autoscaling:DescribeAutoScalingGroups",
- "autoscaling:DescribeAutoScalingInstances",
- "autoscaling:DescribeLaunchConfigurations",
- "autoscaling:DescribeTags",
- "autoscaling:SetDesiredCapacity",
- "autoscaling:TerminateInstanceInAutoScalingGroup",
- "ec2:DescribeLaunchTemplateVersions",
- },
- },
- { // for artifact uploads from worker nodes
- Effect: "Allow",
- Resource: fmt.Sprintf("arn:%s:s3:::%s/*", partition, bucketName),
- Action: []string{
- "s3:ListBucket",
- "s3:GetObject",
- "s3:PutObject",
- },
- },
- { // arn:aws:iam::aws:policy/AmazonS3FullAccess
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "s3:*",
- },
- },
- { // arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
- Effect: "Allow",
- Resource: "*",
- Action: []string{
- "ecr:GetAuthorizationToken",
- "ecr:BatchCheckLayerAvailability",
- "ecr:GetDownloadUrlForLayer",
- "ecr:GetRepositoryPolicy",
- "ecr:DescribeRepositories",
- "ecr:ListImages",
- "ecr:DescribeImages",
- "ecr:BatchGetImage",
- "ecr:GetLifecyclePolicy",
- "ecr:GetLifecyclePolicyPreview",
- "ecr:ListTagsForResource",
- "ecr:DescribeImageScanFindings",
- },
- },
- }
-}
-
-func (ts *tester) createInstanceProfile() error {
- ts.cfg.Logger.Info("creating instance profile")
- out, err := ts.cfg.IAMAPIV2.CreateInstanceProfile(
- context.Background(),
- &aws_iam_v2.CreateInstanceProfileInput{
- InstanceProfileName: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.InstanceProfileName),
- Path: aws_v2.String("/"),
- },
- )
- if err != nil {
- return err
- }
- ts.cfg.EKSConfig.AddOnNodeGroups.Role.InstanceProfileARN = aws_v2.ToString(out.InstanceProfile.Arn)
- ts.cfg.EKSConfig.Sync()
-
- _, err = ts.cfg.IAMAPIV2.AddRoleToInstanceProfile(
- context.Background(),
- &aws_iam_v2.AddRoleToInstanceProfileInput{
- InstanceProfileName: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.InstanceProfileName),
- RoleName: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name),
- },
- )
- if err != nil {
- return err
- }
-
- ts.cfg.Logger.Info("created instance profile")
- return nil
-}
-
-func (ts *tester) deleteInstanceProfile() error {
- ts.cfg.Logger.Info("deleting instance profile")
- if _, ok := ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnNodeGroups.Role.InstanceProfileName]; ok {
- return nil
- }
-
- _, err := ts.cfg.IAMAPIV2.RemoveRoleFromInstanceProfile(
- context.Background(),
- &aws_iam_v2.RemoveRoleFromInstanceProfileInput{
- InstanceProfileName: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.InstanceProfileName),
- RoleName: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.Name),
- },
- )
- if err != nil {
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") || strings.Contains(apiErr.ErrorCode(), "NotFoundException") {
- ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnNodeGroups.Role.InstanceProfileName] = "AddOnNodeGroups.Role.InstanceProfileName"
- ts.cfg.EKSConfig.Sync()
- return nil
- }
- }
- return err
- }
-
- _, err = ts.cfg.IAMAPIV2.DeleteInstanceProfile(
- context.Background(),
- &aws_iam_v2.DeleteInstanceProfileInput{
- InstanceProfileName: aws_v2.String(ts.cfg.EKSConfig.AddOnNodeGroups.Role.InstanceProfileName),
- },
- )
- if err != nil {
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") || strings.Contains(apiErr.ErrorCode(), "NotFoundException") {
- ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnNodeGroups.Role.InstanceProfileName] = "AddOnNodeGroups.Role.InstanceProfileName"
- ts.cfg.EKSConfig.Sync()
- return nil
- }
- }
- return err
- }
-
- ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.AddOnNodeGroups.Role.InstanceProfileName] = "AddOnNodeGroups.Role.InstanceProfileName"
- ts.cfg.EKSConfig.Sync()
- ts.cfg.Logger.Info("deleted instance profile")
-
- return nil
-}
diff --git a/eks/ng/security-groups.go b/eks/ng/security-groups.go
deleted file mode 100644
index d2cfa2709..000000000
--- a/eks/ng/security-groups.go
+++ /dev/null
@@ -1,819 +0,0 @@
-package ng
-
-import (
- "context"
- "errors"
- "fmt"
- "strings"
-
- aws_v2 "github.com/aws/aws-sdk-go-v2/aws"
- aws_ec2_v2 "github.com/aws/aws-sdk-go-v2/service/ec2"
- aws_ec2_v2_types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
- smithy "github.com/aws/smithy-go"
- "go.uber.org/zap"
-)
-
-// see https://github.com/aws/aws-k8s-tester/blob/v1.6.0/eks/ng/security-groups.go for CloudFormation based workflow
-
-// "[sig-network] Networking Granular Checks" in "test/e2e/network/dns.go"
-// requires "e2enetwork.EndpointUDPPort/EndpointHTTPPort", 8081 and 8080
-// just open all for now...
-// TODO: restrict ports
-
-// AWS::EC2::SecurityGroup
-func (ts *tester) createSecurityGroups() error {
- ts.cfg.Logger.Info("creating security group")
-
- sout, err := ts.cfg.EC2APIV2.CreateSecurityGroup(
- context.Background(),
- &aws_ec2_v2.CreateSecurityGroupInput{
- GroupName: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupName),
- Description: aws_v2.String("Security group for all nodes in the cluster"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- TagSpecifications: []aws_ec2_v2_types.TagSpecification{
- {
- ResourceType: aws_ec2_v2_types.ResourceTypeSecurityGroup,
- Tags: []aws_ec2_v2_types.Tag{
- {
- Key: aws_v2.String(fmt.Sprintf("kubernetes.io/cluster/%s", ts.cfg.EKSConfig.Name)),
- Value: aws_v2.String("owned"),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to create security group", zap.Error(err))
- return err
- }
-
- ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID = aws_v2.ToString(sout.GroupId)
- ts.cfg.EKSConfig.Sync()
- ts.cfg.Logger.Info("created security group", zap.String("security-group-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
-
- return nil
-}
-
-func (ts *tester) deleteSecurityGroups() (err error) {
- ts.cfg.Logger.Info("deleting security group")
- if ts.cfg.EKSConfig.VPC.ID == "" {
- return nil
- }
- if _, ok := ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID]; ok {
- return nil
- }
-
- _, err = ts.cfg.EC2APIV2.DeleteSecurityGroup(
- context.Background(),
- &aws_ec2_v2.DeleteSecurityGroupInput{
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to delete security group", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID] = "VPC.NodeGroupSecurityGroupID"
- ts.cfg.EKSConfig.Sync()
- return nil
- }
- }
- return err
- }
-
- ts.cfg.Logger.Info("deleted security group")
- ts.cfg.EKSConfig.Status.DeletedResources[ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID] = "VPC.NodeGroupSecurityGroupID"
- ts.cfg.EKSConfig.Sync()
-
- return nil
-}
-
-func (ts *tester) authorizeSecurityGroup() error {
- ts.cfg.Logger.Info("authorizing security group",
- zap.String("api-server-node-security-group-id", ts.cfg.EKSConfig.VPC.SecurityGroupID),
- zap.String("node-group-security-group-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- )
-
- // allow node to communicate with each other
- ts.cfg.Logger.Info("authorizing IngressWithinNodeGroupSecurityGroup", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err := ts.cfg.EC2APIV2.AuthorizeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
-
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("-1"),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- Description: aws_v2.String("allow node to communicate with each other"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized IngressWithinNodeGroupSecurityGroup")
-
- // allow pods to communicate with the cluster API Server
- ts.cfg.Logger.Info("authorizing Ingress443FromNGtoCP", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
-
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(443),
- ToPort: aws_v2.Int32(443),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- Description: aws_v2.String("allow pods to communicate with the cluster API Server"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized Ingress443FromNGtoCP")
-
- // allow pods running extension API servers on port 443
- // to receive communication from cluster control plane
- ts.cfg.Logger.Info("authorizing Ingress443FromCPtoNG", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupIngressInput{
- // egress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(443),
- ToPort: aws_v2.Int32(443),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- Description: aws_v2.String("receive communication from cluster control plane"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized Ingress443FromCPtoNG")
-
- // allow the cluster control plane to communicate with pods running extension API servers on port 443
- ts.cfg.Logger.Info("authorizing Egress443FromCPtoNG", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupEgress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupEgressInput{
- // egress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(443),
- ToPort: aws_v2.Int32(443),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- Description: aws_v2.String("communicate with pods running extension API servers on port 443"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize egress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized Egress443FromCPtoNG")
-
- // allow worker Kubelets and pods to receive communication from the cluster control plane
- ts.cfg.Logger.Info("authorizing IngressAllFromCPtoNG", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(0),
- ToPort: aws_v2.Int32(65535),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- Description: aws_v2.String("receive communication from the cluster control plane"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized IngressAllFromCPtoNG")
-
- // allow the cluster control plane to communicate with worker Kubelet and pods
- ts.cfg.Logger.Info("authorizing EgressAllFromCPtoNG", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupEgress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupEgressInput{
- // egress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(0),
- ToPort: aws_v2.Int32(65535),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- Description: aws_v2.String("communicate with worker Kubelet and pods"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize egress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized EgressAllFromCPtoNG")
-
- ts.cfg.Logger.Info("authorizing Ingress22ForSSH", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- IpRanges: []aws_ec2_v2_types.IpRange{
- {
- CidrIp: aws_v2.String("0.0.0.0/0"),
- },
- },
- FromPort: aws_v2.Int32(22),
- ToPort: aws_v2.Int32(22),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized Ingress22ForSSH")
-
- ts.cfg.Logger.Info("authorizing IngressForGuestBook", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- IpRanges: []aws_ec2_v2_types.IpRange{
- {
- CidrIp: aws_v2.String("0.0.0.0/0"),
- },
- },
- FromPort: aws_v2.Int32(1),
- ToPort: aws_v2.Int32(10000),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized IngressForGuestBook")
-
- ts.cfg.Logger.Info("authorizing EgressForGuestBook", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupEgress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupEgressInput{
- // egress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(1),
- ToPort: aws_v2.Int32(10000),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize egress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized EgressForGuestBook")
-
- ts.cfg.Logger.Info("authorizing IngressForNodePortConformance", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.AuthorizeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.AuthorizeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- IpRanges: []aws_ec2_v2_types.IpRange{
- {
- CidrIp: aws_v2.String("0.0.0.0/0"),
- },
- },
- FromPort: aws_v2.Int32(1),
- ToPort: aws_v2.Int32(32767),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to authorize ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "Duplicate") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("authorized IngressForNodePortConformance")
-
- ts.cfg.Logger.Info("authorized security group")
- return nil
-}
-
-func (ts *tester) revokeSecurityGroups() (err error) {
- ts.cfg.Logger.Info("revoking security group")
- if ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID == "" {
- return nil
- }
-
- // allow node to communicate with each other
- ts.cfg.Logger.Info("revoking IngressWithinNodeGroupSecurityGroup", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
-
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("-1"),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- Description: aws_v2.String("allow node to communicate with each other"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked IngressWithinNodeGroupSecurityGroup")
-
- // allow pods to communicate with the cluster API Server
- ts.cfg.Logger.Info("revoking Ingress443FromNGtoCP", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
-
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(443),
- ToPort: aws_v2.Int32(443),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- Description: aws_v2.String("allow pods to communicate with the cluster API Server"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked Ingress443FromNGtoCP")
-
- // allow pods running extension API servers on port 443
- // to receive communication from cluster control plane
- ts.cfg.Logger.Info("revoking Ingress443FromCPtoNG", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupIngressInput{
- // egress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(443),
- ToPort: aws_v2.Int32(443),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- Description: aws_v2.String("receive communication from cluster control plane"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked Ingress443FromCPtoNG")
-
- // allow the cluster control plane to communicate with pods running extension API servers on port 443
- ts.cfg.Logger.Info("revoking Egress443FromCPtoNG", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupEgress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupEgressInput{
- // egress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(443),
- ToPort: aws_v2.Int32(443),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- Description: aws_v2.String("communicate with pods running extension API servers on port 443"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke egress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked Egress443FromCPtoNG")
-
- // allow worker Kubelets and pods to receive communication from the cluster control plane
- ts.cfg.Logger.Info("revoking IngressAllFromCPtoNG", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(0),
- ToPort: aws_v2.Int32(65535),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- Description: aws_v2.String("receive communication from the cluster control plane"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked IngressAllFromCPtoNG")
-
- // allow the cluster control plane to communicate with worker Kubelet and pods
- ts.cfg.Logger.Info("revoking EgressAllFromCPtoNG", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupEgress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupEgressInput{
- // egress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(0),
- ToPort: aws_v2.Int32(65535),
- UserIdGroupPairs: []aws_ec2_v2_types.UserIdGroupPair{
- {
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- Description: aws_v2.String("communicate with worker Kubelet and pods"),
- VpcId: aws_v2.String(ts.cfg.EKSConfig.VPC.ID),
- },
- },
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke egress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked EgressAllFromCPtoNG")
-
- ts.cfg.Logger.Info("revoking Ingress22ForSSH", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- IpRanges: []aws_ec2_v2_types.IpRange{
- {
- CidrIp: aws_v2.String("0.0.0.0/0"),
- },
- },
- FromPort: aws_v2.Int32(22),
- ToPort: aws_v2.Int32(22),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked Ingress22ForSSH")
-
- ts.cfg.Logger.Info("revoking IngressForGuestBook", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- IpRanges: []aws_ec2_v2_types.IpRange{
- {
- CidrIp: aws_v2.String("0.0.0.0/0"),
- },
- },
- FromPort: aws_v2.Int32(1),
- ToPort: aws_v2.Int32(10000),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked IngressForGuestBook")
-
- ts.cfg.Logger.Info("revoking EgressForGuestBook", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupEgress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupEgressInput{
- // egress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.SecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- FromPort: aws_v2.Int32(1),
- ToPort: aws_v2.Int32(10000),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke egress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked EgressForGuestBook")
-
- ts.cfg.Logger.Info("revoking IngressForNodePortConformance", zap.String("sg-id", ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID))
- _, err = ts.cfg.EC2APIV2.RevokeSecurityGroupIngress(
- context.Background(),
- &aws_ec2_v2.RevokeSecurityGroupIngressInput{
- // ingress target
- GroupId: aws_v2.String(ts.cfg.EKSConfig.VPC.NodeGroupSecurityGroupID),
- IpPermissions: []aws_ec2_v2_types.IpPermission{
- {
- IpProtocol: aws_v2.String("tcp"),
- IpRanges: []aws_ec2_v2_types.IpRange{
- {
- CidrIp: aws_v2.String("0.0.0.0/0"),
- },
- },
- FromPort: aws_v2.Int32(1),
- ToPort: aws_v2.Int32(32767),
- },
- },
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to revoke ingress", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- err = nil
- }
- }
- if err != nil {
- return err
- }
- }
- ts.cfg.Logger.Info("revoked IngressForNodePortConformance")
-
- ts.cfg.Logger.Info("revoked security group")
- return nil
-}
diff --git a/eks/ng/ssm.go b/eks/ng/ssm.go
deleted file mode 100644
index 25a770e13..000000000
--- a/eks/ng/ssm.go
+++ /dev/null
@@ -1,274 +0,0 @@
-package ng
-
-import (
- "context"
- "errors"
- "fmt"
- "path"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/pkg/randutil"
- aws_v2 "github.com/aws/aws-sdk-go-v2/aws"
- aws_ssm_v2 "github.com/aws/aws-sdk-go-v2/service/ssm"
- aws_ssm_v2_types "github.com/aws/aws-sdk-go-v2/service/ssm/types"
- smithy "github.com/aws/smithy-go"
- "github.com/dustin/go-humanize"
- "go.uber.org/zap"
-)
-
-func (ts *tester) createSSM() error {
- if err := ts.createSSMDocument(); err != nil {
- return err
- }
- if err := ts.sendSSMDocumentCommand(); err != nil {
- return err
- }
- return nil
-}
-
-func (ts *tester) deleteSSM() error {
- if err := ts.deleteSSMDocument(); err != nil {
- return err
- }
- return nil
-}
-
-func (ts *tester) createSSMDocument() error {
- createStart := time.Now()
-
- for asgName, cur := range ts.cfg.EKSConfig.AddOnNodeGroups.ASGs {
- if cur.SSM == nil {
- continue
- }
- if !cur.SSM.DocumentCreate {
- ts.cfg.Logger.Info("skipping SSM document create",
- zap.String("asg-name", asgName),
- zap.String("ssm-document-name", cur.SSM.DocumentName),
- )
- continue
- }
-
- // ref. https://docs.aws.amazon.com/systems-manager/latest/userguide/create-ssm-document-api.html
- content := `---
-schemaVersion: '2.2'
-description: aws:runShellScript
-parameters:
- executionTimeoutSeconds:
- type: String
- description: 'timeout for script, in seconds'
- commands:
- type: String
- description: "(Required) The commands to run or the path to an existing script on the instance."
- default: echo Hello World
-mainSteps:
-- action: aws:runShellScript
- name: %s
- inputs:
- timeoutSeconds: '{{ executionTimeoutSeconds }}'
- runCommand:
- - "{{ commands }}"
-`
- ts.cfg.Logger.Info("creating SSM document",
- zap.String("asg-name", asgName),
- zap.String("ssm-document-name", cur.SSM.DocumentName),
- )
- _, err := ts.cfg.SSMAPIV2.CreateDocument(
- context.Background(),
- &aws_ssm_v2.CreateDocumentInput{
- Name: aws_v2.String(cur.SSM.DocumentName),
- DocumentFormat: aws_ssm_v2_types.DocumentFormatYaml,
- DocumentType: aws_ssm_v2_types.DocumentTypeCommand,
- VersionName: aws_v2.String("v1"),
- Tags: []aws_ssm_v2_types.Tag{
- {
- Key: aws_v2.String("Name"),
- Value: aws_v2.String(ts.cfg.EKSConfig.Name),
- },
- {
- Key: aws_v2.String("DocumentName"),
- Value: aws_v2.String(cur.SSM.DocumentName),
- },
- {
- Key: aws_v2.String("DocumentVersion"),
- Value: aws_v2.String("v1"),
- },
- },
- // ref. https://docs.aws.amazon.com/systems-manager/latest/userguide/create-ssm-document-api.html
- Content: aws_v2.String(fmt.Sprintf(content, cur.SSM.DocumentName)),
- },
- )
- if err != nil {
- return err
- }
-
- ts.cfg.Logger.Info("created SSM Document",
- zap.String("asg-name", cur.Name),
- zap.String("ssm-document-name", cur.SSM.DocumentName),
- zap.Int("instances", len(cur.Instances)),
- zap.String("started", humanize.RelTime(createStart, time.Now(), "ago", "from now")),
- )
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) deleteSSMDocument() (err error) {
- for asgName, cur := range ts.cfg.EKSConfig.AddOnNodeGroups.ASGs {
- if cur.SSM == nil {
- continue
- }
- if !cur.SSM.DocumentCreate {
- ts.cfg.Logger.Info("skipping SSM document delete",
- zap.String("asg-name", asgName),
- zap.String("ssm-document-name", cur.SSM.DocumentName),
- )
- continue
- }
-
- ts.cfg.Logger.Info("deleting SSM document",
- zap.String("asg-name", cur.Name),
- zap.String("ssm-document-name", cur.SSM.DocumentName),
- )
- _, err = ts.cfg.SSMAPIV2.DeleteDocument(
- context.Background(),
- &aws_ssm_v2.DeleteDocumentInput{
- Name: aws_v2.String(cur.SSM.DocumentName),
- Force: true,
- },
- )
- if err != nil {
- ts.cfg.Logger.Warn("failed to delete SSM document", zap.Error(err))
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- if strings.Contains(apiErr.ErrorCode(), "NotFound") {
- ts.cfg.EKSConfig.Status.DeletedResources[cur.SSM.DocumentName] = "SSM.DocumentName"
- ts.cfg.EKSConfig.Sync()
- err = nil
- }
- }
- // InvalidDocument: Document eks2021071804awseyzymhjfdInstallBottlerocket does not exist in your account
- if err != nil {
- if strings.Contains(err.Error(), "does not exist") {
- ts.cfg.EKSConfig.Status.DeletedResources[cur.SSM.DocumentName] = "SSM.DocumentName"
- ts.cfg.EKSConfig.Sync()
- err = nil
- }
- }
- } else {
- ts.cfg.EKSConfig.Status.DeletedResources[cur.SSM.DocumentName] = "SSM.DocumentName"
- ts.cfg.EKSConfig.Sync()
- }
- if err == nil {
- ts.cfg.EKSConfig.RecordStatus(fmt.Sprintf("%q/%s", cur.SSM.DocumentName, ec2config.StatusDELETEDORNOTEXIST))
- }
-
- ts.cfg.Logger.Info("deleted SSM document",
- zap.String("asg-name", cur.Name),
- zap.String("ssm-document-name", cur.SSM.DocumentName),
- )
- }
-
- ts.cfg.EKSConfig.Sync()
- return err
-}
-
-func (ts *tester) sendSSMDocumentCommand() error {
- for asgName, cur := range ts.cfg.EKSConfig.AddOnNodeGroups.ASGs {
- if cur.SSM == nil {
- continue
- }
- if cur.SSM.DocumentName == "" {
- ts.cfg.Logger.Info("skipping SSM document send",
- zap.String("asg-name", asgName),
- zap.String("ssm-document-name", cur.SSM.DocumentName),
- )
- continue
- }
- if len(cur.Instances) == 0 {
- return fmt.Errorf("no instance found to run SSM document %q (asg name %q)", cur.SSM.DocumentName, asgName)
- }
-
- ts.cfg.Logger.Info("waiting before sending SSM document")
- select {
- case <-ts.cfg.Stopc:
- return errors.New("stopped")
- case <-time.After(15 * time.Second):
- }
-
- ids := make([]string, 0)
- for id := range cur.Instances {
- ids = append(ids, id)
- }
-
- // batch by 50
- // e.g. 'instanceIds' failed to satisfy constraint: Member must have length less than or equal to 50
- ts.cfg.Logger.Info("start sending SSM document",
- zap.String("asg-name", asgName),
- zap.String("ssm-document-name", cur.SSM.DocumentName),
- zap.Strings("instance-ids", ids),
- )
-
- left := make([]string, len(ids))
- copy(left, ids)
- for len(left) > 0 {
- batch := make([]string, 0)
- switch {
- case len(left) <= 50:
- batch = append(batch, left...)
- left = left[:0:0]
- case len(left) > 50:
- batch = append(batch, left[:50]...)
- left = left[50:]
- }
- ts.cfg.Logger.Info("batching SSM document", zap.Strings("batch", batch))
- ssmInput := &aws_ssm_v2.SendCommandInput{
- DocumentName: aws_v2.String(cur.SSM.DocumentName),
- Comment: aws_v2.String(cur.SSM.DocumentName + "-" + randutil.String(10)),
- InstanceIds: batch,
- MaxConcurrency: aws_v2.String(fmt.Sprintf("%d", len(batch))),
- Parameters: map[string][]string{
- "executionTimeoutSeconds": {fmt.Sprintf("%d", cur.SSM.DocumentExecutionTimeoutSeconds)},
- "commands": {cur.SSM.DocumentCommands},
- },
- OutputS3BucketName: aws_v2.String(ts.cfg.EKSConfig.S3.BucketName),
- OutputS3KeyPrefix: aws_v2.String(path.Join(ts.cfg.EKSConfig.Name, "ssm-outputs")),
- }
- cmd, err := ts.cfg.SSMAPIV2.SendCommand(
- context.Background(),
- ssmInput,
- )
- if err != nil {
- return err
- }
- docName := aws_v2.ToString(cmd.Command.DocumentName)
- if docName != cur.SSM.DocumentName {
- return fmt.Errorf("SSM Document Name expected %q, got %q", cur.SSM.DocumentName, docName)
- }
- cmdID := aws_v2.ToString(cmd.Command.CommandId)
- cur.SSM.DocumentCommandIDs = append(cur.SSM.DocumentCommandIDs, cmdID)
-
- ts.cfg.Logger.Info("sent SSM document",
- zap.String("asg-name", asgName),
- zap.String("ssm-document-name", cur.SSM.DocumentName),
- zap.String("ssm-command-id", cmdID),
- zap.Int("sent-instance-ids", len(batch)),
- zap.Int("left-instance-ids", len(left)),
- )
- if len(left) == 0 {
- break
- }
-
- ts.cfg.Logger.Info("waiting for next SSM run batch", zap.Int("left", len(left)))
- time.Sleep(15 * time.Second)
- }
-
- ts.cfg.EKSConfig.AddOnNodeGroups.ASGs[asgName] = cur
- ts.cfg.EKSConfig.Sync()
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
diff --git a/eks/ng/tuple.go b/eks/ng/tuple.go
deleted file mode 100644
index 86b817469..000000000
--- a/eks/ng/tuple.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package ng
-
-import "time"
-
-type tupleTime struct {
- ts time.Time
- name string
-}
-
-type tupleTimes []tupleTime
-
-func (ts tupleTimes) Len() int { return len(ts) }
-
-func (ts tupleTimes) Less(i, j int) bool {
- return ts[j].ts.After(ts[i].ts)
-}
-
-func (ts tupleTimes) Swap(i, j int) {
- t := ts[i]
- ts[i] = ts[j]
- ts[j] = t
-}
diff --git a/eks/ng/tuple_test.go b/eks/ng/tuple_test.go
deleted file mode 100644
index 599b9499e..000000000
--- a/eks/ng/tuple_test.go
+++ /dev/null
@@ -1,26 +0,0 @@
-package ng
-
-import (
- "reflect"
- "sort"
- "testing"
- "time"
-)
-
-func Test_byTime(t *testing.T) {
- ts := time.Time{}
- tss1 := []tupleTime{
- {ts: ts.Add(time.Second), name: "1"},
- {ts: ts.Add(2 * time.Second), name: "2"},
- {ts: ts.Add(3 * time.Second), name: "3"},
- }
- sort.Sort(sort.Reverse(tupleTimes(tss1)))
- tss2 := []tupleTime{
- {ts: ts.Add(3 * time.Second), name: "3"},
- {ts: ts.Add(2 * time.Second), name: "2"},
- {ts: ts.Add(time.Second), name: "1"},
- }
- if !reflect.DeepEqual(tss1, tss2) {
- t.Fatalf("expected %+v, got %+v", tss2, tss1)
- }
-}
diff --git a/eks/ng/wait/wait.go b/eks/ng/wait/wait.go
deleted file mode 100644
index 5e2f8c980..000000000
--- a/eks/ng/wait/wait.go
+++ /dev/null
@@ -1,345 +0,0 @@
-// Package wait implements node waiter.
-package wait
-
-import (
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "reflect"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/eksconfig"
- aws_ec2 "github.com/aws/aws-k8s-tester/pkg/aws/ec2"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- k8s_object "github.com/aws/aws-k8s-tester/pkg/k8s-object"
- aws_v2 "github.com/aws/aws-sdk-go-v2/aws"
- aws_asg_v2 "github.com/aws/aws-sdk-go-v2/service/autoscaling"
- aws_asg_v2_types "github.com/aws/aws-sdk-go-v2/service/autoscaling/types"
- aws_ec2_v2 "github.com/aws/aws-sdk-go-v2/service/ec2"
- "go.uber.org/zap"
- certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
- v1 "k8s.io/api/core/v1"
-)
-
-// NodeWaiter defines node waiter operation.
-type NodeWaiter interface {
- // Wait waits until all NG and Kubernetes nodes are ready.
- Wait(asgName string, retries int) error
-}
-
-// Config defines version upgrade configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
- EC2APIV2 *aws_ec2_v2.Client
- ASGAPIV2 *aws_asg_v2.Client
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-// New creates a new node waiter.
-func New(cfg Config) NodeWaiter {
- cfg.Logger.Info("creating node waiter", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
-}
-
-func (ts *tester) Wait(asgName string, retries int) error {
- return ts.waitForNodes(asgName, retries)
-}
-
-func (ts *tester) waitForNodes(asgName string, retriesLeft int) error {
- cur, ok := ts.cfg.EKSConfig.AddOnNodeGroups.ASGs[asgName]
- if !ok {
- return fmt.Errorf("ASGs[%q] not found", asgName)
- }
-
- ts.cfg.Logger.Info("checking NG using ASG API", zap.String("asg-name", cur.Name))
- aout, err := ts.cfg.ASGAPIV2.DescribeAutoScalingGroups(
- context.Background(),
- &aws_asg_v2.DescribeAutoScalingGroupsInput{
- AutoScalingGroupNames: []string{cur.Name},
- })
- if err != nil {
- return fmt.Errorf("ASGs[%q] not found (%v)", cur.Name, err)
- }
- if len(aout.AutoScalingGroups) != 1 {
- if retriesLeft > 0 {
- ts.cfg.Logger.Warn("expected only 1 ASG; retrying", zap.String("asg-name", cur.Name), zap.Int("retries-left", retriesLeft))
- time.Sleep(5 * time.Second)
- return ts.waitForNodes(asgName, retriesLeft-1)
- }
- return fmt.Errorf("%q expected only 1 ASG, got %+v", cur.Name, aout.AutoScalingGroups)
- }
- av := aout.AutoScalingGroups[0]
- instanceIDs := make([]string, 0)
- for _, iv := range av.Instances {
- lv := iv.LifecycleState
- switch lv {
- case aws_asg_v2_types.LifecycleStatePending,
- aws_asg_v2_types.LifecycleStatePendingWait,
- aws_asg_v2_types.LifecycleStatePendingProceed,
- aws_asg_v2_types.LifecycleStateInService:
- instanceIDs = append(instanceIDs, aws_v2.ToString(iv.InstanceId))
- default:
- zap.L().Warn("skipping instance due to lifecycle state",
- zap.String("instance-id", aws_v2.ToString(iv.InstanceId)),
- zap.String("lifecycle-state", fmt.Sprint(lv)),
- )
- }
- }
- if len(instanceIDs) != int(cur.ASGDesiredCapacity) {
- if retriesLeft > 0 {
- ts.cfg.Logger.Warn("not enough instances in desired state; retrying",
- zap.Int("instances", len(instanceIDs)),
- zap.String("asg-name", cur.Name),
- zap.Int("retries-left", retriesLeft),
- )
- time.Sleep(5 * time.Second)
- return ts.waitForNodes(asgName, retriesLeft-1)
- }
- return fmt.Errorf("not enough instances in ASG; expected %d, got %d", cur.ASGDesiredCapacity, len(instanceIDs))
- }
-
- checkN := time.Duration(cur.ASGDesiredCapacity)
- if checkN == 0 {
- checkN = time.Duration(cur.ASGMinSize)
- }
- waitDur := 30*time.Minute + 10*time.Second*checkN
- if strings.Contains(cur.InstanceType, ".metal") { // "i3.metal" takes much longer
- ts.cfg.Logger.Info("increasing wait time for metal instance", zap.String("instance-type", cur.InstanceType))
- waitDur = time.Hour + time.Minute*checkN
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), waitDur)
- ec2Instances, err := aws_ec2.WaitUntilRunning(
- ctx,
- ts.cfg.Stopc,
- ts.cfg.EC2APIV2,
- ts.cfg.ASGAPIV2,
- cur.Name,
- )
- cancel()
- if err != nil {
- return err
- }
-
- cur, ok = ts.cfg.EKSConfig.AddOnNodeGroups.ASGs[asgName]
- if !ok {
- return fmt.Errorf("ASG %q not found", asgName)
- }
- cur.Instances = make(map[string]ec2config.Instance)
- for id, vv := range ec2Instances {
- ivv := ec2config.ConvertInstance(vv)
- ivv.RemoteAccessUserName = cur.RemoteAccessUserName
- cur.Instances[id] = ivv
- }
- ts.cfg.Logger.Info("waited for ASG", zap.String("asg-name", cur.Name), zap.Int("instances", len(cur.Instances)))
-
- for _, inst := range cur.Instances {
- ts.cfg.EKSConfig.Status.PrivateDNSToNodeInfo[inst.PrivateDNSName] = eksconfig.NodeInfo{
- NodeGroupName: cur.Name,
- AMIType: cur.AMIType,
- PublicIP: inst.PublicIP,
- PublicDNSName: inst.PublicDNSName,
- UserName: cur.RemoteAccessUserName,
- }
- }
- ts.cfg.EKSConfig.AddOnNodeGroups.ASGs[asgName] = cur
- ts.cfg.EKSConfig.Sync()
-
- cur, ok = ts.cfg.EKSConfig.AddOnNodeGroups.ASGs[asgName]
- if !ok {
- return fmt.Errorf("ASG %q not found", asgName)
- }
-
- // Hostname/InternalDNS == EC2 private DNS
- // TODO: handle DHCP option domain name
- ec2PrivateDNS := make(map[string]struct{})
- for _, v := range cur.Instances {
- ts.cfg.Logger.Debug("found private DNS for an EC2 instance", zap.String("instance-id", v.InstanceID), zap.String("private-dns-name", v.PrivateDNSName))
- ec2PrivateDNS[v.PrivateDNSName] = struct{}{}
- // "ip-192-168-81-186" from "ip-192-168-81-186.my-private-dns"
- ec2PrivateDNS[strings.Split(v.PrivateDNSName, ".")[0]] = struct{}{}
- }
-
- ts.cfg.Logger.Info("checking nodes readiness", zap.Duration("wait", waitDur))
- retryStart := time.Now()
- ready := false
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("checking node aborted")
- case <-time.After(5 * time.Second):
- }
-
- nodes, err := ts.cfg.K8SClient.ListNodes(1000, 5*time.Second)
- if err != nil {
- ts.cfg.Logger.Warn("get nodes failed", zap.Error(err))
- continue
- }
-
- readies := 0
- for _, node := range nodes {
- labels := node.GetLabels()
- if labels["NGName"] != asgName {
- continue
- }
- nodeName := node.GetName()
- nodeInfo, _ := json.Marshal(k8s_object.ParseNodeInfo(node.Status.NodeInfo))
-
- // e.g. given node name ip-192-168-81-186.us-west-2.compute.internal + DHCP option my-private-dns
- // InternalIP == 192.168.81.186
- // ExternalIP == 52.38.118.149
- // Hostname == my-private-dns (without DHCP option, it's "ip-192-168-81-186.my-private-dns", private DNS, InternalDNS)
- // InternalDNS == ip-192-168-81-186.my-private-dns
- // ExternalDNS == ec2-52-38-118-149.us-west-2.compute.amazonaws.com
- ts.cfg.Logger.Debug("checking node address with EC2 Private DNS",
- zap.String("node-name", nodeName),
- zap.String("node-info", string(nodeInfo)),
- zap.String("labels", fmt.Sprintf("%v", labels)),
- )
-
- hostName := ""
- for _, av := range node.Status.Addresses {
- ts.cfg.Logger.Debug("node status address",
- zap.String("node-name", nodeName),
- zap.String("type", string(av.Type)),
- zap.String("address", string(av.Address)),
- )
- if av.Type != v1.NodeHostName && av.Type != v1.NodeInternalDNS {
- continue
- }
- // handle when node is configured DHCP
- hostName = av.Address
- _, ok := ec2PrivateDNS[hostName]
- if !ok {
- // "ip-192-168-81-186" from "ip-192-168-81-186.my-private-dns"
- _, ok = ec2PrivateDNS[strings.Split(hostName, ".")[0]]
- }
- if ok {
- break
- }
- }
- if hostName == "" {
- return fmt.Errorf("%q not found for node %q", v1.NodeHostName, nodeName)
- }
- _, ok := ec2PrivateDNS[hostName]
- if !ok {
- // "ip-192-168-81-186" from "ip-192-168-81-186.my-private-dns"
- _, ok = ec2PrivateDNS[strings.Split(hostName, ".")[0]]
- }
- if !ok {
- ts.cfg.Logger.Warn("node may not belong to this ASG", zap.String("host-name", hostName), zap.Int("ec2-private-dnss", len(ec2PrivateDNS)))
- continue
- }
- ts.cfg.Logger.Debug("checked node host name with EC2 Private DNS", zap.String("name", nodeName), zap.String("host-name", hostName))
-
- for _, cond := range node.Status.Conditions {
- if cond.Status != v1.ConditionTrue {
- continue
- }
- if cond.Type != v1.NodeReady {
- continue
- }
- ts.cfg.Logger.Debug("node is ready!",
- zap.String("name", nodeName),
- zap.String("status-type", fmt.Sprint(cond.Type)),
- zap.String("status", fmt.Sprint(cond.Status)),
- )
- readies++
- break
- }
- }
- /*
- e.g.
- "/tmp/kubectl-test-v1.16.9 --kubeconfig=/tmp/leegyuho-test-eks.kubeconfig.yaml get csr -o=wide":
- NAME AGE REQUESTOR CONDITION
- csr-4msk5 58s system:node:ip-192-168-65-124.us-west-2.compute.internal Approved,Issued
- csr-9dbs8 57s system:node:ip-192-168-208-6.us-west-2.compute.internal Approved,Issued
- */
- allCSRs := make(map[string]int)
- output, err := ts.cfg.K8SClient.ListCSRs(1000, 5*time.Second)
- if err != nil {
- ts.cfg.Logger.Warn("list CSRs failed", zap.Error(err))
- } else {
- for _, cv := range output {
- k := extractCSRStatus(cv)
- ts.cfg.Logger.Debug("current CSR",
- zap.String("name", cv.GetName()),
- zap.String("requester", cv.Spec.Username),
- zap.String("status", extractCSRStatus(cv)),
- )
- v, ok := allCSRs[k]
- if !ok {
- allCSRs[k] = 1
- } else {
- allCSRs[k] = v + 1
- }
- }
- }
- ts.cfg.Logger.Info("polling nodes",
- zap.String("command", ts.cfg.EKSConfig.KubectlCommand()+" get nodes"),
- zap.String("ng-name", cur.Name),
- zap.Int("current-ready-nodes", readies),
- zap.Int32("min-ready-nodes", cur.ASGMinSize),
- zap.Int32("desired-ready-nodes", cur.ASGDesiredCapacity),
- zap.String("all-csrs", fmt.Sprintf("%+v", allCSRs)),
- )
- if int32(readies) >= cur.ASGMinSize {
- ready = true
- break
- }
- }
- if !ready {
- if retriesLeft > 0 {
- ts.cfg.Logger.Warn("nodes in ASG are not ready yet; retrying", zap.String("asg-name", cur.Name), zap.Int("retries-left", retriesLeft))
- time.Sleep(5 * time.Second)
- return ts.waitForNodes(asgName, retriesLeft-1)
- }
- return fmt.Errorf("ASG %q not ready", asgName)
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// "pkg/printers/internalversion/printers.go"
-func extractCSRStatus(csr certificatesv1beta1.CertificateSigningRequest) string {
- var approved, denied bool
- for _, c := range csr.Status.Conditions {
- switch c.Type {
- case certificatesv1beta1.CertificateApproved:
- approved = true
- case certificatesv1beta1.CertificateDenied:
- denied = true
- default:
- return ""
- }
- }
- var status string
- // must be in order of presidence
- if denied {
- status += "Denied"
- } else if approved {
- status += "Approved"
- } else {
- status += "Pending"
- }
- if len(csr.Status.Certificate) > 0 {
- status += ",Issued"
- }
- return status
-}
diff --git a/eks/nlb-guestbook/nlb-guestbook.go b/eks/nlb-guestbook/nlb-guestbook.go
deleted file mode 100644
index c5c9ea5f0..000000000
--- a/eks/nlb-guestbook/nlb-guestbook.go
+++ /dev/null
@@ -1,1140 +0,0 @@
-// Package nlbguestbook implements NLB plugin
-// with a simple guestbook service.
-// ref. https://github.com/kubernetes/examples/tree/master/guestbook-go
-// ref. https://docs.aws.amazon.com/eks/latest/userguide/eks-guestbook.html
-package nlbguestbook
-
-import (
- "context"
- "errors"
- "fmt"
- "io"
- "io/ioutil"
- "reflect"
- "strings"
- "time"
-
- eks_tester "github.com/aws/aws-k8s-tester/eks/tester"
- "github.com/aws/aws-k8s-tester/eksconfig"
- "github.com/aws/aws-k8s-tester/pkg/aws/elb"
- "github.com/aws/aws-k8s-tester/pkg/httputil"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface"
- "go.uber.org/zap"
- appsv1 "k8s.io/api/apps/v1"
- v1 "k8s.io/api/core/v1"
- apierrs "k8s.io/apimachinery/pkg/api/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/util/intstr"
- "k8s.io/utils/exec"
-)
-
-// Config defines NLB configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
- ELB2API elbv2iface.ELBV2API
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-// New creates a new Job tester.
-func New(cfg Config) eks_tester.Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
-}
-
-const (
- redisLabelName = "redis"
-
- redisLeaderDeploymentName = "redis-leader"
- redisLeaderAppName = "redis-master"
- redisLeaderAppImageName = "redis:2.8.23" // ref. https://hub.docker.com/_/redis/?tab=tags
- redisLeaderServiceName = "redis-master" // e..g "Connecting to MASTER redis-master:6379"
- redisLeaderRoleName = "master" // TODO: change this to "leader"
-
- redisFollowerDeploymentName = "redis-follower"
- redisFollowerAppName = "redis-slave"
- redisFollowerAppImageName = "k8s.gcr.io/redis-slave:v2" // ref. https://hub.docker.com/_/redis/?tab=tags
- redisFollowerServiceName = "redis-slave"
- redisFollowerRoleName = "slave" // TODO: change this to "follower"
-
- nlbGuestbookDeploymentName = "guestbook"
- nlbGuestbookAppName = "guestbook"
- nlbGuestbookAppImageName = "k8s.gcr.io/guestbook:v3"
- nlbGuestbookServiceName = "guestbook"
-)
-
-func (ts *tester) Create() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnNLBGuestbook() {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
- if ts.cfg.EKSConfig.AddOnNLBGuestbook.Created {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Create", zap.String("tester", pkgName))
- ts.cfg.EKSConfig.AddOnNLBGuestbook.Created = true
- ts.cfg.EKSConfig.Sync()
- createStart := time.Now()
- defer func() {
- createEnd := time.Now()
- ts.cfg.EKSConfig.AddOnNLBGuestbook.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- if err := k8s_client.CreateNamespace(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- ); err != nil {
- return err
- }
-
- if err := ts.createDeploymentRedisLeader(); err != nil {
- return err
- }
- if err := ts.waitDeploymentRedisLeader(); err != nil {
- return err
- }
- if err := ts.createServiceRedisLeader(); err != nil {
- return err
- }
-
- if err := ts.createDeploymentRedisFollower(); err != nil {
- return err
- }
- if err := ts.waitDeploymentRedisFollower(); err != nil {
- return err
- }
- if err := ts.createServiceRedisFollower(); err != nil {
- return err
- }
-
- if err := ts.createDeploymentGuestbook(); err != nil {
- return err
- }
- if err := ts.waitDeploymentGuestbook(); err != nil {
- return err
- }
- if err := ts.createServiceGuestbook(); err != nil {
- return err
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) Delete() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnNLBGuestbook() {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnNLBGuestbook.Created {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Delete", zap.String("tester", pkgName))
- deleteStart := time.Now()
- defer func() {
- deleteEnd := time.Now()
- ts.cfg.EKSConfig.AddOnNLBGuestbook.TimeFrameDelete = timeutil.NewTimeFrame(deleteStart, deleteEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- var errs []string
-
- if err := ts.deleteServiceGuestbook(); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete NLB guestbook Service (%v)", err))
- }
- ts.cfg.Logger.Info("wait for 3-minute after deleting Service")
- time.Sleep(3 * time.Minute)
- if err := ts.deleteDeploymentGuestbook(); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete NLB guestbook Deployment (%v)", err))
- }
- ts.cfg.Logger.Info("wait for a minute after deleting Deployment")
- time.Sleep(time.Minute)
-
- if err := ts.deleteServiceRedisFollower(); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete redis follower Service (%v)", err))
- }
- ts.cfg.Logger.Info("wait for a minute after deleting Service")
- time.Sleep(time.Minute)
- if err := ts.deleteDeploymentRedisFollower(); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete redis follower Deployment (%v)", err))
- }
- ts.cfg.Logger.Info("wait for a minute after deleting Deployment")
- time.Sleep(time.Minute)
-
- if err := ts.deleteServiceRedisLeader(); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete redis leader Service (%v)", err))
- }
- ts.cfg.Logger.Info("wait for a minute after deleting Service")
- time.Sleep(time.Minute)
- if err := ts.deleteDeploymentRedisLeader(); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete redis leader Deployment (%v)", err))
- }
- ts.cfg.Logger.Info("wait for a minute after deleting Deployment")
- time.Sleep(time.Minute)
-
- /*
- # NLB tags
- kubernetes.io/service-name
- leegyuho-test-prod-nlb-guestbook/guestbook-service
-
- kubernetes.io/cluster/leegyuho-test-prod
- owned
- */
- if err := elb.DeleteELBv2(
- ts.cfg.Logger,
- ts.cfg.ELB2API,
- ts.cfg.EKSConfig.AddOnNLBGuestbook.NLBARN,
- ts.cfg.EKSConfig.VPC.ID,
- map[string]string{
- "kubernetes.io/cluster/" + ts.cfg.EKSConfig.Name: "owned",
- "kubernetes.io/service-name": ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace + "/" + nlbGuestbookServiceName,
- },
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete NLB guestbook (%v)", err))
- }
-
- if err := k8s_client.DeleteNamespaceAndWait(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- k8s_client.DefaultNamespaceDeletionInterval,
- k8s_client.DefaultNamespaceDeletionTimeout,
- k8s_client.WithForceDelete(true),
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete NLB namespace (%v)", err))
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- ts.cfg.EKSConfig.AddOnNLBGuestbook.Created = false
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) createDeploymentRedisLeader() error {
- var nodeSelector map[string]string
- if len(ts.cfg.EKSConfig.AddOnNLBGuestbook.DeploymentNodeSelector) > 0 {
- nodeSelector = ts.cfg.EKSConfig.AddOnNLBGuestbook.DeploymentNodeSelector
- } else {
- nodeSelector = nil
- }
- ts.cfg.Logger.Info("creating redis leader Deployment", zap.Any("node-selector", nodeSelector))
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.K8SClient.KubernetesClientSet().
- AppsV1().
- Deployments(ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace).
- Create(
- ctx,
- &appsv1.Deployment{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "apps/v1",
- Kind: "Deployment",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: redisLeaderDeploymentName,
- Namespace: ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- Labels: map[string]string{
- "app.kubernetes.io/name": redisLabelName,
- "role": redisLeaderRoleName,
- },
- },
- Spec: appsv1.DeploymentSpec{
- Replicas: aws.Int32(1),
- Selector: &metav1.LabelSelector{
- MatchLabels: map[string]string{
- "app.kubernetes.io/name": redisLabelName,
- "role": redisLeaderRoleName,
- },
- },
- Template: v1.PodTemplateSpec{
- ObjectMeta: metav1.ObjectMeta{
- Labels: map[string]string{
- "app.kubernetes.io/name": redisLabelName,
- "role": redisLeaderRoleName,
- },
- },
- Spec: v1.PodSpec{
- RestartPolicy: v1.RestartPolicyAlways,
- Containers: []v1.Container{
- {
- Name: redisLeaderAppName,
- Image: redisLeaderAppImageName,
- ImagePullPolicy: v1.PullAlways,
- Ports: []v1.ContainerPort{
- {
- Name: "redis-server",
- Protocol: v1.ProtocolTCP,
- ContainerPort: 6379,
- },
- },
- },
- },
- NodeSelector: nodeSelector,
- },
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create redis leader Deployment (%v)", err)
- }
-
- ts.cfg.Logger.Info("created redis leader Deployment")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) deleteDeploymentRedisLeader() error {
- ts.cfg.Logger.Info("deleting redis leader Deployment")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- AppsV1().
- Deployments(ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace).
- Delete(
- ctx,
- redisLeaderDeploymentName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete redis leader Deployment (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted redis leader Deployment")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) waitDeploymentRedisLeader() (err error) {
- timeout := 7 * time.Minute
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- _, err = k8s_client.WaitForDeploymentCompletes(
- ctx,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.Stopc,
- ts.cfg.K8SClient,
- time.Minute,
- 20*time.Second,
- ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- redisLeaderDeploymentName,
- 1,
- k8s_client.WithQueryFunc(func() {
- descArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- "describe",
- "deployment",
- redisLeaderDeploymentName,
- }
- descCmd := strings.Join(descArgs, " ")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe deployment' failed", zap.Error(err))
- }
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", descCmd, out)
-
- logsArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- "logs",
- "--selector=app.kubernetes.io/name=" + redisLabelName + ",role=" + redisLeaderRoleName,
- "--timestamps",
- }
- logsCmd := strings.Join(logsArgs, " ")
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- logsOutput, err := exec.New().CommandContext(ctx, logsArgs[0], logsArgs[1:]...).CombinedOutput()
- cancel()
- out = string(logsOutput)
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl logs' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\n\"%s\" output:\n\n%s\n\n", logsCmd, out)
- }),
- )
- cancel()
- return err
-}
-
-func (ts *tester) createServiceRedisLeader() error {
- ts.cfg.Logger.Info("creating redis leader Service")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace).
- Create(
- ctx,
- &v1.Service{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "v1",
- Kind: "Service",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: redisLeaderServiceName,
- Namespace: ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- Labels: map[string]string{
- "app.kubernetes.io/name": redisLabelName,
- "role": redisLeaderRoleName,
- },
- },
- Spec: v1.ServiceSpec{
- Selector: map[string]string{
- "app.kubernetes.io/name": redisLabelName,
- "role": redisLeaderRoleName,
- },
- Type: v1.ServiceTypeClusterIP,
- Ports: []v1.ServicePort{
- {
- Protocol: v1.ProtocolTCP,
- Port: 6379,
- TargetPort: intstr.FromString("redis-server"),
- },
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create redis leader Service (%v)", err)
- }
- ts.cfg.Logger.Info("created redis leader Service")
-
- args := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- "describe",
- "svc",
- redisLeaderServiceName,
- }
- argsCmd := strings.Join(args, " ")
-
- waitDur := 3 * time.Minute
- retryStart := time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("redis leader Service creation aborted")
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- cmdOut, err := exec.New().CommandContext(ctx, args[0], args[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe svc' failed", zap.String("command", argsCmd), zap.Error(err))
- } else {
- out := string(cmdOut)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", argsCmd, out)
- }
-
- ts.cfg.Logger.Info("querying redis leader Service")
- ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
- _, err = ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace).
- Get(ctx, redisLeaderServiceName, metav1.GetOptions{})
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("failed to get redis leader Service; retrying", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
-
- ts.cfg.Logger.Info("redis leader Service is ready")
- break
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) deleteServiceRedisLeader() error {
- ts.cfg.Logger.Info("deleting redis leader Service")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace).
- Delete(
- ctx,
- redisLeaderServiceName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete redis leader Service (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted redis leader Service", zap.Error(err))
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) createDeploymentRedisFollower() error {
- var nodeSelector map[string]string
- if len(ts.cfg.EKSConfig.AddOnNLBGuestbook.DeploymentNodeSelector) > 0 {
- nodeSelector = ts.cfg.EKSConfig.AddOnNLBGuestbook.DeploymentNodeSelector
- } else {
- nodeSelector = nil
- }
- ts.cfg.Logger.Info("creating redis follower Deployment", zap.Any("node-selector", nodeSelector))
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.K8SClient.KubernetesClientSet().
- AppsV1().
- Deployments(ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace).
- Create(
- ctx,
- &appsv1.Deployment{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "apps/v1",
- Kind: "Deployment",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: redisFollowerDeploymentName,
- Namespace: ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- Labels: map[string]string{
- "app.kubernetes.io/name": redisLabelName,
- "role": redisFollowerRoleName,
- },
- },
- Spec: appsv1.DeploymentSpec{
- Replicas: aws.Int32(2),
- Selector: &metav1.LabelSelector{
- MatchLabels: map[string]string{
- "app.kubernetes.io/name": redisLabelName,
- "role": redisFollowerRoleName,
- },
- },
- Template: v1.PodTemplateSpec{
- ObjectMeta: metav1.ObjectMeta{
- Labels: map[string]string{
- "app.kubernetes.io/name": redisLabelName,
- "role": redisFollowerRoleName,
- },
- },
- Spec: v1.PodSpec{
- RestartPolicy: v1.RestartPolicyAlways,
- Containers: []v1.Container{
- {
- Name: redisFollowerAppName,
- Image: redisFollowerAppImageName,
- ImagePullPolicy: v1.PullAlways,
- Ports: []v1.ContainerPort{
- {
- Name: "redis-server",
- Protocol: v1.ProtocolTCP,
- ContainerPort: 6379,
- },
- },
- },
- },
- NodeSelector: nodeSelector,
- },
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create redis follower Deployment (%v)", err)
- }
-
- ts.cfg.Logger.Info("created redis follower Deployment")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) deleteDeploymentRedisFollower() error {
- ts.cfg.Logger.Info("deleting redis follower Deployment")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- AppsV1().
- Deployments(ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace).
- Delete(
- ctx,
- redisFollowerDeploymentName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete redis follower Deployment (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted redis follower Deployment")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) waitDeploymentRedisFollower() (err error) {
- timeout := 7 * time.Minute
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- _, err = k8s_client.WaitForDeploymentCompletes(
- ctx,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.Stopc,
- ts.cfg.K8SClient,
- time.Minute,
- 20*time.Second,
- ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- redisFollowerDeploymentName,
- 1,
- k8s_client.WithQueryFunc(func() {
- descArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- "describe",
- "deployment",
- redisFollowerDeploymentName,
- }
- descCmd := strings.Join(descArgs, " ")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe deployment' failed", zap.Error(err))
- }
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", descCmd, out)
-
- logsArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- "logs",
- "--selector=app.kubernetes.io/name=" + redisLabelName + ",role=" + redisFollowerRoleName,
- "--timestamps",
- }
- logsCmd := strings.Join(logsArgs, " ")
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- logsOutput, err := exec.New().CommandContext(ctx, logsArgs[0], logsArgs[1:]...).CombinedOutput()
- cancel()
- out = string(logsOutput)
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl logs' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\n\"%s\" output:\n\n%s\n\n", logsCmd, out)
- }),
- )
- cancel()
- return err
-}
-
-func (ts *tester) createServiceRedisFollower() error {
- ts.cfg.Logger.Info("creating redis follower Service")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace).
- Create(
- ctx,
- &v1.Service{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "v1",
- Kind: "Service",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: redisFollowerServiceName,
- Namespace: ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- Labels: map[string]string{
- "app.kubernetes.io/name": redisLabelName,
- "role": redisFollowerRoleName,
- },
- },
- Spec: v1.ServiceSpec{
- Selector: map[string]string{
- "app.kubernetes.io/name": redisLabelName,
- "role": redisFollowerRoleName,
- },
- Type: v1.ServiceTypeClusterIP,
- Ports: []v1.ServicePort{
- {
- Protocol: v1.ProtocolTCP,
- Port: 6379,
- TargetPort: intstr.FromString("redis-server"),
- },
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create redis follower Service (%v)", err)
- }
- ts.cfg.Logger.Info("created redis follower Service")
-
- args := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- "describe",
- "svc",
- redisFollowerServiceName,
- }
- argsCmd := strings.Join(args, " ")
-
- waitDur := 3 * time.Minute
- retryStart := time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("redis follower Service creation aborted")
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- cmdOut, err := exec.New().CommandContext(ctx, args[0], args[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe svc' failed", zap.String("command", argsCmd), zap.Error(err))
- } else {
- out := string(cmdOut)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", argsCmd, out)
- }
-
- ts.cfg.Logger.Info("querying redis follower Service")
- ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
- _, err = ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace).
- Get(ctx, redisFollowerServiceName, metav1.GetOptions{})
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("failed to get redis follower Service; retrying", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
-
- ts.cfg.Logger.Info("redis follower Service is ready")
- break
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) deleteServiceRedisFollower() error {
- ts.cfg.Logger.Info("deleting redis follower Service")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace).
- Delete(
- ctx,
- redisFollowerServiceName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete redis follower Service (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted redis follower Service", zap.Error(err))
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) createDeploymentGuestbook() error {
- var nodeSelector map[string]string
- if len(ts.cfg.EKSConfig.AddOnNLBGuestbook.DeploymentNodeSelector) > 0 {
- nodeSelector = ts.cfg.EKSConfig.AddOnNLBGuestbook.DeploymentNodeSelector
- } else {
- nodeSelector = nil
- }
- ts.cfg.Logger.Info("creating NLB guestbook Deployment", zap.Any("node-selector", nodeSelector))
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.K8SClient.KubernetesClientSet().
- AppsV1().
- Deployments(ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace).
- Create(
- ctx,
- &appsv1.Deployment{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "apps/v1",
- Kind: "Deployment",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: nlbGuestbookDeploymentName,
- Namespace: ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- Labels: map[string]string{
- "app.kubernetes.io/name": nlbGuestbookAppName,
- },
- },
- Spec: appsv1.DeploymentSpec{
- Replicas: aws.Int32(ts.cfg.EKSConfig.AddOnNLBGuestbook.DeploymentReplicas),
- Selector: &metav1.LabelSelector{
- MatchLabels: map[string]string{
- "app.kubernetes.io/name": nlbGuestbookAppName,
- },
- },
- Template: v1.PodTemplateSpec{
- ObjectMeta: metav1.ObjectMeta{
- Labels: map[string]string{
- "app.kubernetes.io/name": nlbGuestbookAppName,
- },
- },
- Spec: v1.PodSpec{
- RestartPolicy: v1.RestartPolicyAlways,
- Containers: []v1.Container{
- {
- Name: nlbGuestbookAppName,
- Image: nlbGuestbookAppImageName,
- ImagePullPolicy: v1.PullAlways,
- Ports: []v1.ContainerPort{
- {
- Protocol: v1.ProtocolTCP,
- ContainerPort: 3000,
- Name: "http-server",
- },
- },
- },
- },
- NodeSelector: nodeSelector,
- },
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create NLB guestbook Deployment (%v)", err)
- }
-
- ts.cfg.Logger.Info("created NLB guestbook Deployment")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) deleteDeploymentGuestbook() error {
- ts.cfg.Logger.Info("deleting NLB guestbook Deployment")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- AppsV1().
- Deployments(ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace).
- Delete(
- ctx,
- nlbGuestbookDeploymentName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete NLB guestbook Deployment (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted NLB guestbook Deployment")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) waitDeploymentGuestbook() (err error) {
- timeout := 7*time.Minute + time.Duration(ts.cfg.EKSConfig.AddOnNLBGuestbook.DeploymentReplicas)*time.Minute
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- _, err = k8s_client.WaitForDeploymentCompletes(
- ctx,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.Stopc,
- ts.cfg.K8SClient,
- time.Minute,
- 20*time.Second,
- ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- nlbGuestbookDeploymentName,
- ts.cfg.EKSConfig.AddOnNLBGuestbook.DeploymentReplicas,
- k8s_client.WithQueryFunc(func() {
- descArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- "describe",
- "deployment",
- nlbGuestbookDeploymentName,
- }
- descCmd := strings.Join(descArgs, " ")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe deployment' failed", zap.Error(err))
- }
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", descCmd, out)
-
- logsArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- "logs",
- "--selector=app.kubernetes.io/name=" + nlbGuestbookAppName,
- "--timestamps",
- }
- logsCmd := strings.Join(logsArgs, " ")
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- logsOutput, err := exec.New().CommandContext(ctx, logsArgs[0], logsArgs[1:]...).CombinedOutput()
- cancel()
- out = string(logsOutput)
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl logs' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\n\"%s\" output:\n\n%s\n\n", logsCmd, out)
- }),
- )
- cancel()
- return err
-}
-
-func (ts *tester) createServiceGuestbook() error {
- ts.cfg.Logger.Info("creating NLB guestbook Service")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace).
- Create(
- ctx,
- &v1.Service{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "v1",
- Kind: "Service",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: nlbGuestbookServiceName,
- Namespace: ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- Labels: map[string]string{
- "app.kubernetes.io/name": nlbGuestbookAppName,
- },
- },
- Spec: v1.ServiceSpec{
- Selector: map[string]string{
- "app.kubernetes.io/name": nlbGuestbookAppName,
- },
- Type: v1.ServiceTypeLoadBalancer,
- Ports: []v1.ServicePort{
- {
- Protocol: v1.ProtocolTCP,
- Port: 80,
- TargetPort: intstr.FromString("http-server"),
- },
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create NLB guestbook Service (%v)", err)
- }
- ts.cfg.Logger.Info("created NLB guestbook Service")
-
- waitDur := 3 * time.Minute
- ts.cfg.Logger.Info("waiting for NLB guestbook Service", zap.Duration("wait", waitDur))
- select {
- case <-ts.cfg.Stopc:
- return errors.New("NLB guestbook Service creation aborted")
- case <-time.After(waitDur):
- }
-
- args := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace,
- "describe",
- "svc",
- nlbGuestbookServiceName,
- }
- argsCmd := strings.Join(args, " ")
-
- hostName := ""
- retryStart := time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("NLB guestbook Service creation aborted")
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- cmdOut, err := exec.New().CommandContext(ctx, args[0], args[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe svc' failed", zap.String("command", argsCmd), zap.Error(err))
- } else {
- out := string(cmdOut)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", argsCmd, out)
- }
-
- ts.cfg.Logger.Info("querying NLB guestbook Service for HTTP endpoint")
- ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
- so, err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace).
- Get(ctx, nlbGuestbookServiceName, metav1.GetOptions{})
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("failed to get NLB guestbook Service; retrying", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
-
- ts.cfg.Logger.Info(
- "NLB guestbook Service has been linked to LoadBalancer",
- zap.String("load-balancer", fmt.Sprintf("%+v", so.Status.LoadBalancer)),
- )
- for _, ing := range so.Status.LoadBalancer.Ingress {
- ts.cfg.Logger.Info(
- "NLB guestbook Service has been linked to LoadBalancer.Ingress",
- zap.String("ingress", fmt.Sprintf("%+v", ing)),
- )
- hostName = ing.Hostname
- break
- }
-
- if hostName != "" {
- ts.cfg.Logger.Info("found NLB host name", zap.String("host-name", hostName))
- break
- }
- }
-
- if hostName == "" {
- return errors.New("failed to find NLB host name")
- }
-
- // TODO: is there any better way to find out the NLB name?
- ts.cfg.EKSConfig.AddOnNLBGuestbook.NLBName = strings.Split(hostName, "-")[0]
- ss := strings.Split(hostName, ".")[0]
- ss = strings.Replace(ss, "-", "/", -1)
- ts.cfg.EKSConfig.AddOnNLBGuestbook.NLBARN = fmt.Sprintf(
- "arn:aws:elasticloadbalancing:%s:%s:loadbalancer/net/%s",
- ts.cfg.EKSConfig.Region,
- ts.cfg.EKSConfig.Status.AWSAccountID,
- ss,
- )
- ts.cfg.EKSConfig.AddOnNLBGuestbook.URL = "http://" + hostName
- ts.cfg.EKSConfig.Sync()
-
- fmt.Fprintf(ts.cfg.LogWriter, "\nNLB guestbook ARN: %s\n", ts.cfg.EKSConfig.AddOnNLBGuestbook.NLBARN)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB guestbook Name: %s\n", ts.cfg.EKSConfig.AddOnNLBGuestbook.NLBName)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB guestbook URL: %s\n\n", ts.cfg.EKSConfig.AddOnNLBGuestbook.URL)
-
- ts.cfg.Logger.Info("waiting before testing guestbook Service")
- time.Sleep(20 * time.Second)
-
- htmlChecked := false
- retryStart = time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("guestbook Service creation aborted")
- case <-time.After(5 * time.Second):
- }
-
- out, err := httputil.ReadInsecure(ts.cfg.Logger, ioutil.Discard, ts.cfg.EKSConfig.AddOnNLBGuestbook.URL)
- if err != nil {
- ts.cfg.Logger.Warn("failed to read NLB guestbook Service; retrying", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
- httpOutput := string(out)
- fmt.Fprintf(ts.cfg.LogWriter, "\nNLB guestbook Service output:\n%s\n", httpOutput)
-
- if strings.Contains(httpOutput, `Guestbook
`) {
- ts.cfg.Logger.Info("read guestbook Service; exiting", zap.String("host-name", hostName))
- htmlChecked = true
- break
- }
-
- ts.cfg.Logger.Warn("unexpected guestbook Service output; retrying")
- }
-
- fmt.Fprintf(ts.cfg.LogWriter, "\nNLB guestbook ARN: %s\n", ts.cfg.EKSConfig.AddOnNLBGuestbook.NLBARN)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB guestbook Name: %s\n", ts.cfg.EKSConfig.AddOnNLBGuestbook.NLBName)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB guestbook URL: %s\n\n", ts.cfg.EKSConfig.AddOnNLBGuestbook.URL)
-
- if !htmlChecked {
- return fmt.Errorf("NLB hello-world %q did not return expected HTML output", ts.cfg.EKSConfig.AddOnNLBGuestbook.URL)
- }
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) deleteServiceGuestbook() error {
- ts.cfg.Logger.Info("deleting NLB guestbook Service")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(ts.cfg.EKSConfig.AddOnNLBGuestbook.Namespace).
- Delete(
- ctx,
- nlbGuestbookServiceName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete NLB guestbook Service (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted NLB guestbook Service", zap.Error(err))
- ts.cfg.EKSConfig.Sync()
- return nil
-}
diff --git a/eks/nlb-hello-world/nlb-hello-world.go b/eks/nlb-hello-world/nlb-hello-world.go
deleted file mode 100644
index 86edcfaa6..000000000
--- a/eks/nlb-hello-world/nlb-hello-world.go
+++ /dev/null
@@ -1,507 +0,0 @@
-// Package nlbhelloworld implements NLB plugin
-// with a simple hello world service.
-package nlbhelloworld
-
-import (
- "context"
- "errors"
- "fmt"
- "io"
- "io/ioutil"
- "reflect"
- "strings"
- "time"
-
- eks_tester "github.com/aws/aws-k8s-tester/eks/tester"
- "github.com/aws/aws-k8s-tester/eksconfig"
- "github.com/aws/aws-k8s-tester/pkg/aws/elb"
- "github.com/aws/aws-k8s-tester/pkg/httputil"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface"
- "go.uber.org/zap"
- appsv1 "k8s.io/api/apps/v1"
- v1 "k8s.io/api/core/v1"
- apierrs "k8s.io/apimachinery/pkg/api/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/util/intstr"
- "k8s.io/utils/exec"
-)
-
-// Config defines NLB configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
- ELB2API elbv2iface.ELBV2API
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-// New creates a new Job tester.
-func New(cfg Config) eks_tester.Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
-}
-
-const (
- nlbHelloWorldDeploymentName = "hello-world-deployment"
- nlbHelloWorldAppName = "hello-world"
- nlbHelloWorldAppImageName = "dockercloud/hello-world"
- nlbHelloWorldServiceName = "hello-world-service"
-)
-
-func (ts *tester) Create() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnNLBHelloWorld() {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
- if ts.cfg.EKSConfig.AddOnNLBHelloWorld.Created {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Create", zap.String("tester", pkgName))
- ts.cfg.EKSConfig.AddOnNLBHelloWorld.Created = true
- ts.cfg.EKSConfig.Sync()
- createStart := time.Now()
- defer func() {
- createEnd := time.Now()
- ts.cfg.EKSConfig.AddOnNLBHelloWorld.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- if err := k8s_client.CreateNamespace(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnNLBHelloWorld.Namespace,
- ); err != nil {
- return err
- }
- if err := ts.createDeployment(); err != nil {
- return err
- }
- if err := ts.waitDeployment(); err != nil {
- return err
- }
- if err := ts.createService(); err != nil {
- return err
- }
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) Delete() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnNLBHelloWorld() {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnNLBHelloWorld.Created {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Delete", zap.String("tester", pkgName))
- deleteStart := time.Now()
- defer func() {
- deleteEnd := time.Now()
- ts.cfg.EKSConfig.AddOnNLBHelloWorld.TimeFrameDelete = timeutil.NewTimeFrame(deleteStart, deleteEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- var errs []string
-
- if err := ts.deleteService(); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete NLB hello-world Service (%v)", err))
- }
- ts.cfg.Logger.Info("wait for a minute after deleting Service")
- time.Sleep(time.Minute)
-
- if err := ts.deleteDeployment(); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete NLB hello-world Deployment (%v)", err))
- }
- ts.cfg.Logger.Info("wait for a minute after deleting Deployment")
- time.Sleep(time.Minute)
-
- /*
- # NLB tags
- kubernetes.io/service-name
- leegyuho-test-prod-nlb-hello-world/hello-world-service
-
- kubernetes.io/cluster/leegyuho-test-prod
- owned
- */
- if err := elb.DeleteELBv2(
- ts.cfg.Logger,
- ts.cfg.ELB2API,
- ts.cfg.EKSConfig.AddOnNLBHelloWorld.NLBARN,
- ts.cfg.EKSConfig.VPC.ID,
- map[string]string{
- "kubernetes.io/cluster/" + ts.cfg.EKSConfig.Name: "owned",
- "kubernetes.io/service-name": ts.cfg.EKSConfig.AddOnNLBHelloWorld.Namespace + "/" + nlbHelloWorldServiceName,
- },
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete NLB hello-world (%v)", err))
- }
-
- if err := k8s_client.DeleteNamespaceAndWait(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnNLBHelloWorld.Namespace,
- k8s_client.DefaultNamespaceDeletionInterval,
- k8s_client.DefaultNamespaceDeletionTimeout,
- k8s_client.WithForceDelete(true),
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete NLB namespace (%v)", err))
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- ts.cfg.EKSConfig.AddOnNLBHelloWorld.Created = false
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) createDeployment() error {
- var nodeSelector map[string]string
- if len(ts.cfg.EKSConfig.AddOnNLBHelloWorld.DeploymentNodeSelector) > 0 {
- nodeSelector = ts.cfg.EKSConfig.AddOnNLBHelloWorld.DeploymentNodeSelector
- } else {
- nodeSelector = nil
- }
- ts.cfg.Logger.Info("creating NLB hello-world Deployment", zap.Any("node-selector", nodeSelector))
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.K8SClient.KubernetesClientSet().
- AppsV1().
- Deployments(ts.cfg.EKSConfig.AddOnNLBHelloWorld.Namespace).
- Create(
- ctx,
- &appsv1.Deployment{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "apps/v1",
- Kind: "Deployment",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: nlbHelloWorldDeploymentName,
- Namespace: ts.cfg.EKSConfig.AddOnNLBHelloWorld.Namespace,
- Labels: map[string]string{
- "app.kubernetes.io/name": nlbHelloWorldAppName,
- },
- },
- Spec: appsv1.DeploymentSpec{
- Replicas: aws.Int32(ts.cfg.EKSConfig.AddOnNLBHelloWorld.DeploymentReplicas),
- Selector: &metav1.LabelSelector{
- MatchLabels: map[string]string{
- "app.kubernetes.io/name": nlbHelloWorldAppName,
- },
- },
- Template: v1.PodTemplateSpec{
- ObjectMeta: metav1.ObjectMeta{
- Labels: map[string]string{
- "app.kubernetes.io/name": nlbHelloWorldAppName,
- },
- },
- Spec: v1.PodSpec{
- RestartPolicy: v1.RestartPolicyAlways,
- Containers: []v1.Container{
- {
- Name: nlbHelloWorldAppName,
- Image: nlbHelloWorldAppImageName,
- ImagePullPolicy: v1.PullAlways,
- Ports: []v1.ContainerPort{
- {
- Protocol: v1.ProtocolTCP,
- ContainerPort: 80,
- },
- },
- },
- },
- NodeSelector: nodeSelector,
- },
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create NLB hello-world Deployment (%v)", err)
- }
-
- ts.cfg.Logger.Info("created NLB hello-world Deployment")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) deleteDeployment() error {
- ts.cfg.Logger.Info("deleting NLB hello-world Deployment")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- AppsV1().
- Deployments(ts.cfg.EKSConfig.AddOnNLBHelloWorld.Namespace).
- Delete(
- ctx,
- nlbHelloWorldDeploymentName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete NLB hello-world Deployment (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted NLB hello-world Deployment")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) waitDeployment() (err error) {
- timeout := 7*time.Minute + time.Duration(ts.cfg.EKSConfig.AddOnNLBHelloWorld.DeploymentReplicas)*time.Minute
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- _, err = k8s_client.WaitForDeploymentCompletes(
- ctx,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.Stopc,
- ts.cfg.K8SClient,
- time.Minute,
- 20*time.Second,
- ts.cfg.EKSConfig.AddOnNLBHelloWorld.Namespace,
- nlbHelloWorldDeploymentName,
- ts.cfg.EKSConfig.AddOnNLBHelloWorld.DeploymentReplicas,
- k8s_client.WithQueryFunc(func() {
- descArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnNLBHelloWorld.Namespace,
- "describe",
- "deployment",
- nlbHelloWorldDeploymentName,
- }
- descCmd := strings.Join(descArgs, " ")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe deployment' failed", zap.Error(err))
- }
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", descCmd, out)
- }),
- )
- cancel()
- return err
-}
-
-func (ts *tester) createService() error {
- ts.cfg.Logger.Info("creating NLB hello-world Service")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(ts.cfg.EKSConfig.AddOnNLBHelloWorld.Namespace).
- Create(
- ctx,
- &v1.Service{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "v1",
- Kind: "Service",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: nlbHelloWorldServiceName,
- Namespace: ts.cfg.EKSConfig.AddOnNLBHelloWorld.Namespace,
- Annotations: map[string]string{
- "service.beta.kubernetes.io/aws-load-balancer-type": "nlb",
- },
- },
- Spec: v1.ServiceSpec{
- Selector: map[string]string{
- "app.kubernetes.io/name": nlbHelloWorldAppName,
- },
- Type: v1.ServiceTypeLoadBalancer,
- Ports: []v1.ServicePort{
- {
- Protocol: v1.ProtocolTCP,
- Port: 80,
- TargetPort: intstr.FromInt(80),
- },
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create NLB hello-world Service (%v)", err)
- }
- ts.cfg.Logger.Info("created NLB hello-world Service")
-
- waitDur := 3 * time.Minute
- ts.cfg.Logger.Info("waiting for NLB hello-world Service", zap.Duration("wait", waitDur))
- select {
- case <-ts.cfg.Stopc:
- return errors.New("NLB hello-world Service creation aborted")
- case <-time.After(waitDur):
- }
-
- args := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnNLBHelloWorld.Namespace,
- "describe",
- "svc",
- nlbHelloWorldServiceName,
- }
- argsCmd := strings.Join(args, " ")
- hostName := ""
- retryStart := time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("NLB hello-world Service creation aborted")
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- cmdOut, err := exec.New().CommandContext(ctx, args[0], args[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe svc' failed", zap.String("command", argsCmd), zap.Error(err))
- } else {
- out := string(cmdOut)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", argsCmd, out)
- }
-
- ts.cfg.Logger.Info("querying NLB hello-world Service for HTTP endpoint")
- ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
- so, err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(ts.cfg.EKSConfig.AddOnNLBHelloWorld.Namespace).
- Get(ctx, nlbHelloWorldServiceName, metav1.GetOptions{})
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("failed to get NLB hello-world Service; retrying", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
-
- ts.cfg.Logger.Info(
- "NLB hello-world Service has been linked to LoadBalancer",
- zap.String("load-balancer", fmt.Sprintf("%+v", so.Status.LoadBalancer)),
- )
- for _, ing := range so.Status.LoadBalancer.Ingress {
- ts.cfg.Logger.Info(
- "NLB hello-world Service has been linked to LoadBalancer.Ingress",
- zap.String("ingress", fmt.Sprintf("%+v", ing)),
- )
- hostName = ing.Hostname
- break
- }
-
- if hostName != "" {
- ts.cfg.Logger.Info("found NLB host name", zap.String("host-name", hostName))
- break
- }
- }
-
- if hostName == "" {
- return errors.New("failed to find NLB host name")
- }
-
- // TODO: is there any better way to find out the NLB name?
- ts.cfg.EKSConfig.AddOnNLBHelloWorld.NLBName = strings.Split(hostName, "-")[0]
- ss := strings.Split(hostName, ".")[0]
- ss = strings.Replace(ss, "-", "/", -1)
- ts.cfg.EKSConfig.AddOnNLBHelloWorld.NLBARN = fmt.Sprintf(
- "arn:aws:elasticloadbalancing:%s:%s:loadbalancer/net/%s",
- ts.cfg.EKSConfig.Region,
- ts.cfg.EKSConfig.Status.AWSAccountID,
- ss,
- )
- ts.cfg.EKSConfig.AddOnNLBHelloWorld.URL = "http://" + hostName
- ts.cfg.EKSConfig.Sync()
-
- fmt.Fprintf(ts.cfg.LogWriter, "\nNLB hello-world ARN: %s\n", ts.cfg.EKSConfig.AddOnNLBHelloWorld.NLBARN)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB hello-world Name: %s\n", ts.cfg.EKSConfig.AddOnNLBHelloWorld.NLBName)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB hello-world URL: %s\n\n", ts.cfg.EKSConfig.AddOnNLBHelloWorld.URL)
-
- ts.cfg.Logger.Info("waiting before testing hello-world Service")
- time.Sleep(20 * time.Second)
-
- htmlChecked := false
- retryStart = time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("hello-world Service creation aborted")
- case <-time.After(5 * time.Second):
- }
-
- out, err := httputil.ReadInsecure(ts.cfg.Logger, ioutil.Discard, ts.cfg.EKSConfig.AddOnNLBHelloWorld.URL)
- if err != nil {
- ts.cfg.Logger.Warn("failed to read NLB hello-world Service; retrying", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
- httpOutput := string(out)
- fmt.Fprintf(ts.cfg.LogWriter, "\nNLB hello-world Service output:\n%s\n", httpOutput)
-
- if strings.Contains(httpOutput, `Hello world!
`) {
- ts.cfg.Logger.Info("read hello-world Service; exiting", zap.String("host-name", hostName))
- htmlChecked = true
- break
- }
-
- ts.cfg.Logger.Warn("unexpected hello-world Service output; retrying")
- }
-
- fmt.Fprintf(ts.cfg.LogWriter, "\nNLB hello-world ARN: %s\n", ts.cfg.EKSConfig.AddOnNLBHelloWorld.NLBARN)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB hello-world Name: %s\n", ts.cfg.EKSConfig.AddOnNLBHelloWorld.NLBName)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB hello-world URL: %s\n\n", ts.cfg.EKSConfig.AddOnNLBHelloWorld.URL)
-
- if !htmlChecked {
- return fmt.Errorf("NLB hello-world %q did not return expected HTML output", ts.cfg.EKSConfig.AddOnNLBHelloWorld.URL)
- }
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) deleteService() error {
- ts.cfg.Logger.Info("deleting NLB hello-world Service")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(ts.cfg.EKSConfig.AddOnNLBHelloWorld.Namespace).
- Delete(
- ctx,
- nlbHelloWorldServiceName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete NLB hello-world Service (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted NLB hello-world Service", zap.Error(err))
- ts.cfg.EKSConfig.Sync()
- return nil
-}
diff --git a/eks/overprovisioning/overprovisioning.go b/eks/overprovisioning/overprovisioning.go
deleted file mode 100644
index fe1e76065..000000000
--- a/eks/overprovisioning/overprovisioning.go
+++ /dev/null
@@ -1,56 +0,0 @@
-package overprovisioning
-
-import (
- "fmt"
-
- "github.com/aws/aws-k8s-tester/eksconfig"
- k8sclient "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- gotemplate "github.com/aws/aws-k8s-tester/pkg/util"
-)
-
-// Overprovisioning is an addon that installs extra pods to the cluster
-type Overprovisioning struct {
- K8sClient k8sclient.EKS
- Config *eksconfig.Config
-}
-
-// IsEnabled returns true if enabled
-func (c *Overprovisioning) IsEnabled() bool {
- return c.Config.Spec.Overprovisioning != nil
-}
-
-// Apply installs the addon
-func (c *Overprovisioning) Apply() (err error) {
- template, err := gotemplate.FromLocalDirectory(c.Config.Spec.Overprovisioning)
- if err != nil {
- return fmt.Errorf("while building templates, %v", err)
- }
- if err := c.K8sClient.Apply(template.String()); err != nil {
- return fmt.Errorf("while applying resources, %v", err)
- }
- c.Config.Status.Overprovisioning = &eksconfig.OverprovisioningStatus{
- AddonStatus: eksconfig.AddonStatus{
- Installed: true,
- Ready: true,
- },
- }
- return nil
-}
-
-// Delete removes the addon
-func (c *Overprovisioning) Delete() (err error) {
- template, err := gotemplate.FromLocalDirectory(c.Config.Spec.Overprovisioning)
- if err != nil {
- return fmt.Errorf("while building templates, %v", err)
- }
- if err := c.K8sClient.Delete(template.String()); err != nil {
- return fmt.Errorf("while deleting resources, %v", err)
- }
- c.Config.Status.Overprovisioning = &eksconfig.OverprovisioningStatus{
- AddonStatus: eksconfig.AddonStatus{
- Installed: false,
- Ready: false,
- },
- }
- return nil
-}
diff --git a/eks/overprovisioning/overprovisioning.gotmpl b/eks/overprovisioning/overprovisioning.gotmpl
deleted file mode 100644
index 133c75ae6..000000000
--- a/eks/overprovisioning/overprovisioning.gotmpl
+++ /dev/null
@@ -1,54 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
- name: {{.Namespace}}
-
----
-apiVersion: scheduling.k8s.io/v1
-description: Priority class used by cluster autoscaler for overprovisioning.
-kind: PriorityClass
-metadata:
- name: {{.Namespace}}
-value: -1
-
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: overprovisioning
- namespace: {{.Namespace}}
-spec:
- replicas: {{.Replicas}}
- selector:
- matchLabels:
- app: overprovisioning
- template:
- metadata:
- labels:
- app: overprovisioning
- spec:
- containers:
- - image: {{.Image}}
- name: overprovisioning
- resources:
- requests:
- cpu: {{.Resources.Requests.Cpu}}
- memory: {{.Resources.Requests.Memory}}
- priorityClassName: overprovisioning
- {{if .KubemarkEnabled}}
- affinity:
- nodeAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- nodeSelectorTerms:
- - matchExpressions:
- - key: NodeType
- operator: In
- values:
- - hollow-nodes
- tolerations:
- - effect: NoSchedule
- key: provider
- operator: Equal
- value: kubemark
- {{end}}
\ No newline at end of file
diff --git a/eks/php-apache/php-apache.go b/eks/php-apache/php-apache.go
deleted file mode 100644
index 25a5f11e4..000000000
--- a/eks/php-apache/php-apache.go
+++ /dev/null
@@ -1,287 +0,0 @@
-// Package phpapache implements PHP Apache
-// with a simple PHP app.
-package phpapache
-
-import (
- "context"
- "errors"
- "fmt"
- "io"
- "reflect"
- "strings"
- "time"
-
- eks_tester "github.com/aws/aws-k8s-tester/eks/tester"
- "github.com/aws/aws-k8s-tester/eksconfig"
- aws_ecr "github.com/aws/aws-k8s-tester/pkg/aws/ecr"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/service/ecr/ecriface"
- "go.uber.org/zap"
- appsv1 "k8s.io/api/apps/v1"
- v1 "k8s.io/api/core/v1"
- apierrs "k8s.io/apimachinery/pkg/api/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/utils/exec"
-)
-
-// Config defines configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
- ECRAPI ecriface.ECRAPI
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-// New creates a new Job tester.
-func New(cfg Config) eks_tester.Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg, phpApacheImg: phpApacheAppImageName}
-}
-
-type tester struct {
- cfg Config
-
- phpApacheImg string
-}
-
-const (
- phpApacheDeploymentName = "php-apache-deployment"
- phpApacheAppName = "php-apache"
- phpApacheAppImageName = "pjlewis/php-apache"
-)
-
-func (ts *tester) Create() (err error) {
- if !ts.cfg.EKSConfig.IsEnabledAddOnPHPApache() {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
- if ts.cfg.EKSConfig.AddOnPHPApache.Created {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Create", zap.String("tester", pkgName))
- ts.cfg.EKSConfig.AddOnPHPApache.Created = true
- ts.cfg.EKSConfig.Sync()
- createStart := time.Now()
- defer func() {
- createEnd := time.Now()
- ts.cfg.EKSConfig.AddOnPHPApache.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- if ts.cfg.EKSConfig.AddOnPHPApache.RepositoryAccountID != "" &&
- ts.cfg.EKSConfig.AddOnPHPApache.RepositoryRegion != "" &&
- ts.cfg.EKSConfig.AddOnPHPApache.RepositoryName != "" &&
- ts.cfg.EKSConfig.AddOnPHPApache.RepositoryImageTag != "" {
- if ts.phpApacheImg, _, err = aws_ecr.Check(
- ts.cfg.Logger,
- ts.cfg.ECRAPI,
- ts.cfg.EKSConfig.Partition,
- ts.cfg.EKSConfig.AddOnPHPApache.RepositoryAccountID,
- ts.cfg.EKSConfig.AddOnPHPApache.RepositoryRegion,
- ts.cfg.EKSConfig.AddOnPHPApache.RepositoryName,
- ts.cfg.EKSConfig.AddOnPHPApache.RepositoryImageTag,
- ); err != nil {
- return err
- }
- }
- if err := k8s_client.CreateNamespace(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnPHPApache.Namespace,
- ); err != nil {
- return err
- }
- if err := ts.createDeployment(); err != nil {
- return err
- }
- if err := ts.waitDeployment(); err != nil {
- return err
- }
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) Delete() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnPHPApache() {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnPHPApache.Created {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Delete", zap.String("tester", pkgName))
- deleteStart := time.Now()
- defer func() {
- deleteEnd := time.Now()
- ts.cfg.EKSConfig.AddOnPHPApache.TimeFrameDelete = timeutil.NewTimeFrame(deleteStart, deleteEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- var errs []string
-
- if err := ts.deleteDeployment(); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete php-apache Deployment (%v)", err))
- }
- ts.cfg.Logger.Info("wait for a minute after deleting Deployment")
- time.Sleep(time.Minute)
-
- if err := k8s_client.DeleteNamespaceAndWait(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnPHPApache.Namespace,
- k8s_client.DefaultNamespaceDeletionInterval,
- k8s_client.DefaultNamespaceDeletionTimeout,
- k8s_client.WithForceDelete(true),
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete namespace (%v)", err))
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- ts.cfg.EKSConfig.AddOnPHPApache.Created = false
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) createDeployment() error {
- var nodeSelector map[string]string
- if len(ts.cfg.EKSConfig.AddOnPHPApache.DeploymentNodeSelector) > 0 {
- nodeSelector = ts.cfg.EKSConfig.AddOnPHPApache.DeploymentNodeSelector
- } else {
- nodeSelector = nil
- }
- ts.cfg.Logger.Info("creating php-apache Deployment", zap.Any("node-selector", nodeSelector))
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.K8SClient.KubernetesClientSet().
- AppsV1().
- Deployments(ts.cfg.EKSConfig.AddOnPHPApache.Namespace).
- Create(
- ctx,
- &appsv1.Deployment{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "apps/v1",
- Kind: "Deployment",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: phpApacheDeploymentName,
- Namespace: ts.cfg.EKSConfig.AddOnPHPApache.Namespace,
- Labels: map[string]string{
- "app.kubernetes.io/name": phpApacheAppName,
- },
- },
- Spec: appsv1.DeploymentSpec{
- Replicas: aws.Int32(ts.cfg.EKSConfig.AddOnPHPApache.DeploymentReplicas),
- Selector: &metav1.LabelSelector{
- MatchLabels: map[string]string{
- "app.kubernetes.io/name": phpApacheAppName,
- },
- },
- Template: v1.PodTemplateSpec{
- ObjectMeta: metav1.ObjectMeta{
- Labels: map[string]string{
- "app.kubernetes.io/name": phpApacheAppName,
- },
- },
- Spec: v1.PodSpec{
- RestartPolicy: v1.RestartPolicyAlways,
- Containers: []v1.Container{
- {
- Name: phpApacheAppName,
- Image: ts.phpApacheImg,
- ImagePullPolicy: v1.PullAlways,
- },
- },
- NodeSelector: nodeSelector,
- },
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create php-apache Deployment (%v)", err)
- }
-
- ts.cfg.Logger.Info("created php-apache Deployment")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) deleteDeployment() error {
- ts.cfg.Logger.Info("deleting php-apache Deployment")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- AppsV1().
- Deployments(ts.cfg.EKSConfig.AddOnPHPApache.Namespace).
- Delete(
- ctx,
- phpApacheDeploymentName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete php-apache Deployment (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted php-apache Deployment")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) waitDeployment() (err error) {
- timeout := 7*time.Minute + time.Duration(ts.cfg.EKSConfig.AddOnPHPApache.DeploymentReplicas)*time.Minute
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- _, err = k8s_client.WaitForDeploymentCompletes(
- ctx,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.Stopc,
- ts.cfg.K8SClient,
- time.Minute,
- 20*time.Second,
- ts.cfg.EKSConfig.AddOnPHPApache.Namespace,
- phpApacheDeploymentName,
- ts.cfg.EKSConfig.AddOnPHPApache.DeploymentReplicas,
- k8s_client.WithQueryFunc(func() {
- descArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnPHPApache.Namespace,
- "describe",
- "deployment",
- phpApacheDeploymentName,
- }
- descCmd := strings.Join(descArgs, " ")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe deployment' failed", zap.Error(err))
- }
- out := string(output)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", descCmd, out)
- }),
- )
- cancel()
- return err
-}
diff --git a/eks/prometheus-grafana/prometheus-grafana.go b/eks/prometheus-grafana/prometheus-grafana.go
deleted file mode 100644
index 0ff1e4cc3..000000000
--- a/eks/prometheus-grafana/prometheus-grafana.go
+++ /dev/null
@@ -1,572 +0,0 @@
-// Package prometheusgrafana implements Prometheus/Grafana add-on.
-package prometheusgrafana
-
-import (
- "context"
- "errors"
- "fmt"
- "io"
- "io/ioutil"
- "reflect"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/eks/helm"
- eks_tester "github.com/aws/aws-k8s-tester/eks/tester"
- "github.com/aws/aws-k8s-tester/eksconfig"
- "github.com/aws/aws-k8s-tester/pkg/aws/elb"
- "github.com/aws/aws-k8s-tester/pkg/httputil"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface"
- "go.uber.org/zap"
- apierrs "k8s.io/apimachinery/pkg/api/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/utils/exec"
-)
-
-// Config defines Wordpress configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
- ELB2API elbv2iface.ELBV2API
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-func New(cfg Config) eks_tester.Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
-}
-
-const (
- chartRepoName = "stable"
- chartRepoURL = "https://kubernetes-charts.storage.googleapis.com"
-
- chartNamespacePrometheus = "prometheus"
- chartNamespaceGrafana = "grafana"
-
- chartNamePrometheus = "prometheus"
- chartNameGrafana = "grafana"
-)
-
-func (ts *tester) Create() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnPrometheusGrafana() {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
- if ts.cfg.EKSConfig.AddOnPrometheusGrafana.Created {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Create", zap.String("tester", pkgName))
- ts.cfg.EKSConfig.AddOnPrometheusGrafana.Created = true
- ts.cfg.EKSConfig.Sync()
- createStart := time.Now()
- defer func() {
- createEnd := time.Now()
- ts.cfg.EKSConfig.AddOnPrometheusGrafana.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- if err := k8s_client.CreateNamespace(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- chartNamespacePrometheus); err != nil {
- return err
- }
- if err := k8s_client.CreateNamespace(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- chartNamespaceGrafana); err != nil {
- return err
- }
- if err := helm.RepoAdd(ts.cfg.Logger, chartRepoName, chartRepoURL); err != nil {
- return err
- }
- if err := ts.createHelmPrometheus(); err != nil {
- return err
- }
- if err := ts.createHelmGrafana(); err != nil {
- return err
- }
- if err := ts.waitServiceGrafana(); err != nil {
- return err
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) Delete() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnPrometheusGrafana() {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnPrometheusGrafana.Created {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Delete", zap.String("tester", pkgName))
- deleteStart := time.Now()
- defer func() {
- deleteEnd := time.Now()
- ts.cfg.EKSConfig.AddOnPrometheusGrafana.TimeFrameDelete = timeutil.NewTimeFrame(deleteStart, deleteEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- var errs []string
-
- if err := ts.deleteHelmGrafana(); err != nil {
- errs = append(errs, err.Error())
- }
-
- time.Sleep(15 * time.Second)
-
- if err := ts.deleteHelmPrometheus(); err != nil {
- errs = append(errs, err.Error())
- }
-
- if err := ts.deleteGrafanaService(); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete Grafana Service (%v)", err))
- }
- ts.cfg.Logger.Info("wait for 3-minute after deleting Service")
- time.Sleep(3 * time.Minute)
-
- /*
- # NLB tags
- kubernetes.io/service-name
- leegyuho-test-prod-nlb-hello-world/hello-world-service
-
- kubernetes.io/cluster/leegyuho-test-prod
- owned
- */
- if err := elb.DeleteELBv2(
- ts.cfg.Logger,
- ts.cfg.ELB2API,
- ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaNLBARN,
- ts.cfg.EKSConfig.VPC.ID,
- map[string]string{
- "kubernetes.io/cluster/" + ts.cfg.EKSConfig.Name: "owned",
- "kubernetes.io/service-name": "grafana/" + grafanaServiceName,
- },
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete Grafana (%v)", err))
- }
-
- if err := k8s_client.DeleteNamespaceAndWait(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- chartNamespaceGrafana,
- k8s_client.DefaultNamespaceDeletionInterval,
- k8s_client.DefaultNamespaceDeletionTimeout,
- k8s_client.WithForceDelete(true),
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete Grafana namespace (%v)", err))
- }
-
- if err := k8s_client.DeleteNamespaceAndWait(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- chartNamespacePrometheus,
- k8s_client.DefaultNamespaceDeletionInterval,
- k8s_client.DefaultNamespaceDeletionTimeout,
- k8s_client.WithForceDelete(true),
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete Prometheus namespace (%v)", err))
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- ts.cfg.EKSConfig.AddOnPrometheusGrafana.Created = false
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// https://eksworkshop.com/intermediate/240_monitoring/deploy-prometheus
-// https://github.com/helm/charts/blob/master/stable/prometheus/values.yaml
-func (ts *tester) createHelmPrometheus() error {
- ngType := "custom"
- if ts.cfg.EKSConfig.IsEnabledAddOnManagedNodeGroups() {
- ngType = "managed"
- }
-
- // https://github.com/helm/charts/blob/master/stable/prometheus/values.yaml
- values := map[string]interface{}{
- "alertmanager": map[string]interface{}{
- "nodeSelector": map[string]interface{}{
- // do not deploy in bottlerocket; PVC not working
- "AMIType": ec2config.AMITypeAL2X8664,
- "NGType": ngType,
- },
- "persistentVolume": map[string]interface{}{
- // takes >=5-min for PVC, user emptyDir for testing
- "enabled": false,
- },
- },
- "server": map[string]interface{}{
- "nodeSelector": map[string]interface{}{
- // do not deploy in bottlerocket; PVC not working
- "AMIType": ec2config.AMITypeAL2X8664,
- "NGType": ngType,
- },
- "persistentVolume": map[string]interface{}{
- "enabled": true,
- // use CSI driver with volume type "gp2", as in launch configuration
- "storageClass": "gp2",
- },
- },
- }
-
- descArgsSvc := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + chartNamespaceGrafana,
- "describe",
- "service/grafana",
- }
- descCmdSvc := strings.Join(descArgsSvc, " ")
-
- return helm.Install(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Stopc: ts.cfg.Stopc,
- Timeout: 15 * time.Minute,
- KubeConfigPath: ts.cfg.EKSConfig.KubeConfigPath,
- Namespace: chartNamespacePrometheus,
- ChartRepoURL: chartRepoURL,
- ChartName: chartNamePrometheus,
- ReleaseName: chartNamePrometheus,
- Values: values,
- QueryFunc: func() {
- fmt.Fprintf(ts.cfg.LogWriter, "\n")
-
- // to catch errors
- // e.g. "Error syncing load balancer: failed to ensure load balancer: TooManyLoadBalancers: Exceeded quota of account 123123"
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, descArgsSvc[0], descArgsSvc[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe service/grafana' failed", zap.Error(err))
- } else {
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmdSvc, out)
- }
- },
- QueryInterval: 30 * time.Second,
- })
-}
-
-func (ts *tester) deleteHelmPrometheus() error {
- return helm.Uninstall(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Timeout: 15 * time.Minute,
- KubeConfigPath: ts.cfg.EKSConfig.KubeConfigPath,
- Namespace: chartNamespacePrometheus,
- ChartName: chartNamePrometheus,
- ReleaseName: chartNamePrometheus,
- })
-}
-
-// https://eksworkshop.com/intermediate/240_monitoring/deploy-grafana
-// https://github.com/helm/charts/blob/master/stable/grafana/values.yaml
-func (ts *tester) createHelmGrafana() error {
- ngType := "custom"
- if ts.cfg.EKSConfig.IsEnabledAddOnManagedNodeGroups() {
- ngType = "managed"
- }
- // https://github.com/helm/charts/blob/master/stable/grafana/values.yaml
- values := map[string]interface{}{
- "adminUser": ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaAdminUserName,
- "adminPassword": ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaAdminPassword,
- "nodeSelector": map[string]interface{}{
- // do not deploy in bottlerocket; PVC not working
- "AMIType": ec2config.AMITypeAL2X8664,
- "NGType": ngType,
- },
- // standard_init_linux.go:211: exec user process caused "exec format error"
- // if set to "1.31.1"
- // ref. https://github.com/helm/charts/pull/23195
- // ref. https://github.com/aws/aws-k8s-tester/issues/131
- // make sure these are default empty in case chart version is <=5.4.0
- // ref. https://github.com/helm/charts/pull/23240
- "initChownData": map[string]interface{}{
- "image": map[string]interface{}{
- "tag": "latest",
- "sha": "",
- },
- },
- "persistence": map[string]interface{}{
- "enabled": true,
- // use CSI driver with volume type "gp2", as in launch configuration
- "storageClass": "gp2",
- },
- "service": map[string]interface{}{
- "type": "LoadBalancer",
- },
- "datasources": map[string]interface{}{
- "datasources.yaml": map[string]interface{}{
- "apiVersion": 1,
- "datasources": []map[string]interface{}{
- {
- "name": "Prometheus",
- "type": "prometheus",
- "url": "http://prometheus-server.prometheus.svc.cluster.local",
- "access": "proxy",
- "isDefault": true,
- },
- },
- },
- },
- "dashboardProviders": map[string]interface{}{
- "dashboardproviders.yaml": map[string]interface{}{
- "apiVersion": 1,
- "providers": []map[string]interface{}{
- {
- "disableDeletion": false,
- "editable": true,
- "folder": "",
- "name": "default",
- "options": map[string]interface{}{
- "path": "/var/lib/grafana/dashboards/default",
- },
- "orgId": 1,
- "type": "file",
- },
- },
- },
- },
- "dashboards": map[string]interface{}{
- "default": map[string]interface{}{
- "kubernetes-cluster": map[string]interface{}{
- "gnetId": 6417,
- "revision": 1,
- "datasource": "Prometheus",
- },
- },
- },
- }
- return helm.Install(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Stopc: ts.cfg.Stopc,
- Timeout: 15 * time.Minute,
- KubeConfigPath: ts.cfg.EKSConfig.KubeConfigPath,
- Namespace: chartNamespaceGrafana,
- ChartRepoURL: chartRepoURL,
- ChartName: chartNameGrafana,
- ReleaseName: chartNameGrafana,
- Values: values,
- QueryFunc: func() {
- getAllArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + chartNamespaceGrafana,
- "get",
- "all",
- }
- getAllCmd := strings.Join(getAllArgs, " ")
-
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, getAllArgs[0], getAllArgs[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl get all' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getAllCmd, out)
- },
- QueryInterval: 30 * time.Second,
- })
-}
-
-func (ts *tester) deleteHelmGrafana() error {
- return helm.Uninstall(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Timeout: 15 * time.Minute,
- KubeConfigPath: ts.cfg.EKSConfig.KubeConfigPath,
- Namespace: chartNamespaceGrafana,
- ChartName: chartNameGrafana,
- ReleaseName: chartNameGrafana,
- })
-}
-
-func (ts *tester) waitServiceGrafana() error {
- ts.cfg.Logger.Info("waiting for Grafana service")
-
- waitDur := 2 * time.Minute
- ts.cfg.Logger.Info("waiting for Grafana service", zap.Duration("wait", waitDur))
- select {
- case <-ts.cfg.Stopc:
- return errors.New("Grafana service creation aborted")
- case <-time.After(waitDur):
- }
-
- args := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + chartNameGrafana,
- "describe",
- "svc",
- grafanaServiceName,
- }
- argsCmd := strings.Join(args, " ")
- hostName := ""
- retryStart := time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("Grafana service creation aborted")
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- cmdOut, err := exec.New().CommandContext(ctx, args[0], args[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe svc' failed", zap.String("command", argsCmd), zap.Error(err))
- } else {
- out := string(cmdOut)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", argsCmd, out)
- }
-
- ts.cfg.Logger.Info("querying Grafana service for HTTP endpoint")
- ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
- so, err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(chartNameGrafana).
- Get(ctx, grafanaServiceName, metav1.GetOptions{})
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("failed to get Grafana service; retrying", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
-
- ts.cfg.Logger.Info(
- "Grafana service has been linked to LoadBalancer",
- zap.String("load-balancer", fmt.Sprintf("%+v", so.Status.LoadBalancer)),
- )
- for _, ing := range so.Status.LoadBalancer.Ingress {
- ts.cfg.Logger.Info(
- "Grafana service has been linked to LoadBalancer.Ingress",
- zap.String("ingress", fmt.Sprintf("%+v", ing)),
- )
- hostName = ing.Hostname
- break
- }
-
- if hostName != "" {
- ts.cfg.Logger.Info("found host name", zap.String("host-name", hostName))
- break
- }
- }
-
- if hostName == "" {
- return errors.New("failed to find host name")
- }
-
- ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaURL = "http://" + hostName + "/login"
-
- // TODO: is there any better way to find out the NLB name?
- ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaNLBName = strings.Split(hostName, "-")[0]
- ss := strings.Split(hostName, ".")[0]
- ss = strings.Replace(ss, "-", "/", -1)
- ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaNLBARN = fmt.Sprintf(
- "arn:aws:elasticloadbalancing:%s:%s:loadbalancer/net/%s",
- ts.cfg.EKSConfig.Region,
- ts.cfg.EKSConfig.Status.AWSAccountID,
- ss,
- )
-
- fmt.Fprintf(ts.cfg.LogWriter, "\nNLB Grafana ARN: %s\n", ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaNLBARN)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB Grafana Name: %s\n", ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaNLBName)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB Grafana URL: %s\n", ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaURL)
- fmt.Fprintf(ts.cfg.LogWriter, "Grafana Admin User Name: %s\n", ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaAdminUserName)
- fmt.Fprintf(ts.cfg.LogWriter, "Grafana Admin Password: %d characters\n\n", len(ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaAdminPassword))
-
- ts.cfg.Logger.Info("waiting before testing Grafana Service")
- time.Sleep(20 * time.Second)
-
- retryStart = time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("Grafana Service creation aborted")
- case <-time.After(5 * time.Second):
- }
-
- out, err := httputil.ReadInsecure(ts.cfg.Logger, ioutil.Discard, ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaURL)
- if err != nil {
- ts.cfg.Logger.Warn("failed to read NLB Grafana Service; retrying", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
- httpOutput := string(out)
- fmt.Fprintf(ts.cfg.LogWriter, "\nNLB Grafana Service output:\n%s\n", httpOutput)
-
- if strings.Contains(httpOutput, `Loading Grafana`) {
- ts.cfg.Logger.Info(
- "read Grafana Service; exiting",
- zap.String("host-name", hostName),
- )
- break
- }
-
- ts.cfg.Logger.Warn("unexpected Grafana Service output; retrying")
- }
-
- fmt.Fprintf(ts.cfg.LogWriter, "\nNLB Grafana ARN: %s\n", ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaNLBARN)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB Grafana Name: %s\n", ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaNLBName)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB Grafana URL: %s\n", ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaURL)
- fmt.Fprintf(ts.cfg.LogWriter, "Grafana Admin User Name: %s\n", ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaAdminUserName)
- fmt.Fprintf(ts.cfg.LogWriter, "Grafana Admin Password: %d characters\n\n", len(ts.cfg.EKSConfig.AddOnPrometheusGrafana.GrafanaAdminPassword))
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-const grafanaServiceName = "grafana"
-
-func (ts *tester) deleteGrafanaService() error {
- ts.cfg.Logger.Info("deleting grafana Service")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(chartNamespaceGrafana).
- Delete(
- ctx,
- grafanaServiceName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete grafana Service (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted grafana Service", zap.Error(err))
- ts.cfg.EKSConfig.Sync()
- return nil
-}
diff --git a/eks/s3.go b/eks/s3.go
deleted file mode 100644
index 24ebd3c5c..000000000
--- a/eks/s3.go
+++ /dev/null
@@ -1,84 +0,0 @@
-package eks
-
-import (
- "errors"
- "path"
- "path/filepath"
-
- aws_s3 "github.com/aws/aws-k8s-tester/pkg/aws/s3"
- "github.com/aws/aws-k8s-tester/pkg/fileutil"
- "go.uber.org/zap"
-)
-
-func (ts *Tester) createS3() (err error) {
- if ts.cfg.S3.BucketCreate {
- if ts.cfg.S3.BucketName == "" {
- return errors.New("empty S3 bucket name")
- }
- if err = aws_s3.CreateBucket(ts.lg, ts.s3API, ts.cfg.S3.BucketName, ts.cfg.Region, ts.cfg.Name, ts.cfg.S3.BucketLifecycleExpirationDays); err != nil {
- return err
- }
- } else {
- ts.lg.Info("skipping S3 bucket creation")
- }
- if ts.cfg.S3.BucketName == "" {
- ts.lg.Info("skipping s3 bucket creation")
- return nil
- }
- return ts.cfg.Sync()
-}
-
-func (ts *Tester) deleteS3() error {
- if !ts.cfg.S3.BucketCreate {
- ts.lg.Info("skipping S3 bucket deletion", zap.String("s3-bucket-name", ts.cfg.S3.BucketName))
- return nil
- }
- if ts.cfg.S3.BucketCreateKeep {
- ts.lg.Info("skipping S3 bucket deletion", zap.String("s3-bucket-name", ts.cfg.S3.BucketName), zap.Bool("s3-bucket-create-keep", ts.cfg.S3.BucketCreateKeep))
- return nil
- }
- if err := aws_s3.EmptyBucket(ts.lg, ts.s3API, ts.cfg.S3.BucketName); err != nil {
- return err
- }
- return aws_s3.DeleteBucket(ts.lg, ts.s3API, ts.cfg.S3.BucketName)
-}
-
-func (ts *Tester) uploadToS3() (err error) {
- if ts.cfg.S3.BucketName == "" {
- ts.lg.Info("skipping s3 uploads; s3 bucket name is empty")
- return nil
- }
-
- if fileutil.Exist(ts.cfg.ConfigPath) {
- if err = aws_s3.Upload(
- ts.lg,
- ts.s3API,
- ts.cfg.S3.BucketName,
- path.Join(ts.cfg.Name, "aws-k8s-tester-eks.config.yaml"),
- ts.cfg.ConfigPath,
- ); err != nil {
- return err
- }
- }
-
- logFilePath := ""
- for _, fpath := range ts.cfg.LogOutputs {
- if filepath.Ext(fpath) == ".log" {
- logFilePath = fpath
- break
- }
- }
- if fileutil.Exist(logFilePath) {
- if err = aws_s3.Upload(
- ts.lg,
- ts.s3API,
- ts.cfg.S3.BucketName,
- path.Join(ts.cfg.Name, "aws-k8s-tester-eks.log"),
- logFilePath,
- ); err != nil {
- return err
- }
- }
-
- return err
-}
diff --git a/eks/secrets/local/secrets.go b/eks/secrets/local/secrets.go
deleted file mode 100644
index 04dade138..000000000
--- a/eks/secrets/local/secrets.go
+++ /dev/null
@@ -1,503 +0,0 @@
-// Package local implements Secrets plugin.
-package local
-
-import (
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "io/ioutil"
- "path"
- "reflect"
- "sort"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/eks/secrets"
- eks_tester "github.com/aws/aws-k8s-tester/eks/tester"
- "github.com/aws/aws-k8s-tester/eksconfig"
- "github.com/aws/aws-k8s-tester/pkg/aws/cw"
- aws_s3 "github.com/aws/aws-k8s-tester/pkg/aws/s3"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/metrics"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/service/cloudwatch"
- "github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
- "github.com/aws/aws-sdk-go/service/s3"
- "github.com/aws/aws-sdk-go/service/s3/s3iface"
- "go.uber.org/zap"
- apierrs "k8s.io/apimachinery/pkg/api/errors"
-)
-
-// Config defines secrets local tester configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
- S3API s3iface.S3API
- CWAPI cloudwatchiface.CloudWatchAPI
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-func New(cfg Config) eks_tester.Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
-}
-
-func (ts *tester) Create() (err error) {
- if !ts.cfg.EKSConfig.IsEnabledAddOnSecretsLocal() {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
- if ts.cfg.EKSConfig.AddOnSecretsLocal.Created {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Create", zap.String("tester", pkgName))
- ts.cfg.EKSConfig.AddOnSecretsLocal.Created = true
- ts.cfg.EKSConfig.Sync()
- createStart := time.Now()
- defer func() {
- createEnd := time.Now()
- ts.cfg.EKSConfig.AddOnSecretsLocal.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- if err := k8s_client.CreateNamespace(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnSecretsLocal.Namespace,
- ); err != nil {
- return err
- }
-
- loader := secrets.New(secrets.Config{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Stopc: ts.cfg.Stopc,
- S3API: ts.cfg.S3API,
- S3BucketName: ts.cfg.EKSConfig.S3.BucketName,
- Client: ts.cfg.K8SClient,
- ClientTimeout: ts.cfg.EKSConfig.ClientTimeout,
- Namespace: ts.cfg.EKSConfig.AddOnSecretsLocal.Namespace,
- NamePrefix: ts.cfg.EKSConfig.AddOnSecretsLocal.NamePrefix,
- Objects: ts.cfg.EKSConfig.AddOnSecretsLocal.Objects,
- ObjectSize: ts.cfg.EKSConfig.AddOnSecretsLocal.ObjectSize,
- RequestsRawWritesJSONPath: ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawWritesJSONPath,
- RequestsRawWritesJSONS3Key: ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawWritesJSONS3Key,
- RequestsSummaryWritesJSONPath: ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesJSONPath,
- RequestsSummaryWritesJSONS3Key: ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesJSONS3Key,
- RequestsSummaryWritesTablePath: ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesTablePath,
- RequestsSummaryWritesTableS3Key: ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesTableS3Key,
- RequestsRawReadsJSONPath: ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawReadsJSONPath,
- RequestsRawReadsJSONS3Key: ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawReadsJSONS3Key,
- RequestsSummaryReadsJSONPath: ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsJSONPath,
- RequestsSummaryReadsJSONS3Key: ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsJSONS3Key,
- RequestsSummaryReadsTablePath: ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsTablePath,
- RequestsSummaryReadsTableS3Key: ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsTableS3Key,
- })
- loader.Start()
- loader.Stop()
-
- ts.cfg.Logger.Info("completing secrets local tester")
- var curWriteLatencies metrics.Durations
- var curReadLatencies metrics.Durations
- curWriteLatencies, ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWrites, curReadLatencies, ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReads, err = loader.CollectMetrics()
- ts.cfg.EKSConfig.Sync()
- if err != nil {
- ts.cfg.Logger.Warn("failed to get metrics", zap.Error(err))
- return err
- }
-
- if err = ts.checkResults(curWriteLatencies, curReadLatencies); err != nil {
- return err
- }
- if err = ts.publishResults(); err != nil {
- return err
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) Delete() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnSecretsLocal() {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnSecretsLocal.Created {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Delete", zap.String("tester", pkgName))
- deleteStart := time.Now()
- defer func() {
- deleteEnd := time.Now()
- ts.cfg.EKSConfig.AddOnSecretsLocal.TimeFrameDelete = timeutil.NewTimeFrame(deleteStart, deleteEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- var errs []string
-
- if err := k8s_client.DeleteNamespaceAndWait(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnSecretsLocal.Namespace,
- k8s_client.DefaultNamespaceDeletionInterval,
- k8s_client.DefaultNamespaceDeletionTimeout,
- k8s_client.WithForceDelete(true),
- ); err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- errs = append(errs, fmt.Sprintf("failed to delete secrets local tester namespace (%v)", err))
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- ts.cfg.EKSConfig.AddOnSecretsLocal.Created = false
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// 1. if previous summary exists, download and compare
-// 2. upload new summary and overwrite the previous s3 key
-func (ts *tester) checkResults(curWriteLatencies metrics.Durations, curReadLatencies metrics.Durations) (err error) {
- curTS := time.Now().UTC().Format(time.RFC3339Nano)
- ts.cfg.Logger.Info("checking results", zap.String("timestamp", curTS))
-
- s3Objects := make([]*s3.Object, 0)
- if ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesCompareS3Dir != "" {
- s3Objects, err = aws_s3.ListInDescendingLastModified(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Clean(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesCompareS3Dir)+"/",
- )
- }
- canCompare := len(s3Objects) > 0 && err == nil
- if canCompare {
- reqSummaryS3Key := aws.StringValue(s3Objects[0].Key)
- durRawS3Key := path.Join(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawWritesCompareS3Dir, path.Base(reqSummaryS3Key))
-
- var prevSummary metrics.RequestsSummary
- prevSummary, err = metrics.DownloadRequestsSummaryFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, reqSummaryS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesCompare, err = metrics.CompareRequestsSummary(prevSummary, ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWrites)
- if err != nil {
- ts.cfg.Logger.Warn("failed to compare results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesCompareJSONPath, []byte(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesCompare.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesCompareJSONS3Key,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesCompareJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesCompareTablePath, []byte(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesCompare.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesCompareTableS3Key,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesCompareTablePath,
- ); err != nil {
- return err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryWritesCompare:\n%s\n", ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesCompare.Table())
-
- var prevDurations metrics.Durations
- prevDurations, err = metrics.DownloadDurationsFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, durRawS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- prevDurationsWithLabels := metrics.LabelDurations(prevDurations, prevSummary.TestID)
- curDurationsWithLabels := metrics.LabelDurations(curWriteLatencies, ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWrites.TestID)
- allDurationsWithLabels := append(prevDurationsWithLabels, curDurationsWithLabels...)
- sortStart := time.Now()
- ts.cfg.Logger.Info("sorting before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- )
- sort.Sort(allDurationsWithLabels)
- ts.cfg.Logger.Info("sorted before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- zap.String("took", time.Since(sortStart).String()),
- )
- allDataJSON, err := json.Marshal(allDurationsWithLabels)
- if err != nil {
- ts.cfg.Logger.Warn("failed to marshal results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawWritesCompareAllJSONPath, []byte(allDataJSON), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawWritesCompareAllJSONS3Key,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawWritesCompareAllJSONPath,
- ); err != nil {
- return err
- }
- if err = allDurationsWithLabels.CSV(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawWritesCompareAllCSVPath); err != nil {
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawWritesCompareAllCSVS3Key,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawWritesCompareAllCSVPath,
- ); err != nil {
- return err
- }
- } else {
- ts.cfg.Logger.Warn("previous writes summary not found; skipping comparison", zap.Error(err))
- }
- ts.cfg.Logger.Info("uploading new writes summary to s3 bucket to overwrite the previous")
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawWritesCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawWritesJSONPath,
- ); err != nil {
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWritesJSONPath,
- ); err != nil {
- return err
- }
-
- s3Objects = make([]*s3.Object, 0)
- if ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsCompareS3Dir != "" {
- s3Objects, err = aws_s3.ListInDescendingLastModified(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Clean(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsCompareS3Dir)+"/",
- )
- }
- canCompare = len(s3Objects) > 0 && err == nil
- if canCompare {
- reqSummaryS3Key := aws.StringValue(s3Objects[0].Key)
- durRawS3Key := path.Join(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawReadsCompareS3Dir, path.Base(reqSummaryS3Key))
-
- var prevSummary metrics.RequestsSummary
- prevSummary, err = metrics.DownloadRequestsSummaryFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, reqSummaryS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsCompare, err = metrics.CompareRequestsSummary(prevSummary, ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReads)
- if err != nil {
- ts.cfg.Logger.Warn("failed to compare results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsCompareJSONPath, []byte(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsCompare.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsCompareJSONS3Key,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsCompareJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsCompareTablePath, []byte(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsCompare.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsCompareTableS3Key,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsCompareTablePath,
- ); err != nil {
- return err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryReadsCompare:\n%s\n", ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsCompare.Table())
-
- var prevDurations metrics.Durations
- prevDurations, err = metrics.DownloadDurationsFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, durRawS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- prevDurationsWithLabels := metrics.LabelDurations(prevDurations, prevSummary.TestID)
- curDurationsWithLabels := metrics.LabelDurations(curWriteLatencies, ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReads.TestID)
- allDurationsWithLabels := append(prevDurationsWithLabels, curDurationsWithLabels...)
- sortStart := time.Now()
- ts.cfg.Logger.Info("sorting before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- )
- sort.Sort(allDurationsWithLabels)
- ts.cfg.Logger.Info("sorted before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- zap.String("took", time.Since(sortStart).String()),
- )
- allDataJSON, err := json.Marshal(allDurationsWithLabels)
- if err != nil {
- ts.cfg.Logger.Warn("failed to marshal results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawReadsCompareAllJSONPath, []byte(allDataJSON), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawReadsCompareAllJSONS3Key,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawReadsCompareAllJSONPath,
- ); err != nil {
- return err
- }
- if err = allDurationsWithLabels.CSV(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawReadsCompareAllCSVPath); err != nil {
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawReadsCompareAllCSVS3Key,
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawReadsCompareAllCSVPath,
- ); err != nil {
- return err
- }
- } else {
- ts.cfg.Logger.Warn("previous writes summary not found; skipping comparison", zap.Error(err))
- }
- ts.cfg.Logger.Info("uploading new reads summary to s3 bucket to overwrite the previous")
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawReadsCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsRawReadsJSONPath,
- ); err != nil {
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReadsJSONPath,
- ); err != nil {
- return err
- }
-
- return nil
-}
-
-func (ts *tester) publishResults() (err error) {
- tv := aws.Time(time.Now().UTC())
- datums := make([]*cloudwatch.MetricDatum, 0)
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-local-writes-latency-p50"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWrites.LantencyP50.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-local-writes-latency-p90"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWrites.LantencyP90.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-local-writes-latency-p99"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWrites.LantencyP99.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-local-writes-latency-p999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWrites.LantencyP999.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-local-writes-latency-p9999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryWrites.LantencyP9999.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-local-reads-latency-p50"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReads.LantencyP50.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-local-reads-latency-p90"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReads.LantencyP90.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-local-reads-latency-p99"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReads.LantencyP99.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-local-reads-latency-p999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReads.LantencyP999.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-local-reads-latency-p9999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsLocal.RequestsSummaryReads.LantencyP9999.Milliseconds())),
- })
- return cw.PutData(ts.cfg.Logger, ts.cfg.CWAPI, ts.cfg.EKSConfig.CWNamespace, 20, datums...)
-}
diff --git a/eks/secrets/remote/secrets.go b/eks/secrets/remote/secrets.go
deleted file mode 100644
index e9db27a98..000000000
--- a/eks/secrets/remote/secrets.go
+++ /dev/null
@@ -1,1397 +0,0 @@
-// Package remote implements Secrets plugin.
-package remote
-
-import (
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "io/ioutil"
- "os"
- "path"
- "path/filepath"
- "reflect"
- "sort"
- "strings"
- "time"
-
- eks_tester "github.com/aws/aws-k8s-tester/eks/tester"
- "github.com/aws/aws-k8s-tester/eksconfig"
- "github.com/aws/aws-k8s-tester/pkg/aws/cw"
- aws_ecr "github.com/aws/aws-k8s-tester/pkg/aws/ecr"
- aws_s3 "github.com/aws/aws-k8s-tester/pkg/aws/s3"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/metrics"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/service/cloudwatch"
- "github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
- "github.com/aws/aws-sdk-go/service/ecr/ecriface"
- "github.com/aws/aws-sdk-go/service/s3"
- "github.com/aws/aws-sdk-go/service/s3/s3iface"
- "github.com/dustin/go-humanize"
- "go.uber.org/zap"
- batchv1 "k8s.io/api/batch/v1"
- v1 "k8s.io/api/core/v1"
- rbacv1 "k8s.io/api/rbac/v1"
- apierrs "k8s.io/apimachinery/pkg/api/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/utils/exec"
- "sigs.k8s.io/yaml"
-)
-
-// Config defines secrets configuration.
-// ref. https://github.com/kubernetes/perf-tests
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
- S3API s3iface.S3API
- CWAPI cloudwatchiface.CloudWatchAPI
- ECRAPI ecriface.ECRAPI
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-func New(cfg Config) eks_tester.Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
- ecrImage string
-}
-
-func (ts *tester) Create() (err error) {
- if !ts.cfg.EKSConfig.IsEnabledAddOnSecretsRemote() {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
- if ts.cfg.EKSConfig.AddOnSecretsRemote.Created {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Create", zap.String("tester", pkgName))
- ts.cfg.EKSConfig.AddOnSecretsRemote.Created = true
- ts.cfg.EKSConfig.Sync()
- createStart := time.Now()
- defer func() {
- createEnd := time.Now()
- ts.cfg.EKSConfig.AddOnSecretsRemote.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- if ts.ecrImage, _, err = aws_ecr.Check(
- ts.cfg.Logger,
- ts.cfg.ECRAPI,
- ts.cfg.EKSConfig.Partition,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RepositoryAccountID,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RepositoryRegion,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RepositoryName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RepositoryImageTag,
- ); err != nil {
- return err
- }
- if err = k8s_client.CreateNamespace(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace,
- ); err != nil {
- return err
- }
- if err = ts.createServiceAccount(); err != nil {
- return err
- }
- if err = ts.createRBACClusterRole(); err != nil {
- return err
- }
- if err = ts.createRBACClusterRoleBinding(); err != nil {
- return err
- }
- if err = ts.createConfigMap(); err != nil {
- return err
- }
-
- if err = ts.createJob(); err != nil {
- return err
- }
- timeout := 5*time.Minute + 5*time.Minute*time.Duration(ts.cfg.EKSConfig.AddOnSecretsRemote.Completes) + time.Minute*time.Duration(ts.cfg.EKSConfig.AddOnSecretsRemote.Objects/100)
- if timeout > 3*time.Hour {
- timeout = 3 * time.Hour
- }
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- var pods []v1.Pod
- _, pods, err = k8s_client.WaitForJobCompletes(
- ctx,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.Stopc,
- ts.cfg.K8SClient,
- time.Minute,
- 10*time.Second,
- ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace,
- secretsJobName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.Completes,
- k8s_client.WithQueryFunc(func() {
- descArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace,
- "describe",
- "job",
- secretsJobName,
- }
- descCmd := strings.Join(descArgs, " ")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- descOutput, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe job' failed", zap.Error(err))
- }
- out := string(descOutput)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\n\"%s\" output:\n\n%s\n\n", descCmd, out)
- }),
- k8s_client.WithPodFunc(func(pod v1.Pod) {
- switch pod.Status.Phase {
- case v1.PodFailed:
- ts.cfg.Logger.Warn("pod failed",
- zap.String("namespace", pod.Namespace),
- zap.String("pod-name", pod.Name),
- zap.String("pod-status-phase", fmt.Sprintf("%v", pod.Status.Phase)),
- )
- descArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + pod.Namespace,
- "describe",
- "pod",
- pod.Name,
- }
- descCmd := strings.Join(descArgs, " ")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- cmdOutput, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe job' failed", zap.Error(err))
- }
- out := string(cmdOutput)
- fmt.Fprintf(ts.cfg.LogWriter, "\"%s\" output:\n\n%s\n\n", descCmd, out)
-
- logsArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + pod.Namespace,
- "logs",
- fmt.Sprintf("pod/%s", pod.Name),
- "--timestamps",
- }
- logsCmd := strings.Join(logsArgs, " ")
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- cmdOutput, err = exec.New().CommandContext(ctx, logsArgs[0], logsArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl logs' failed", zap.Error(err))
- }
- out = string(cmdOutput)
- fmt.Fprintf(ts.cfg.LogWriter, "\"%s\" output:\n\n%s\n\n", logsCmd, out)
- }
- }),
- )
- cancel()
- if err != nil {
- return err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n")
- for _, item := range pods {
- fmt.Fprintf(ts.cfg.LogWriter, "Job Pod %q: %q\n", item.Name, item.Status.Phase)
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n")
-
- if err = ts.checkResults(); err == nil {
- return err
- }
- if err = ts.publishResults(); err != nil {
- return err
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) Delete() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnSecretsRemote() {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnSecretsRemote.Created {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Delete", zap.String("tester", pkgName))
- deleteStart := time.Now()
- defer func() {
- deleteEnd := time.Now()
- ts.cfg.EKSConfig.AddOnSecretsRemote.TimeFrameDelete = timeutil.NewTimeFrame(deleteStart, deleteEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- var errs []string
-
- if err := ts.deleteJob(); err != nil {
- errs = append(errs, err.Error())
- }
- time.Sleep(2 * time.Minute)
-
- if err := ts.deleteConfigMap(); err != nil {
- errs = append(errs, err.Error())
- }
- if err := ts.deleteRBACClusterRoleBinding(); err != nil {
- errs = append(errs, err.Error())
- }
- if err := ts.deleteRBACClusterRole(); err != nil {
- errs = append(errs, err.Error())
- }
- if err := ts.deleteServiceAccount(); err != nil {
- errs = append(errs, err.Error())
- }
-
- if err := k8s_client.DeleteNamespaceAndWait(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace,
- k8s_client.DefaultNamespaceDeletionInterval,
- k8s_client.DefaultNamespaceDeletionTimeout,
- k8s_client.WithForceDelete(true),
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete secrets namespace (%v)", err))
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- ts.cfg.EKSConfig.AddOnSecretsRemote.Created = false
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-const (
- secretsServiceAccountName = "secrets-remote-service-account"
- secretsRBACRoleName = "secrets-remote-rbac-role"
- secretsRBACClusterRoleBindingName = "secrets-remote-rbac-role-binding"
- secretsKubeConfigConfigMapName = "secrets-remote-kubeconfig-configmap"
- secretsKubeConfigConfigMapFileName = "secrets-remote-kubeconfig-configmap.yaml"
- secretsAppName = "secrets-remote-app"
- secretsJobName = "secrets-remote-job"
-)
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) createServiceAccount() error {
- ts.cfg.Logger.Info("creating secrets ServiceAccount")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- ServiceAccounts(ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace).
- Create(
- ctx,
- &v1.ServiceAccount{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "v1",
- Kind: "ServiceAccount",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: secretsServiceAccountName,
- Namespace: ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace,
- Labels: map[string]string{
- "app.kubernetes.io/name": secretsAppName,
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create secrets ServiceAccount (%v)", err)
- }
-
- ts.cfg.Logger.Info("created secrets ServiceAccount")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) deleteServiceAccount() error {
- ts.cfg.Logger.Info("deleting secrets ServiceAccount")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- ServiceAccounts(ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace).
- Delete(
- ctx,
- secretsServiceAccountName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete secrets ServiceAccount (%v)", err)
- }
- ts.cfg.Logger.Info("deleted secrets ServiceAccount", zap.Error(err))
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) createRBACClusterRole() error {
- ts.cfg.Logger.Info("creating secrets RBAC ClusterRole")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.K8SClient.KubernetesClientSet().
- RbacV1().
- ClusterRoles().
- Create(
- ctx,
- &rbacv1.ClusterRole{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "rbac.authorization.k8s.io/v1",
- Kind: "ClusterRole",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: secretsRBACRoleName,
- Namespace: "default",
- Labels: map[string]string{
- "app.kubernetes.io/name": secretsAppName,
- },
- },
- Rules: []rbacv1.PolicyRule{
- {
- APIGroups: []string{
- "*",
- },
- Resources: []string{
- "secrets",
- },
- Verbs: []string{
- "create",
- "get",
- "list",
- "update",
- "watch",
- "patch",
- },
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create secrets RBAC ClusterRole (%v)", err)
- }
-
- ts.cfg.Logger.Info("created secrets RBAC ClusterRole")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) deleteRBACClusterRole() error {
- ts.cfg.Logger.Info("deleting secrets RBAC ClusterRole")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- RbacV1().
- ClusterRoles().
- Delete(
- ctx,
- secretsRBACRoleName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete secrets RBAC ClusterRole (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted secrets RBAC ClusterRole", zap.Error(err))
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) createRBACClusterRoleBinding() error {
- ts.cfg.Logger.Info("creating secrets RBAC ClusterRoleBinding")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.K8SClient.KubernetesClientSet().
- RbacV1().
- ClusterRoleBindings().
- Create(
- ctx,
- &rbacv1.ClusterRoleBinding{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "rbac.authorization.k8s.io/v1",
- Kind: "ClusterRoleBinding",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: secretsRBACClusterRoleBindingName,
- Namespace: "default",
- Labels: map[string]string{
- "app.kubernetes.io/name": secretsAppName,
- },
- },
- RoleRef: rbacv1.RoleRef{
- APIGroup: "rbac.authorization.k8s.io",
- Kind: "ClusterRole",
- Name: secretsRBACRoleName,
- },
- Subjects: []rbacv1.Subject{
- {
- APIGroup: "",
- Kind: "ServiceAccount",
- Name: secretsServiceAccountName,
- Namespace: ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace,
- },
- { // https://kubernetes.io/docs/reference/access-authn-authz/rbac/
- APIGroup: "rbac.authorization.k8s.io",
- Kind: "User",
- Name: "system:node",
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create secrets RBAC ClusterRoleBinding (%v)", err)
- }
-
- ts.cfg.Logger.Info("created secrets RBAC ClusterRoleBinding")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) deleteRBACClusterRoleBinding() error {
- ts.cfg.Logger.Info("deleting secrets RBAC ClusterRoleBinding")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- RbacV1().
- ClusterRoleBindings().
- Delete(
- ctx,
- secretsRBACClusterRoleBindingName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete secrets RBAC ClusterRoleBinding (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted secrets RBAC ClusterRoleBinding", zap.Error(err))
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) createConfigMap() error {
- ts.cfg.Logger.Info("creating config map")
-
- b, err := ioutil.ReadFile(ts.cfg.EKSConfig.KubeConfigPath)
- if err != nil {
- return err
- }
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err = ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- ConfigMaps(ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace).
- Create(
- ctx,
- &v1.ConfigMap{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "v1",
- Kind: "ConfigMap",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: secretsKubeConfigConfigMapName,
- Namespace: ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace,
- Labels: map[string]string{
- "name": secretsKubeConfigConfigMapName,
- },
- },
- Data: map[string]string{
- secretsKubeConfigConfigMapFileName: string(b),
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return err
- }
-
- ts.cfg.Logger.Info("created config map")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) deleteConfigMap() error {
- ts.cfg.Logger.Info("deleting config map")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- ConfigMaps(ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace).
- Delete(
- ctx,
- secretsKubeConfigConfigMapName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil {
- return err
- }
- ts.cfg.Logger.Info("deleted config map")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) createJob() (err error) {
- obj, b, err := ts.createObject()
- if err != nil {
- return err
- }
-
- ts.cfg.Logger.Info("creating Job",
- zap.String("name", secretsJobName),
- zap.String("object-size", humanize.Bytes(uint64(len(b)))),
- )
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err = ts.cfg.K8SClient.KubernetesClientSet().
- BatchV1().
- Jobs(ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace).
- Create(ctx, &obj, metav1.CreateOptions{})
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create Job (%v)", err)
- }
-
- ts.cfg.Logger.Info("created Job")
- return nil
-}
-
-func (ts *tester) createObject() (batchv1.Job, string, error) {
- // "/opt/"+secretsKubeConfigConfigMapFileName,
- // do not specify "kubeconfig", and use in-cluster config via "pkg/k8s-client"
- // otherwise, error "namespaces is forbidden: User "system:node:ip-192-168-84..."
- // ref. https://github.com/kubernetes/client-go/blob/master/examples/in-cluster-client-configuration/main.go
- testerCmd := fmt.Sprintf("/aws-k8s-tester eks create secrets --partition=%s --region=%s --s3-bucket-name=%s --clients=%d --client-qps=%f --client-burst=%d --client-timeout=%s --namespace=%s --name-prefix=%s --objects=%d --object-size=%d --requests-raw-writes-json-s3-dir=%s --requests-summary-writes-json-s3-dir=%s --requests-summary-writes-table-s3-dir=%s --requests-raw-reads-json-s3-dir=%s --requests-summary-reads-json-s3-dir=%s --requests-summary-reads-table-s3-dir=%s --writes-output-name-prefix=%s --reads-output-name-prefix=%s",
- ts.cfg.EKSConfig.Partition,
- ts.cfg.EKSConfig.Region,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.Clients,
- ts.cfg.EKSConfig.ClientQPS,
- ts.cfg.EKSConfig.ClientBurst,
- ts.cfg.EKSConfig.ClientTimeout,
- ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace,
- ts.cfg.EKSConfig.AddOnSecretsRemote.NamePrefix,
- ts.cfg.EKSConfig.AddOnSecretsRemote.Objects,
- ts.cfg.EKSConfig.AddOnSecretsRemote.ObjectSize,
- path.Dir(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawWritesJSONS3Key),
- path.Dir(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesJSONS3Key),
- path.Dir(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesTableS3Key),
- path.Dir(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawReadsJSONS3Key),
- path.Dir(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsJSONS3Key),
- path.Dir(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsTableS3Key),
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesOutputNamePrefix,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsOutputNamePrefix,
- )
-
- dirOrCreate := v1.HostPathDirectoryOrCreate
- podSpec := v1.PodTemplateSpec{
- ObjectMeta: metav1.ObjectMeta{
- Labels: map[string]string{
- "app.kubernetes.io/name": secretsAppName,
- },
- },
- Spec: v1.PodSpec{
- ServiceAccountName: secretsServiceAccountName,
-
- // spec.template.spec.restartPolicy: Unsupported value: "Always": supported values: "OnFailure", "Never"
- // ref. https://github.com/kubernetes/kubernetes/issues/54870
- RestartPolicy: v1.RestartPolicyNever,
-
- // TODO: set resource limits
- Containers: []v1.Container{
- {
- Name: secretsAppName,
- Image: ts.ecrImage,
- ImagePullPolicy: v1.PullAlways,
-
- Command: []string{
- "/bin/sh",
- "-ec",
- testerCmd,
- },
-
- // grant access "/dev/kmsg"
- SecurityContext: &v1.SecurityContext{
- Privileged: aws.Bool(true),
- },
-
- // ref. https://kubernetes.io/docs/concepts/cluster-administration/logging/
- VolumeMounts: []v1.VolumeMount{
- { // to execute
- Name: secretsKubeConfigConfigMapName,
- MountPath: "/opt",
- },
- { // to write
- Name: "varlog",
- MountPath: "/var/log",
- ReadOnly: false,
- },
- },
- },
- },
-
- // ref. https://kubernetes.io/docs/concepts/cluster-administration/logging/
- Volumes: []v1.Volume{
- { // to execute
- Name: secretsKubeConfigConfigMapName,
- VolumeSource: v1.VolumeSource{
- ConfigMap: &v1.ConfigMapVolumeSource{
- LocalObjectReference: v1.LocalObjectReference{
- Name: secretsKubeConfigConfigMapName,
- },
- DefaultMode: aws.Int32(0777),
- },
- },
- },
- { // to write
- Name: "varlog",
- VolumeSource: v1.VolumeSource{
- HostPath: &v1.HostPathVolumeSource{
- Path: "/var/log",
- Type: &dirOrCreate,
- },
- },
- },
- },
-
- NodeSelector: map[string]string{
- // do not deploy in fake nodes, obviously
- "NodeType": "regular",
- },
- },
- }
-
- jobObj := batchv1.Job{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "batch/v1",
- Kind: "Job",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: secretsJobName,
- Namespace: ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace,
- },
- Spec: batchv1.JobSpec{
- Completions: aws.Int32(int32(ts.cfg.EKSConfig.AddOnSecretsRemote.Completes)),
- Parallelism: aws.Int32(int32(ts.cfg.EKSConfig.AddOnSecretsRemote.Parallels)),
- Template: podSpec,
- // TODO: 'TTLSecondsAfterFinished' is still alpha
- // https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/
- },
- }
- b, err := yaml.Marshal(jobObj)
- return jobObj, string(b), err
-}
-
-func (ts *tester) deleteJob() (err error) {
- foreground := metav1.DeletePropagationForeground
- ts.cfg.Logger.Info("deleting Job", zap.String("name", secretsJobName))
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err = ts.cfg.
- K8SClient.KubernetesClientSet().
- BatchV1().
- Jobs(ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace).
- Delete(
- ctx,
- secretsJobName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err == nil {
- ts.cfg.Logger.Info("deleted Job", zap.String("name", secretsJobName))
- } else {
- ts.cfg.Logger.Warn("failed to delete Job", zap.Error(err))
- }
- return err
-}
-
-// 1. if previous summary exists, download and compare
-// 2. upload new summary and overwrite the previous s3 key
-func (ts *tester) checkResults() (err error) {
- curTS := time.Now().UTC().Format(time.RFC3339Nano)
- ts.cfg.Logger.Info("checking results", zap.String("timestamp", curTS))
-
- writesSummary := metrics.RequestsSummary{TestID: curTS}
- curWriteLatencies := make(metrics.Durations, 0, 20000)
- writesDirRaw := ""
- writesDirSummary := ""
-
- readsSummary := metrics.RequestsSummary{TestID: curTS}
- curReadLatencies := make(metrics.Durations, 0, 20000)
- readsDirRaw := ""
- readsDirSummary := ""
-
- writesDirRaw, err = aws_s3.DownloadDir(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Dir(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawWritesJSONS3Key),
- )
- if err == nil {
- ts.cfg.Logger.Info("reading writes results raw",
- zap.String("writes-dir", writesDirRaw),
- zap.String("s3-dir", path.Dir(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawWritesJSONS3Key)),
- )
- cnt := 0
- err = filepath.Walk(writesDirRaw, func(fpath string, info os.FileInfo, werr error) error {
- if werr != nil {
- return werr
- }
- if info.IsDir() {
- return nil
- }
- cnt++
- switch {
- case strings.HasSuffix(fpath, "-writes-raw.json"):
- b, err := ioutil.ReadFile(fpath)
- if err != nil {
- return fmt.Errorf("failed to open %q (%v)", fpath, err)
- }
- var r metrics.Durations
- if err = json.Unmarshal(b, &r); err != nil {
- return fmt.Errorf("failed to unmarshal %q (%s, %v)", fpath, string(b), err)
- }
- curWriteLatencies = append(curWriteLatencies, r...)
- }
- return nil
- })
- if err != nil || cnt == 0 {
- ts.cfg.Logger.Warn("failed to read writes results", zap.Int("file-count", cnt), zap.Error(err))
- os.RemoveAll(writesDirRaw)
- writesDirRaw = ""
- }
- }
- writesDirSummary, err = aws_s3.DownloadDir(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Dir(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesJSONS3Key),
- )
- if err == nil {
- ts.cfg.Logger.Info("reading writes results summary",
- zap.String("writes-dir", writesDirSummary),
- zap.String("s3-dir", path.Dir(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesJSONS3Key)),
- )
- cnt := 0
- err = filepath.Walk(writesDirSummary, func(fpath string, info os.FileInfo, werr error) error {
- if werr != nil {
- return werr
- }
- if info.IsDir() {
- return nil
- }
- cnt++
- switch {
- case strings.HasSuffix(fpath, "-writes-summary.json"):
- b, err := ioutil.ReadFile(fpath)
- if err != nil {
- return fmt.Errorf("failed to open %q (%v)", fpath, err)
- }
- var r metrics.RequestsSummary
- if err = json.Unmarshal(b, &r); err != nil {
- return fmt.Errorf("failed to unmarshal %q (%s, %v)", fpath, string(b), err)
- }
- writesSummary.SuccessTotal += r.SuccessTotal
- writesSummary.FailureTotal += r.FailureTotal
- if writesSummary.LatencyHistogram == nil || len(writesSummary.LatencyHistogram) == 0 {
- writesSummary.LatencyHistogram = r.LatencyHistogram
- } else {
- writesSummary.LatencyHistogram, err = metrics.MergeHistograms(writesSummary.LatencyHistogram, r.LatencyHistogram)
- if err != nil {
- return fmt.Errorf("failed to merge histograms (%v)", err)
- }
- }
- }
- return nil
- })
- if err != nil || cnt == 0 {
- ts.cfg.Logger.Warn("failed to read writes results", zap.Int("file-count", cnt), zap.Error(err))
- os.RemoveAll(writesDirSummary)
- writesDirSummary = ""
- }
- }
-
- readsDirRaw, err = aws_s3.DownloadDir(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Dir(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawReadsJSONS3Key),
- )
- if err == nil {
- ts.cfg.Logger.Info("reading reads results raw",
- zap.String("reads-dir", readsDirRaw),
- zap.String("s3-dir", path.Dir(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawReadsJSONS3Key)),
- )
- cnt := 0
- err = filepath.Walk(readsDirRaw, func(fpath string, info os.FileInfo, werr error) error {
- if werr != nil {
- return werr
- }
- if info.IsDir() {
- return nil
- }
- cnt++
- switch {
- case strings.HasSuffix(fpath, "-reads-raw.json"):
- b, err := ioutil.ReadFile(fpath)
- if err != nil {
- return fmt.Errorf("failed to open %q (%v)", fpath, err)
- }
- var r metrics.Durations
- if err = json.Unmarshal(b, &r); err != nil {
- return fmt.Errorf("failed to unmarshal %q (%s, %v)", fpath, string(b), err)
- }
- curReadLatencies = append(curReadLatencies, r...)
- }
- return nil
- })
- if err != nil || cnt == 0 {
- ts.cfg.Logger.Warn("failed to read reads results", zap.Int("file-count", cnt), zap.Error(err))
- os.RemoveAll(readsDirRaw)
- readsDirRaw = ""
- }
- }
- readsDirSummary, err = aws_s3.DownloadDir(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Dir(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsJSONS3Key),
- )
- if err == nil {
- ts.cfg.Logger.Info("reading reads results summary",
- zap.String("reads-dir", readsDirSummary),
- zap.String("s3-dir", path.Dir(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsJSONS3Key)),
- )
- cnt := 0
- err = filepath.Walk(readsDirSummary, func(fpath string, info os.FileInfo, werr error) error {
- if werr != nil {
- return werr
- }
- if info.IsDir() {
- return nil
- }
- cnt++
- switch {
- case strings.HasSuffix(fpath, "-reads-summary.json"):
- b, err := ioutil.ReadFile(fpath)
- if err != nil {
- return fmt.Errorf("failed to open %q (%v)", fpath, err)
- }
- var r metrics.RequestsSummary
- if err = json.Unmarshal(b, &r); err != nil {
- return fmt.Errorf("failed to unmarshal %q (%s, %v)", fpath, string(b), err)
- }
- readsSummary.SuccessTotal += r.SuccessTotal
- readsSummary.FailureTotal += r.FailureTotal
- if readsSummary.LatencyHistogram == nil || len(readsSummary.LatencyHistogram) == 0 {
- readsSummary.LatencyHistogram = r.LatencyHistogram
- } else {
- readsSummary.LatencyHistogram, err = metrics.MergeHistograms(readsSummary.LatencyHistogram, r.LatencyHistogram)
- if err != nil {
- return fmt.Errorf("failed to merge histograms (%v)", err)
- }
- }
- }
- return nil
- })
- if err != nil || cnt == 0 {
- ts.cfg.Logger.Warn("failed to read reads results", zap.Int("file-count", cnt), zap.Error(err))
- os.RemoveAll(readsDirSummary)
- readsDirSummary = ""
- }
- }
-
- sortStart := time.Now()
- ts.cfg.Logger.Info("sorting write latencies", zap.Int("data", len(curWriteLatencies)))
- sort.Sort(curWriteLatencies)
- ts.cfg.Logger.Info("sorted write latencies", zap.String("took", time.Since(sortStart).String()))
- writesSummary.LantencyP50 = curWriteLatencies.PickLantencyP50()
- writesSummary.LantencyP90 = curWriteLatencies.PickLantencyP90()
- writesSummary.LantencyP99 = curWriteLatencies.PickLantencyP99()
- writesSummary.LantencyP999 = curWriteLatencies.PickLantencyP999()
- writesSummary.LantencyP9999 = curWriteLatencies.PickLantencyP9999()
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWrites = writesSummary
- ts.cfg.EKSConfig.Sync()
-
- sortStart = time.Now()
- ts.cfg.Logger.Info("sorting read latencies", zap.Int("data", len(curReadLatencies)))
- sort.Sort(curReadLatencies)
- ts.cfg.Logger.Info("sorted read latencies", zap.String("took", time.Since(sortStart).String()))
- readsSummary.LantencyP50 = curReadLatencies.PickLantencyP50()
- readsSummary.LantencyP90 = curReadLatencies.PickLantencyP90()
- readsSummary.LantencyP99 = curReadLatencies.PickLantencyP99()
- readsSummary.LantencyP999 = curReadLatencies.PickLantencyP999()
- readsSummary.LantencyP9999 = curReadLatencies.PickLantencyP9999()
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReads = readsSummary
- ts.cfg.EKSConfig.Sync()
-
- wb, err := json.Marshal(curWriteLatencies)
- if err != nil {
- ts.cfg.Logger.Warn("failed to encode JSON", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawWritesJSONPath, wb, 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawWritesJSONS3Key,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawWritesJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesJSONPath, []byte(writesSummary.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesJSONS3Key,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesTablePath, []byte(writesSummary.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesTableS3Key,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesTablePath,
- ); err != nil {
- return err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryWrites:\n%s\n", writesSummary.Table())
-
- rb, err := json.Marshal(curReadLatencies)
- if err != nil {
- ts.cfg.Logger.Warn("failed to encode JSON", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawReadsJSONPath, rb, 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawReadsJSONS3Key,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawReadsJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsJSONPath, []byte(readsSummary.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsJSONS3Key,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsTablePath, []byte(readsSummary.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsTableS3Key,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsTablePath,
- ); err != nil {
- return err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryReads:\n%s\n", readsSummary.Table())
-
- s3Objects := make([]*s3.Object, 0)
- if ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesCompareS3Dir != "" {
- s3Objects, err = aws_s3.ListInDescendingLastModified(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Clean(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesCompareS3Dir)+"/",
- )
- }
- canCompare := len(s3Objects) > 0 && err == nil
- if canCompare {
- reqSummaryS3Key := aws.StringValue(s3Objects[0].Key)
- durRawS3Key := path.Join(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawWritesCompareS3Dir, path.Base(reqSummaryS3Key))
-
- var prevSummary metrics.RequestsSummary
- prevSummary, err = metrics.DownloadRequestsSummaryFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, reqSummaryS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesCompare, err = metrics.CompareRequestsSummary(prevSummary, ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWrites)
- if err != nil {
- ts.cfg.Logger.Warn("failed to compare results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesCompareJSONPath, []byte(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesCompare.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesCompareJSONS3Key,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesCompareJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesCompareTablePath, []byte(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesCompare.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesCompareTableS3Key,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesCompareTablePath,
- ); err != nil {
- return err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryWritesCompare:\n%s\n", ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesCompare.Table())
-
- var prevDurations metrics.Durations
- prevDurations, err = metrics.DownloadDurationsFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, durRawS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- prevDurationsWithLabels := metrics.LabelDurations(prevDurations, prevSummary.TestID)
- curDurationsWithLabels := metrics.LabelDurations(curWriteLatencies, ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWrites.TestID)
- allDurationsWithLabels := append(prevDurationsWithLabels, curDurationsWithLabels...)
- sortStart := time.Now()
- ts.cfg.Logger.Info("sorting before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- )
- sort.Sort(allDurationsWithLabels)
- ts.cfg.Logger.Info("sorted before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- zap.String("took", time.Since(sortStart).String()),
- )
- allDataJSON, err := json.Marshal(allDurationsWithLabels)
- if err != nil {
- ts.cfg.Logger.Warn("failed to marshal results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawWritesCompareAllJSONPath, []byte(allDataJSON), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawWritesCompareAllJSONS3Key,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawWritesCompareAllJSONPath,
- ); err != nil {
- return err
- }
- if err = allDurationsWithLabels.CSV(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawWritesCompareAllCSVPath); err != nil {
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawWritesCompareAllCSVS3Key,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawWritesCompareAllCSVPath,
- ); err != nil {
- return err
- }
- } else {
- ts.cfg.Logger.Warn("previous writes summary not found; skipping comparison", zap.Error(err))
- }
- ts.cfg.Logger.Info("uploading new writes summary to s3 bucket to overwrite the previous")
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawWritesCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawWritesJSONPath,
- ); err != nil {
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWritesJSONPath,
- ); err != nil {
- return err
- }
-
- s3Objects = make([]*s3.Object, 0)
- if ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsCompareS3Dir != "" {
- s3Objects, err = aws_s3.ListInDescendingLastModified(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Clean(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsCompareS3Dir)+"/",
- )
- }
- canCompare = len(s3Objects) > 0 && err == nil
- if canCompare {
- reqSummaryS3Key := aws.StringValue(s3Objects[0].Key)
- durRawS3Key := path.Join(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawReadsCompareS3Dir, path.Base(reqSummaryS3Key))
-
- var prevSummary metrics.RequestsSummary
- prevSummary, err = metrics.DownloadRequestsSummaryFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, reqSummaryS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsCompare, err = metrics.CompareRequestsSummary(prevSummary, ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReads)
- if err != nil {
- ts.cfg.Logger.Warn("failed to compare results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsCompareJSONPath, []byte(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsCompare.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsCompareJSONS3Key,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsCompareJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsCompareTablePath, []byte(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsCompare.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsCompareTableS3Key,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsCompareTablePath,
- ); err != nil {
- return err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryReadsCompare:\n%s\n", ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsCompare.Table())
-
- var prevDurations metrics.Durations
- prevDurations, err = metrics.DownloadDurationsFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, durRawS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- prevDurationsWithLabels := metrics.LabelDurations(prevDurations, prevSummary.TestID)
- curDurationsWithLabels := metrics.LabelDurations(curWriteLatencies, ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReads.TestID)
- allDurationsWithLabels := append(prevDurationsWithLabels, curDurationsWithLabels...)
- sortStart := time.Now()
- ts.cfg.Logger.Info("sorting before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- )
- sort.Sort(allDurationsWithLabels)
- ts.cfg.Logger.Info("sorted before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- zap.String("took", time.Since(sortStart).String()),
- )
- allDataJSON, err := json.Marshal(allDurationsWithLabels)
- if err != nil {
- ts.cfg.Logger.Warn("failed to marshal results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawReadsCompareAllJSONPath, []byte(allDataJSON), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawReadsCompareAllJSONS3Key,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawReadsCompareAllJSONPath,
- ); err != nil {
- return err
- }
- if err = allDurationsWithLabels.CSV(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawReadsCompareAllCSVPath); err != nil {
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawReadsCompareAllCSVS3Key,
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawReadsCompareAllCSVPath,
- ); err != nil {
- return err
- }
- } else {
- ts.cfg.Logger.Warn("previous writes summary not found; skipping comparison", zap.Error(err))
- }
- ts.cfg.Logger.Info("uploading new reads summary to s3 bucket to overwrite the previous")
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawReadsCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsRawReadsJSONPath,
- ); err != nil {
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReadsJSONPath,
- ); err != nil {
- return err
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) publishResults() (err error) {
- tv := aws.Time(time.Now().UTC())
- datums := make([]*cloudwatch.MetricDatum, 0)
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-remote-writes-latency-p50"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWrites.LantencyP50.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-remote-writes-latency-p90"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWrites.LantencyP90.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-remote-writes-latency-p99"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWrites.LantencyP99.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-remote-writes-latency-p999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWrites.LantencyP999.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-remote-writes-latency-p9999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryWrites.LantencyP9999.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-remote-reads-latency-p50"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReads.LantencyP50.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-remote-reads-latency-p90"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReads.LantencyP90.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-remote-reads-latency-p99"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReads.LantencyP99.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-remote-reads-latency-p999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReads.LantencyP999.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-secrets-remote-reads-latency-p9999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnSecretsRemote.RequestsSummaryReads.LantencyP9999.Milliseconds())),
- })
- return cw.PutData(ts.cfg.Logger, ts.cfg.CWAPI, ts.cfg.EKSConfig.CWNamespace, 20, datums...)
-}
diff --git a/eks/secrets/secrets.go b/eks/secrets/secrets.go
deleted file mode 100644
index 52d418332..000000000
--- a/eks/secrets/secrets.go
+++ /dev/null
@@ -1,431 +0,0 @@
-// Package secrets implements Secrets plugin.
-package secrets
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "io"
- "io/ioutil"
- "sort"
- "sync"
- "time"
-
- aws_s3 "github.com/aws/aws-k8s-tester/pkg/aws/s3"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/metrics"
- "github.com/aws/aws-k8s-tester/pkg/randutil"
- "github.com/aws/aws-sdk-go/service/s3/s3iface"
- "github.com/prometheus/client_golang/prometheus"
- "go.uber.org/zap"
- v1 "k8s.io/api/core/v1"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/client-go/kubernetes"
-)
-
-var (
- writeRequestsSuccessTotal = prometheus.NewGauge(
- prometheus.GaugeOpts{
- Namespace: "secrets",
- Subsystem: "client",
- Name: "write_requests_success_total",
- Help: "Total number of successful write requests.",
- })
- writeRequestsFailureTotal = prometheus.NewGauge(
- prometheus.GaugeOpts{
- Namespace: "secrets",
- Subsystem: "client",
- Name: "write_requests_failure_total",
- Help: "Total number of successful write requests.",
- })
- writeRequestLatencyMs = prometheus.NewHistogram(
- prometheus.HistogramOpts{
- Namespace: "secrets",
- Subsystem: "client",
- Name: "write_request_latency_milliseconds",
- Help: "Bucketed histogram of client-side write request and response latency.",
-
- // lowest bucket start of upper bound 0.5 ms with factor 2
- // highest bucket start of 0.5 ms * 2^13 == 4.096 sec
- Buckets: prometheus.ExponentialBuckets(0.5, 2, 14),
- })
-
- readRequestsSuccessTotal = prometheus.NewGauge(
- prometheus.GaugeOpts{
- Namespace: "secrets",
- Subsystem: "client",
- Name: "read_requests_success_total",
- Help: "Total number of successful read requests.",
- })
- readRequestsFailureTotal = prometheus.NewGauge(
- prometheus.GaugeOpts{
- Namespace: "secrets",
- Subsystem: "client",
- Name: "read_requests_failure_total",
- Help: "Total number of successful read requests.",
- })
- readRequestLatencyMs = prometheus.NewHistogram(
- prometheus.HistogramOpts{
- Namespace: "secrets",
- Subsystem: "client",
- Name: "read_request_latency_milliseconds",
- Help: "Bucketed histogram of client-side read request and response latency.",
-
- // lowest bucket start of upper bound 0.5 ms with factor 2
- // highest bucket start of 0.5 ms * 2^13 == 4.096 sec
- Buckets: prometheus.ExponentialBuckets(0.5, 2, 14),
- })
-)
-
-func init() {
- prometheus.MustRegister(writeRequestsSuccessTotal)
- prometheus.MustRegister(writeRequestsFailureTotal)
- prometheus.MustRegister(writeRequestLatencyMs)
- prometheus.MustRegister(readRequestsSuccessTotal)
- prometheus.MustRegister(readRequestsFailureTotal)
- prometheus.MustRegister(readRequestLatencyMs)
-}
-
-// Config configures Secret loader.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
-
- Stopc chan struct{}
-
- S3API s3iface.S3API
- S3BucketName string
-
- Client k8s_client.EKS
- ClientTimeout time.Duration
-
- Namespace string
-
- // NamePrefix is the prefix of Secret name.
- // If multiple Secret loader is running,
- // this must be unique per worker to avoid name conflicts.
- NamePrefix string
-
- Objects int
- ObjectSize int
-
- RequestsRawWritesJSONPath string
- RequestsRawWritesJSONS3Key string
- RequestsSummaryWritesJSONPath string
- RequestsSummaryWritesJSONS3Key string
- RequestsSummaryWritesTablePath string
- RequestsSummaryWritesTableS3Key string
-
- RequestsRawReadsJSONPath string
- RequestsRawReadsJSONS3Key string
- RequestsSummaryReadsJSONPath string
- RequestsSummaryReadsJSONS3Key string
- RequestsSummaryReadsTablePath string
- RequestsSummaryReadsTableS3Key string
-}
-
-// Loader defines Secret loader operations.
-type Loader interface {
- Start()
- Stop()
- CollectMetrics() (writeLatencies metrics.Durations, writesSummary metrics.RequestsSummary, readLatencies metrics.Durations, readsSummary metrics.RequestsSummary, err error)
-}
-
-type loader struct {
- cfg Config
- donec chan struct{}
- donecCloseOnce *sync.Once
-
- writeLatencies metrics.Durations
- readLatencies metrics.Durations
-}
-
-func New(cfg Config) Loader {
- return &loader{
- cfg: cfg,
- donec: make(chan struct{}),
- donecCloseOnce: new(sync.Once),
- }
-}
-
-func (ld *loader) Start() {
- ld.cfg.Logger.Info("starting write function", zap.String("namespace-write", ld.cfg.Namespace))
- var created []string
- ld.writeLatencies, created = startWrites(ld.cfg.Logger, ld.cfg.Client.KubernetesClientSet(), ld.cfg.ClientTimeout, ld.cfg.Namespace, ld.cfg.NamePrefix, ld.cfg.Objects, ld.cfg.ObjectSize, ld.cfg.Stopc, ld.donec)
- ld.cfg.Logger.Info("completed write function", zap.String("namespace-write", ld.cfg.Namespace))
-
- // TODO: create Pod with created secrets mounted as volume, read them, measure latency
- // as implemented in aws-k8s-tester <= v1.2.1
- // ref. https://github.com/aws/aws-k8s-tester/blob/v1.2.1/eks/secrets/secrets.go#L404-L514
-
- ld.cfg.Logger.Info("starting read function", zap.String("namespace-read", ld.cfg.Namespace))
- ld.readLatencies = startReads(ld.cfg.Logger, ld.cfg.Client.KubernetesClientSet(), ld.cfg.ClientTimeout, ld.cfg.Namespace, created, ld.cfg.Stopc, ld.donec)
- ld.cfg.Logger.Info("completed read function", zap.String("namespace-read", ld.cfg.Namespace))
-}
-
-func (ld *loader) Stop() {
- ld.cfg.Logger.Info("stopping and waiting")
- ld.donecCloseOnce.Do(func() {
- close(ld.donec)
- })
- ld.cfg.Logger.Info("stopped and waited")
-}
-
-// GetMetrics locally fetches output from registered metrics.
-// ref. https://pkg.go.dev/github.com/prometheus/client_golang@v1.6.0/prometheus/promhttp?tab=doc#Handler
-func (ts *loader) CollectMetrics() (writeLatencies metrics.Durations, writesSummary metrics.RequestsSummary, readLatencies metrics.Durations, readsSummary metrics.RequestsSummary, err error) {
- curTS := time.Now().UTC().Format(time.RFC3339Nano)
- writesSummary = metrics.RequestsSummary{TestID: curTS}
- readsSummary = metrics.RequestsSummary{TestID: curTS}
-
- // https://pkg.go.dev/github.com/prometheus/client_golang/prometheus?tab=doc#Gatherer
- mfs, err := prometheus.DefaultGatherer.Gather()
- if err != nil {
- ts.cfg.Logger.Warn("failed to gather prometheus metrics", zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- for _, mf := range mfs {
- if mf == nil {
- continue
- }
- switch *mf.Name {
- case "secrets_client_write_requests_success_total":
- gg := mf.Metric[0].GetGauge()
- writesSummary.SuccessTotal = gg.GetValue()
- case "secrets_client_write_requests_failure_total":
- gg := mf.Metric[0].GetGauge()
- writesSummary.FailureTotal = gg.GetValue()
- case "secrets_client_write_request_latency_milliseconds":
- writesSummary.LatencyHistogram, err = metrics.ParseHistogram("milliseconds", mf.Metric[0].GetHistogram())
- if err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
-
- case "secrets_client_read_requests_success_total":
- gg := mf.Metric[0].GetGauge()
- readsSummary.SuccessTotal = gg.GetValue()
- case "secrets_client_read_requests_failure_total":
- gg := mf.Metric[0].GetGauge()
- readsSummary.FailureTotal = gg.GetValue()
- case "secrets_client_read_request_latency_milliseconds":
- readsSummary.LatencyHistogram, err = metrics.ParseHistogram("milliseconds", mf.Metric[0].GetHistogram())
- if err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- }
- }
-
- ts.cfg.Logger.Info("sorting write latency results", zap.Int("total-data-points", ts.writeLatencies.Len()))
- now := time.Now()
- sort.Sort(ts.writeLatencies)
- ts.cfg.Logger.Info("sorted write latency results", zap.Int("total-data-points", ts.writeLatencies.Len()), zap.String("took", time.Since(now).String()))
- writesSummary.LantencyP50 = ts.writeLatencies.PickLantencyP50()
- writesSummary.LantencyP90 = ts.writeLatencies.PickLantencyP90()
- writesSummary.LantencyP99 = ts.writeLatencies.PickLantencyP99()
- writesSummary.LantencyP999 = ts.writeLatencies.PickLantencyP999()
- writesSummary.LantencyP9999 = ts.writeLatencies.PickLantencyP9999()
-
- ts.cfg.Logger.Info("writing latency results in JSON to disk", zap.String("path", ts.cfg.RequestsRawWritesJSONPath))
- wb, err := json.Marshal(ts.writeLatencies)
- if err != nil {
- ts.cfg.Logger.Warn("failed to encode latency results in JSON", zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = ioutil.WriteFile(ts.cfg.RequestsRawWritesJSONPath, wb, 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write latency results in JSON to disk", zap.String("path", ts.cfg.RequestsRawWritesJSONPath), zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.S3BucketName,
- ts.cfg.RequestsRawWritesJSONS3Key,
- ts.cfg.RequestsRawWritesJSONPath,
- ); err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
-
- ts.cfg.Logger.Info("sorting read latency results", zap.Int("total-data-points", ts.readLatencies.Len()))
- now = time.Now()
- sort.Sort(ts.readLatencies)
- ts.cfg.Logger.Info("sorted read latency results", zap.Int("total-data-points", ts.readLatencies.Len()), zap.String("took", time.Since(now).String()))
- readsSummary.LantencyP50 = ts.readLatencies.PickLantencyP50()
- readsSummary.LantencyP90 = ts.readLatencies.PickLantencyP90()
- readsSummary.LantencyP99 = ts.readLatencies.PickLantencyP99()
- readsSummary.LantencyP999 = ts.readLatencies.PickLantencyP999()
- readsSummary.LantencyP9999 = ts.readLatencies.PickLantencyP9999()
-
- ts.cfg.Logger.Info("writing latency results in JSON to disk", zap.String("path", ts.cfg.RequestsRawReadsJSONPath))
- wb, err = json.Marshal(ts.readLatencies)
- if err != nil {
- ts.cfg.Logger.Warn("failed to encode latency results in JSON", zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = ioutil.WriteFile(ts.cfg.RequestsRawReadsJSONPath, wb, 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write latency results in JSON to disk", zap.String("path", ts.cfg.RequestsRawReadsJSONPath), zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.S3BucketName,
- ts.cfg.RequestsRawReadsJSONS3Key,
- ts.cfg.RequestsRawReadsJSONPath,
- ); err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
-
- if err = ioutil.WriteFile(ts.cfg.RequestsSummaryWritesJSONPath, []byte(writesSummary.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.S3BucketName,
- ts.cfg.RequestsSummaryWritesJSONS3Key,
- ts.cfg.RequestsSummaryWritesJSONPath,
- ); err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = ioutil.WriteFile(ts.cfg.RequestsSummaryWritesTablePath, []byte(writesSummary.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.S3BucketName,
- ts.cfg.RequestsSummaryWritesTableS3Key,
- ts.cfg.RequestsSummaryWritesTablePath,
- ); err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nSummaryWritesTable:\n%s\n", writesSummary.Table())
-
- if err = ioutil.WriteFile(ts.cfg.RequestsSummaryReadsJSONPath, []byte(readsSummary.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.S3BucketName,
- ts.cfg.RequestsSummaryReadsJSONS3Key,
- ts.cfg.RequestsSummaryReadsJSONPath,
- ); err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = ioutil.WriteFile(ts.cfg.RequestsSummaryReadsTablePath, []byte(readsSummary.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.S3BucketName,
- ts.cfg.RequestsSummaryReadsTableS3Key,
- ts.cfg.RequestsSummaryReadsTablePath,
- ); err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryReadsTable:\n%s\n", readsSummary.Table())
-
- return ts.writeLatencies, writesSummary, ts.readLatencies, readsSummary, nil
-}
-
-func startWrites(lg *zap.Logger, cli *kubernetes.Clientset, timeout time.Duration, namespace string, namePrefix string, objects int, objectSize int, stopc chan struct{}, donec chan struct{}) (ds metrics.Durations, created []string) {
- lg.Info("starting startWrites", zap.Int("objects", objects), zap.Int("object-size", objectSize))
- ds = make(metrics.Durations, 0, 20000)
-
- val := randutil.String(objectSize)
- for i := 0; i < objects; i++ {
- select {
- case <-stopc:
- lg.Warn("writes stopped")
- return ds, created
- case <-donec:
- lg.Info("writes done")
- return ds, created
- default:
- }
-
- key := fmt.Sprintf("%s%d", namePrefix, i)
-
- start := time.Now()
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- _, err := cli.
- CoreV1().
- Secrets(namespace).
- Create(ctx, &v1.Secret{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "v1",
- Kind: "Secret",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: key,
- Namespace: namespace,
- Labels: map[string]string{
- "name": key,
- },
- },
- Type: v1.SecretTypeOpaque,
- Data: map[string][]byte{key: []byte(val)},
- }, metav1.CreateOptions{})
- cancel()
- took := time.Since(start)
- tookMS := float64(took / time.Millisecond)
- writeRequestLatencyMs.Observe(tookMS)
- ds = append(ds, took)
- if err != nil {
- writeRequestsFailureTotal.Inc()
- lg.Warn("write secret failed", zap.String("namespace", namespace), zap.Error(err))
- } else {
- writeRequestsSuccessTotal.Inc()
- created = append(created, key)
- if i%20 == 0 {
- lg.Info("wrote secret", zap.Int("iteration", i), zap.String("namespace", namespace))
- }
- }
- }
- return ds, created
-}
-
-func startReads(lg *zap.Logger, cli *kubernetes.Clientset, timeout time.Duration, namespace string, created []string, stopc chan struct{}, donec chan struct{}) (ds metrics.Durations) {
- lg.Info("starting startReads", zap.Int("created-secrets", len(created)))
- ds = make(metrics.Durations, 0, 20000)
-
- for i, key := range created {
- select {
- case <-stopc:
- lg.Warn("reads stopped")
- return
- case <-donec:
- lg.Info("reads done")
- return
- default:
- }
-
- start := time.Now()
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- _, err := cli.
- CoreV1().
- Secrets(namespace).
- Get(ctx, key, metav1.GetOptions{})
- cancel()
- took := time.Since(start)
- tookMS := float64(took / time.Millisecond)
- readRequestLatencyMs.Observe(tookMS)
- ds = append(ds, took)
- if err != nil {
- readRequestsFailureTotal.Inc()
- lg.Warn("read secret failed", zap.String("namespace", namespace), zap.Error(err))
- } else {
- readRequestsSuccessTotal.Inc()
- if i%20 == 0 {
- lg.Info("read secret", zap.Int("iteration", i), zap.String("namespace", namespace))
- }
- }
- }
- return ds
-}
diff --git a/eks/stresser/local/stresser.go b/eks/stresser/local/stresser.go
deleted file mode 100644
index b5746fb61..000000000
--- a/eks/stresser/local/stresser.go
+++ /dev/null
@@ -1,547 +0,0 @@
-// Package local implements cluster local load tests.
-// ref. https://github.com/kubernetes/perf-tests
-package local
-
-import (
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "io/ioutil"
- "path"
- "reflect"
- "sort"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/eks/stresser"
- eks_tester "github.com/aws/aws-k8s-tester/eks/tester"
- "github.com/aws/aws-k8s-tester/eksconfig"
- "github.com/aws/aws-k8s-tester/pkg/aws/cw"
- aws_s3 "github.com/aws/aws-k8s-tester/pkg/aws/s3"
- "github.com/aws/aws-k8s-tester/pkg/fileutil"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/metrics"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/service/cloudwatch"
- "github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
- "github.com/aws/aws-sdk-go/service/s3"
- "github.com/aws/aws-sdk-go/service/s3/s3iface"
- "go.uber.org/zap"
- apierrs "k8s.io/apimachinery/pkg/api/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-// Config defines stresser configuration.
-// ref. https://github.com/kubernetes/perf-tests
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
- S3API s3iface.S3API
- CWAPI cloudwatchiface.CloudWatchAPI
-}
-
-// TODO: use kubemark
-// nodelease.NewController, kubemark.GetHollowKubeletConfig
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-func New(cfg Config) eks_tester.Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
-}
-
-func (ts *tester) Create() (err error) {
- if !ts.cfg.EKSConfig.IsEnabledAddOnStresserLocal() {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
- if ts.cfg.EKSConfig.AddOnStresserLocal.Created {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Create", zap.String("tester", pkgName))
- ts.cfg.EKSConfig.AddOnStresserLocal.Created = true
- ts.cfg.EKSConfig.Sync()
- createStart := time.Now()
- defer func() {
- createEnd := time.Now()
- ts.cfg.EKSConfig.AddOnStresserLocal.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- if err := k8s_client.CreateNamespace(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnStresserLocal.Namespace,
- ); err != nil {
- return err
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- nss, err := ts.cfg.K8SClient.KubernetesClientSet().CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("list namespaces failed", zap.Error(err))
- return err
- }
- ns := make([]string, 0, len(nss.Items))
- for _, nv := range nss.Items {
- ns = append(ns, nv.GetName())
- }
-
- loader := stresser.New(stresser.Config{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Stopc: ts.cfg.Stopc,
- S3API: ts.cfg.S3API,
- S3BucketName: ts.cfg.EKSConfig.S3.BucketName,
- Client: ts.cfg.K8SClient,
- ClientTimeout: ts.cfg.EKSConfig.ClientTimeout,
- Deadline: time.Now().Add(ts.cfg.EKSConfig.AddOnStresserLocal.Duration),
- NamespaceWrite: ts.cfg.EKSConfig.AddOnStresserLocal.Namespace,
- NamespacesRead: ns,
- ObjectSize: ts.cfg.EKSConfig.AddOnStresserLocal.ObjectSize,
- ListLimit: ts.cfg.EKSConfig.AddOnStresserLocal.ListLimit,
- RequestsRawWritesJSONPath: ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawWritesJSONPath,
- RequestsRawWritesJSONS3Key: ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawWritesJSONS3Key,
- RequestsSummaryWritesJSONPath: ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesJSONPath,
- RequestsSummaryWritesJSONS3Key: ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesJSONS3Key,
- RequestsSummaryWritesTablePath: ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesTablePath,
- RequestsSummaryWritesTableS3Key: ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesTableS3Key,
- RequestsRawReadsJSONPath: ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawReadsJSONPath,
- RequestsRawReadsJSONS3Key: ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawReadsJSONS3Key,
- RequestsSummaryReadsJSONPath: ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsJSONPath,
- RequestsSummaryReadsJSONS3Key: ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsJSONS3Key,
- RequestsSummaryReadsTablePath: ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsTablePath,
- RequestsSummaryReadsTableS3Key: ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsTableS3Key,
- })
- loader.Start()
-
- var curWriteLatencies metrics.Durations
- var curReadLatencies metrics.Durations
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("cluster stresser aborted")
- loader.Stop()
- curWriteLatencies, ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWrites, curReadLatencies, ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReads, err = loader.CollectMetrics()
- ts.cfg.EKSConfig.Sync()
- if err != nil {
- ts.cfg.Logger.Warn("failed to get metrics", zap.Error(err))
- }
- return err
-
- case <-time.After(ts.cfg.EKSConfig.AddOnStresserLocal.Duration):
- ts.cfg.Logger.Info("completing load testing", zap.Duration("duration", ts.cfg.EKSConfig.AddOnStresserLocal.Duration))
- loader.Stop()
- curWriteLatencies, ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWrites, curReadLatencies, ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReads, err = loader.CollectMetrics()
- ts.cfg.EKSConfig.Sync()
- if err != nil {
- ts.cfg.Logger.Warn("failed to get metrics", zap.Error(err))
- return err
- }
-
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("cluster stresser aborted")
- return nil
- case <-time.After(30 * time.Second):
- }
- }
-
- if err = ts.checkResults(curWriteLatencies, curReadLatencies); err != nil {
- return err
- }
- if err = ts.publishResults(); err != nil {
- return err
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) Delete() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnStresserLocal() {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnStresserLocal.Created {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Delete", zap.String("tester", pkgName))
- deleteStart := time.Now()
- defer func() {
- deleteEnd := time.Now()
- ts.cfg.EKSConfig.AddOnStresserLocal.TimeFrameDelete = timeutil.NewTimeFrame(deleteStart, deleteEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- var errs []string
-
- if err := k8s_client.DeleteNamespaceAndWait(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnStresserLocal.Namespace,
- k8s_client.DefaultNamespaceDeletionInterval,
- k8s_client.DefaultNamespaceDeletionTimeout,
- k8s_client.WithForceDelete(true),
- ); err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- errs = append(errs, fmt.Sprintf("failed to delete stresser namespace (%v)", err))
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- ts.cfg.EKSConfig.AddOnStresserLocal.Created = false
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// 1. if previous summary exists, download and compare
-// 2. upload new summary and overwrite the previous s3 key
-func (ts *tester) checkResults(curWriteLatencies metrics.Durations, curReadLatencies metrics.Durations) (err error) {
- curTS := time.Now().UTC().Format(time.RFC3339Nano)
- ts.cfg.Logger.Info("checking results", zap.String("timestamp", curTS))
-
- s3Objects := make([]*s3.Object, 0)
- if ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesCompareS3Dir != "" {
- s3Objects, err = aws_s3.ListInDescendingLastModified(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Clean(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesCompareS3Dir)+"/",
- )
- }
- canCompare := len(s3Objects) > 0 && err == nil
- if canCompare {
- reqSummaryS3Key := aws.StringValue(s3Objects[0].Key)
- durRawS3Key := path.Join(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawWritesCompareS3Dir, path.Base(reqSummaryS3Key))
-
- var prevSummary metrics.RequestsSummary
- prevSummary, err = metrics.DownloadRequestsSummaryFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, reqSummaryS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesCompare, err = metrics.CompareRequestsSummary(prevSummary, ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWrites)
- if err != nil {
- ts.cfg.Logger.Warn("failed to compare results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesCompareJSONPath, []byte(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesCompare.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesCompareJSONS3Key,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesCompareJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesCompareTablePath, []byte(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesCompare.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesCompareTableS3Key,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesCompareTablePath,
- ); err != nil {
- return err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryWritesCompare:\n%s\n", ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesCompare.Table())
-
- var prevDurations metrics.Durations
- prevDurations, err = metrics.DownloadDurationsFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, durRawS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- prevDurationsWithLabels := metrics.LabelDurations(prevDurations, prevSummary.TestID)
- curDurationsWithLabels := metrics.LabelDurations(curWriteLatencies, ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWrites.TestID)
- allDurationsWithLabels := append(prevDurationsWithLabels, curDurationsWithLabels...)
- sortStart := time.Now()
- ts.cfg.Logger.Info("sorting before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- )
- sort.Sort(allDurationsWithLabels)
- ts.cfg.Logger.Info("sorted before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- zap.String("took", time.Since(sortStart).String()),
- )
- allDataJSON, err := json.Marshal(allDurationsWithLabels)
- if err != nil {
- ts.cfg.Logger.Warn("failed to marshal results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawWritesCompareAllJSONPath, []byte(allDataJSON), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawWritesCompareAllJSONS3Key,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawWritesCompareAllJSONPath,
- ); err != nil {
- return err
- }
- if err = allDurationsWithLabels.CSV(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawWritesCompareAllCSVPath); err != nil {
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawWritesCompareAllCSVS3Key,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawWritesCompareAllCSVPath,
- ); err != nil {
- return err
- }
- } else {
- ts.cfg.Logger.Warn("previous writes summary not found; skipping comparison", zap.Error(err))
- }
- ts.cfg.Logger.Info("uploading new writes summary to s3 bucket to overwrite the previous")
- if fileutil.Exist(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawWritesJSONPath) {
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawWritesCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawWritesJSONPath,
- ); err != nil {
- return err
- }
- }
- if fileutil.Exist(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesJSONPath) {
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWritesJSONPath,
- ); err != nil {
- return err
- }
- }
-
- s3Objects = make([]*s3.Object, 0)
- if ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsCompareS3Dir != "" {
- s3Objects, err = aws_s3.ListInDescendingLastModified(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Clean(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsCompareS3Dir)+"/",
- )
- }
- canCompare = len(s3Objects) > 0 && err == nil
- if canCompare {
- reqSummaryS3Key := aws.StringValue(s3Objects[0].Key)
- durRawS3Key := path.Join(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawReadsCompareS3Dir, path.Base(reqSummaryS3Key))
-
- var prevSummary metrics.RequestsSummary
- prevSummary, err = metrics.DownloadRequestsSummaryFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, reqSummaryS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsCompare, err = metrics.CompareRequestsSummary(prevSummary, ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReads)
- if err != nil {
- ts.cfg.Logger.Warn("failed to compare results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsCompareJSONPath, []byte(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsCompare.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsCompareJSONS3Key,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsCompareJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsCompareTablePath, []byte(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsCompare.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsCompareTableS3Key,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsCompareTablePath,
- ); err != nil {
- return err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryReadsCompare:\n%s\n", ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsCompare.Table())
-
- var prevDurations metrics.Durations
- prevDurations, err = metrics.DownloadDurationsFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, durRawS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- prevDurationsWithLabels := metrics.LabelDurations(prevDurations, prevSummary.TestID)
- curDurationsWithLabels := metrics.LabelDurations(curWriteLatencies, ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReads.TestID)
- allDurationsWithLabels := append(prevDurationsWithLabels, curDurationsWithLabels...)
- sortStart := time.Now()
- ts.cfg.Logger.Info("sorting before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- )
- sort.Sort(allDurationsWithLabels)
- ts.cfg.Logger.Info("sorted before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- zap.String("took", time.Since(sortStart).String()),
- )
- allDataJSON, err := json.Marshal(allDurationsWithLabels)
- if err != nil {
- ts.cfg.Logger.Warn("failed to marshal results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawReadsCompareAllJSONPath, []byte(allDataJSON), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawReadsCompareAllJSONS3Key,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawReadsCompareAllJSONPath,
- ); err != nil {
- return err
- }
- if err = allDurationsWithLabels.CSV(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawReadsCompareAllCSVPath); err != nil {
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawReadsCompareAllCSVS3Key,
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawReadsCompareAllCSVPath,
- ); err != nil {
- return err
- }
- } else {
- ts.cfg.Logger.Warn("previous writes summary not found; skipping comparison", zap.Error(err))
- }
- ts.cfg.Logger.Info("uploading new reads summary to s3 bucket to overwrite the previous")
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawReadsCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsRawReadsJSONPath,
- ); err != nil {
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReadsJSONPath,
- ); err != nil {
- return err
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) publishResults() (err error) {
- tv := aws.Time(time.Now().UTC())
- datums := make([]*cloudwatch.MetricDatum, 0)
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-local-writes-latency-p50"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWrites.LantencyP50.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-local-writes-latency-p90"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWrites.LantencyP90.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- MetricName: aws.String("add-on-stresser-local-writes-latency-p99"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWrites.LantencyP99.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-local-writes-latency-p999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWrites.LantencyP999.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- MetricName: aws.String("add-on-stresser-local-writes-latency-p9999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryWrites.LantencyP9999.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-local-reads-latency-p50"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReads.LantencyP50.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-local-reads-latency-p90"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReads.LantencyP90.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-local-reads-latency-p99"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReads.LantencyP99.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-local-reads-latency-p999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReads.LantencyP999.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-local-reads-latency-p9999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserLocal.RequestsSummaryReads.LantencyP9999.Milliseconds())),
- })
- return cw.PutData(ts.cfg.Logger, ts.cfg.CWAPI, ts.cfg.EKSConfig.CWNamespace, 20, datums...)
-}
diff --git a/eks/stresser/remote/stresser.go b/eks/stresser/remote/stresser.go
deleted file mode 100644
index 4771c7ad2..000000000
--- a/eks/stresser/remote/stresser.go
+++ /dev/null
@@ -1,1480 +0,0 @@
-// Package remote implements cluster remote load tests.
-// ref. https://github.com/kubernetes/perf-tests
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-package remote
-
-import (
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "io/ioutil"
- "os"
- "path"
- "path/filepath"
- "reflect"
- "sort"
- "strings"
- "time"
-
- eks_tester "github.com/aws/aws-k8s-tester/eks/tester"
- "github.com/aws/aws-k8s-tester/eksconfig"
- "github.com/aws/aws-k8s-tester/pkg/aws/cw"
- aws_ecr "github.com/aws/aws-k8s-tester/pkg/aws/ecr"
- aws_s3 "github.com/aws/aws-k8s-tester/pkg/aws/s3"
- "github.com/aws/aws-k8s-tester/pkg/fileutil"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/metrics"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/service/cloudwatch"
- "github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
- "github.com/aws/aws-sdk-go/service/ecr/ecriface"
- "github.com/aws/aws-sdk-go/service/s3"
- "github.com/aws/aws-sdk-go/service/s3/s3iface"
- "github.com/dustin/go-humanize"
- "go.uber.org/zap"
- batchv1 "k8s.io/api/batch/v1"
- v1 "k8s.io/api/core/v1"
- rbacv1 "k8s.io/api/rbac/v1"
- apierrs "k8s.io/apimachinery/pkg/api/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/utils/exec"
- "sigs.k8s.io/yaml"
-)
-
-// Config defines stresser configuration.
-// ref. https://github.com/kubernetes/perf-tests
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
- S3API s3iface.S3API
- CWAPI cloudwatchiface.CloudWatchAPI
- ECRAPI ecriface.ECRAPI
-}
-
-// TODO: use kubemark
-// nodelease.NewController, kubemark.GetHollowKubeletConfig
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-func New(cfg Config) eks_tester.Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
- ecrImage string
-}
-
-func (ts *tester) Create() (err error) {
- if !ts.cfg.EKSConfig.IsEnabledAddOnStresserRemote() {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
- if ts.cfg.EKSConfig.AddOnStresserRemote.Created {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Create", zap.String("tester", pkgName))
- ts.cfg.EKSConfig.AddOnStresserRemote.Created = true
- ts.cfg.EKSConfig.Sync()
- createStart := time.Now()
- defer func() {
- createEnd := time.Now()
- ts.cfg.EKSConfig.AddOnStresserRemote.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- if ts.ecrImage, _, err = aws_ecr.Check(
- ts.cfg.Logger,
- ts.cfg.ECRAPI,
- ts.cfg.EKSConfig.Partition,
- ts.cfg.EKSConfig.AddOnStresserRemote.RepositoryAccountID,
- ts.cfg.EKSConfig.AddOnStresserRemote.RepositoryRegion,
- ts.cfg.EKSConfig.AddOnStresserRemote.RepositoryName,
- ts.cfg.EKSConfig.AddOnStresserRemote.RepositoryImageTag,
- ); err != nil {
- return err
- }
- if err = k8s_client.CreateNamespace(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnStresserRemote.Namespace,
- ); err != nil {
- return err
- }
- if err = ts.createServiceAccount(); err != nil {
- return err
- }
- if err = ts.createRBACClusterRole(); err != nil {
- return err
- }
- if err = ts.createRBACClusterRoleBinding(); err != nil {
- return err
- }
- if err = ts.createConfigMap(); err != nil {
- return err
- }
-
- if err = ts.createJob(); err != nil {
- return err
- }
- timeout := 15*time.Minute + ts.cfg.EKSConfig.AddOnStresserRemote.Duration
- if timeout > 5*time.Hour {
- timeout = 5 * time.Hour
- }
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- var pods []v1.Pod
- _, pods, err = k8s_client.WaitForJobCompletes(
- ctx,
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.Stopc,
- ts.cfg.K8SClient,
- time.Minute+ts.cfg.EKSConfig.AddOnStresserRemote.Duration/2,
- 15*time.Second,
- ts.cfg.EKSConfig.AddOnStresserRemote.Namespace,
- stresserJobName,
- ts.cfg.EKSConfig.AddOnStresserRemote.Completes,
- k8s_client.WithQueryFunc(func() {
- descArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnStresserRemote.Namespace,
- "describe",
- "job",
- stresserJobName,
- }
- descCmd := strings.Join(descArgs, " ")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- descOutput, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe job' failed", zap.Error(err))
- }
- out := string(descOutput)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\n\"%s\" output:\n\n%s\n\n", descCmd, out)
-
- argsLogs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnStresserRemote.Namespace,
- "logs",
- "--selector=job-name=" + stresserJobName,
- "--timestamps",
- "--tail=10",
- }
- cmdLogs := strings.Join(argsLogs, " ")
- ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
- logsOutput, err := exec.New().CommandContext(ctx, argsLogs[0], argsLogs[1:]...).CombinedOutput()
- cancel()
- out = string(logsOutput)
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl logs' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\":\n%s\n", cmdLogs, out)
- }),
- k8s_client.WithPodFunc(func(pod v1.Pod) {
- switch pod.Status.Phase {
- case v1.PodFailed:
- ts.cfg.Logger.Warn("pod failed",
- zap.String("namespace", pod.Namespace),
- zap.String("pod-name", pod.Name),
- zap.String("pod-status-phase", fmt.Sprintf("%v", pod.Status.Phase)),
- )
- descArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + pod.Namespace,
- "describe",
- "pod",
- pod.Name,
- }
- descCmd := strings.Join(descArgs, " ")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- cmdOutput, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe job' failed", zap.Error(err))
- }
- out := string(cmdOutput)
- fmt.Fprintf(ts.cfg.LogWriter, "\"%s\" output:\n\n%s\n\n", descCmd, out)
-
- logsArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + pod.Namespace,
- "logs",
- fmt.Sprintf("pod/%s", pod.Name),
- "--timestamps",
- }
- logsCmd := strings.Join(logsArgs, " ")
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- cmdOutput, err = exec.New().CommandContext(ctx, logsArgs[0], logsArgs[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl logs' failed", zap.Error(err))
- }
- out = string(cmdOutput)
- fmt.Fprintf(ts.cfg.LogWriter, "\"%s\" output:\n\n%s\n\n", logsCmd, out)
- }
- }),
- )
- cancel()
- if err != nil {
- return err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n")
- for _, item := range pods {
- fmt.Fprintf(ts.cfg.LogWriter, "Job Pod %q: %q\n", item.Name, item.Status.Phase)
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n")
-
- if err = ts.checkResults(); err == nil {
- return err
- }
- if err = ts.publishResults(); err != nil {
- return err
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) Delete() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnStresserRemote() {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnStresserRemote.Created {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Delete", zap.String("tester", pkgName))
- deleteStart := time.Now()
- defer func() {
- deleteEnd := time.Now()
- ts.cfg.EKSConfig.AddOnStresserRemote.TimeFrameDelete = timeutil.NewTimeFrame(deleteStart, deleteEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- var errs []string
-
- if err := ts.deleteJob(); err != nil {
- errs = append(errs, err.Error())
- }
- time.Sleep(2 * time.Minute)
-
- if err := ts.deleteConfigMap(); err != nil {
- errs = append(errs, err.Error())
- }
- if err := ts.deleteRBACClusterRoleBinding(); err != nil {
- errs = append(errs, err.Error())
- }
- if err := ts.deleteRBACClusterRole(); err != nil {
- errs = append(errs, err.Error())
- }
- if err := ts.deleteServiceAccount(); err != nil {
- errs = append(errs, err.Error())
- }
-
- getAllArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnStresserRemote.Namespace,
- "get",
- "all",
- }
- getAllCmd := strings.Join(getAllArgs, " ")
-
- if err := k8s_client.DeleteNamespaceAndWait(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnStresserRemote.Namespace,
- k8s_client.DefaultNamespaceDeletionInterval,
- k8s_client.DefaultNamespaceDeletionTimeout,
- k8s_client.WithQueryFunc(func() {
- fmt.Fprintf(ts.cfg.LogWriter, "\n")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, getAllArgs[0], getAllArgs[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl get all' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getAllCmd, out)
- }),
- k8s_client.WithForceDelete(true),
- ); err != nil {
- // TODO: error...
- // errs = append(errs, fmt.Sprintf("failed to delete stresser namespace (%v)", err))
- ts.cfg.Logger.Warn("failed to delete namespace", zap.Error(err))
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- ts.cfg.EKSConfig.AddOnStresserRemote.Created = false
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-const (
- stresserServiceAccountName = "stresser-remote-service-account"
- stresserRBACRoleName = "stresser-remote-rbac-role"
- stresserRBACClusterRoleBindingName = "stresser-remote-rbac-role-binding"
- stresserKubeConfigConfigMapName = "stresser-remote-kubeconfig-configmap"
- stresserKubeConfigConfigMapFileName = "stresser-remote-kubeconfig-configmap.yaml"
- stresserAppName = "stresser-remote-app"
- stresserJobName = "stresser-remote-job"
-)
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) createServiceAccount() error {
- ts.cfg.Logger.Info("creating stresser ServiceAccount")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- ServiceAccounts(ts.cfg.EKSConfig.AddOnStresserRemote.Namespace).
- Create(
- ctx,
- &v1.ServiceAccount{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "v1",
- Kind: "ServiceAccount",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: stresserServiceAccountName,
- Namespace: ts.cfg.EKSConfig.AddOnStresserRemote.Namespace,
- Labels: map[string]string{
- "app.kubernetes.io/name": stresserAppName,
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create stresser ServiceAccount (%v)", err)
- }
-
- ts.cfg.Logger.Info("created stresser ServiceAccount")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) deleteServiceAccount() error {
- ts.cfg.Logger.Info("deleting stresser ServiceAccount")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- ServiceAccounts(ts.cfg.EKSConfig.AddOnStresserRemote.Namespace).
- Delete(
- ctx,
- stresserServiceAccountName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete stresser ServiceAccount (%v)", err)
- }
- ts.cfg.Logger.Info("deleted stresser ServiceAccount", zap.Error(err))
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) createRBACClusterRole() error {
- ts.cfg.Logger.Info("creating stresser RBAC ClusterRole")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.K8SClient.KubernetesClientSet().
- RbacV1().
- ClusterRoles().
- Create(
- ctx,
- &rbacv1.ClusterRole{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "rbac.authorization.k8s.io/v1",
- Kind: "ClusterRole",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: stresserRBACRoleName,
- Namespace: "default",
- Labels: map[string]string{
- "app.kubernetes.io/name": stresserAppName,
- },
- },
- Rules: []rbacv1.PolicyRule{
- {
- APIGroups: []string{
- "*",
- },
- Resources: []string{
- "configmaps",
- "leases",
- "nodes",
- "pods",
- "secrets",
- "services",
- "namespaces",
- "stresser",
- "endpoints",
- "events",
- "ingresses",
- "ingresses/status",
- "services",
- "jobs",
- "cronjobs",
- },
- Verbs: []string{
- "create",
- "get",
- "list",
- "update",
- "watch",
- "patch",
- },
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create stresser RBAC ClusterRole (%v)", err)
- }
-
- ts.cfg.Logger.Info("created stresser RBAC ClusterRole")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) deleteRBACClusterRole() error {
- ts.cfg.Logger.Info("deleting stresser RBAC ClusterRole")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- RbacV1().
- ClusterRoles().
- Delete(
- ctx,
- stresserRBACRoleName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete stresser RBAC ClusterRole (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted stresser RBAC ClusterRole", zap.Error(err))
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) createRBACClusterRoleBinding() error {
- ts.cfg.Logger.Info("creating stresser RBAC ClusterRoleBinding")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.K8SClient.KubernetesClientSet().
- RbacV1().
- ClusterRoleBindings().
- Create(
- ctx,
- &rbacv1.ClusterRoleBinding{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "rbac.authorization.k8s.io/v1",
- Kind: "ClusterRoleBinding",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: stresserRBACClusterRoleBindingName,
- Namespace: "default",
- Labels: map[string]string{
- "app.kubernetes.io/name": stresserAppName,
- },
- },
- RoleRef: rbacv1.RoleRef{
- APIGroup: "rbac.authorization.k8s.io",
- Kind: "ClusterRole",
- Name: stresserRBACRoleName,
- },
- Subjects: []rbacv1.Subject{
- {
- APIGroup: "",
- Kind: "ServiceAccount",
- Name: stresserServiceAccountName,
- Namespace: ts.cfg.EKSConfig.AddOnStresserRemote.Namespace,
- },
- { // https://kubernetes.io/docs/reference/access-authn-authz/rbac/
- APIGroup: "rbac.authorization.k8s.io",
- Kind: "User",
- Name: "system:node",
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create stresser RBAC ClusterRoleBinding (%v)", err)
- }
-
- ts.cfg.Logger.Info("created stresser RBAC ClusterRoleBinding")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) deleteRBACClusterRoleBinding() error {
- ts.cfg.Logger.Info("deleting stresser RBAC ClusterRoleBinding")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- RbacV1().
- ClusterRoleBindings().
- Delete(
- ctx,
- stresserRBACClusterRoleBindingName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete stresser RBAC ClusterRoleBinding (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted stresser RBAC ClusterRoleBinding", zap.Error(err))
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// TODO: no need to mount kubeconfig since we are creating in-cluster kubernetes client
-
-func (ts *tester) createConfigMap() error {
- ts.cfg.Logger.Info("creating config map")
-
- b, err := ioutil.ReadFile(ts.cfg.EKSConfig.KubeConfigPath)
- if err != nil {
- return err
- }
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err = ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- ConfigMaps(ts.cfg.EKSConfig.AddOnStresserRemote.Namespace).
- Create(
- ctx,
- &v1.ConfigMap{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "v1",
- Kind: "ConfigMap",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: stresserKubeConfigConfigMapName,
- Namespace: ts.cfg.EKSConfig.AddOnStresserRemote.Namespace,
- Labels: map[string]string{
- "name": stresserKubeConfigConfigMapName,
- },
- },
- Data: map[string]string{
- stresserKubeConfigConfigMapFileName: string(b),
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return err
- }
-
- ts.cfg.Logger.Info("created config map")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) deleteConfigMap() error {
- ts.cfg.Logger.Info("deleting config map")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- ConfigMaps(ts.cfg.EKSConfig.AddOnStresserRemote.Namespace).
- Delete(
- ctx,
- stresserKubeConfigConfigMapName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil {
- return err
- }
- ts.cfg.Logger.Info("deleted config map")
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) createJob() (err error) {
- obj, b, err := ts.createObject()
- if err != nil {
- return err
- }
-
- ts.cfg.Logger.Info("creating Job",
- zap.String("name", stresserJobName),
- zap.String("object-size", humanize.Bytes(uint64(len(b)))),
- )
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err = ts.cfg.K8SClient.KubernetesClientSet().
- BatchV1().
- Jobs(ts.cfg.EKSConfig.AddOnStresserRemote.Namespace).
- Create(ctx, &obj, metav1.CreateOptions{})
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create Job (%v)", err)
- }
-
- ts.cfg.Logger.Info("created Job",
- zap.String("stresser-duration", ts.cfg.EKSConfig.AddOnStresserRemote.Duration.String()),
- )
- return nil
-}
-
-func (ts *tester) createObject() (batchv1.Job, string, error) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- nss, err := ts.cfg.K8SClient.KubernetesClientSet().CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("list namespaces failed", zap.Error(err))
- return batchv1.Job{}, "", err
- }
- ns := make([]string, 0, len(nss.Items))
- for _, nv := range nss.Items {
- ns = append(ns, nv.GetName())
- }
-
- // "/opt/"+stresserKubeConfigConfigMapFileName,
- // do not specify "kubeconfig", and use in-cluster config via "pkg/k8s-client"
- // otherwise, error "namespaces is forbidden: User "system:node:ip-192-168-84..."
- // ref. https://github.com/kubernetes/client-go/blob/master/examples/in-cluster-client-configuration/main.go
- testerCmd := fmt.Sprintf(`/aws-k8s-tester eks create stresser --partition=%s --region=%s --s3-bucket-name=%s --clients=%d --client-qps=%f --client-burst=%d --client-timeout=%s --object-size=%d --list-limit=%d --duration=%s --namespace-write=%s --namespaces-read=%s --requests-raw-writes-json-s3-dir=%s --requests-summary-writes-json-s3-dir=%s --requests-summary-writes-table-s3-dir=%s --requests-raw-reads-json-s3-dir=%s --requests-summary-reads-json-s3-dir=%s --requests-summary-reads-table-s3-dir=%s --writes-output-name-prefix=%s --reads-output-name-prefix=%s`,
- ts.cfg.EKSConfig.Partition,
- ts.cfg.EKSConfig.Region,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.Clients,
- ts.cfg.EKSConfig.ClientQPS,
- ts.cfg.EKSConfig.ClientBurst,
- ts.cfg.EKSConfig.ClientTimeout,
- ts.cfg.EKSConfig.AddOnStresserRemote.ObjectSize,
- ts.cfg.EKSConfig.AddOnStresserRemote.ListLimit,
- ts.cfg.EKSConfig.AddOnStresserRemote.Duration,
- ts.cfg.EKSConfig.AddOnStresserRemote.Namespace,
- strings.Join(ns, ","),
- path.Dir(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesJSONS3Key),
- path.Dir(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesJSONS3Key),
- path.Dir(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesTableS3Key),
- path.Dir(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawReadsJSONS3Key),
- path.Dir(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsJSONS3Key),
- path.Dir(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsTableS3Key),
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesOutputNamePrefix,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsOutputNamePrefix,
- )
-
- dirOrCreate := v1.HostPathDirectoryOrCreate
- podSpec := v1.PodTemplateSpec{
- ObjectMeta: metav1.ObjectMeta{
- Labels: map[string]string{
- "app.kubernetes.io/name": stresserAppName,
- },
- },
- Spec: v1.PodSpec{
- ServiceAccountName: stresserServiceAccountName,
-
- // spec.template.spec.restartPolicy: Unsupported value: "Always": supported values: "OnFailure", "Never"
- // ref. https://github.com/kubernetes/kubernetes/issues/54870
- RestartPolicy: v1.RestartPolicyNever,
-
- // TODO: set resource limits
- Containers: []v1.Container{
- {
- Name: stresserAppName,
- Image: ts.ecrImage,
- ImagePullPolicy: v1.PullAlways,
-
- Command: []string{
- "/bin/sh",
- "-ec",
- testerCmd,
- },
-
- // grant access "/dev/kmsg"
- SecurityContext: &v1.SecurityContext{
- Privileged: aws.Bool(true),
- },
-
- // ref. https://kubernetes.io/docs/concepts/cluster-administration/logging/
- VolumeMounts: []v1.VolumeMount{
- { // to execute
- Name: stresserKubeConfigConfigMapName,
- MountPath: "/opt",
- },
- { // to write
- Name: "varlog",
- MountPath: "/var/log",
- ReadOnly: false,
- },
- },
- },
- },
-
- // ref. https://kubernetes.io/docs/concepts/cluster-administration/logging/
- Volumes: []v1.Volume{
- { // to execute
- Name: stresserKubeConfigConfigMapName,
- VolumeSource: v1.VolumeSource{
- ConfigMap: &v1.ConfigMapVolumeSource{
- LocalObjectReference: v1.LocalObjectReference{
- Name: stresserKubeConfigConfigMapName,
- },
- DefaultMode: aws.Int32(0777),
- },
- },
- },
- { // to write
- Name: "varlog",
- VolumeSource: v1.VolumeSource{
- HostPath: &v1.HostPathVolumeSource{
- Path: "/var/log",
- Type: &dirOrCreate,
- },
- },
- },
- },
-
- NodeSelector: map[string]string{
- // do not deploy in fake nodes, obviously
- "NodeType": "regular",
- },
- },
- }
-
- jobObj := batchv1.Job{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "batch/v1",
- Kind: "Job",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: stresserJobName,
- Namespace: ts.cfg.EKSConfig.AddOnStresserRemote.Namespace,
- },
- Spec: batchv1.JobSpec{
- Completions: aws.Int32(int32(ts.cfg.EKSConfig.AddOnStresserRemote.Completes)),
- Parallelism: aws.Int32(int32(ts.cfg.EKSConfig.AddOnStresserRemote.Parallels)),
- Template: podSpec,
- // TODO: 'TTLSecondsAfterFinished' is still alpha
- // https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/
- },
- }
- b, err := yaml.Marshal(jobObj)
- return jobObj, string(b), err
-}
-
-func (ts *tester) deleteJob() (err error) {
- foreground := metav1.DeletePropagationForeground
- ts.cfg.Logger.Info("deleting Job", zap.String("name", stresserJobName))
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err = ts.cfg.
- K8SClient.KubernetesClientSet().
- BatchV1().
- Jobs(ts.cfg.EKSConfig.AddOnStresserRemote.Namespace).
- Delete(
- ctx,
- stresserJobName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err == nil {
- ts.cfg.Logger.Info("deleted Job", zap.String("name", stresserJobName))
- } else {
- ts.cfg.Logger.Warn("failed to delete Job", zap.Error(err))
- }
- return err
-}
-
-// 1. if previous summary exists, download and compare
-// 2. upload new summary and overwrite the previous s3 key
-func (ts *tester) checkResults() (err error) {
- curTS := time.Now().UTC().Format(time.RFC3339Nano)
- ts.cfg.Logger.Info("checking results", zap.String("timestamp", curTS))
-
- writesSummary := metrics.RequestsSummary{TestID: curTS}
- curWriteLatencies := make(metrics.Durations, 0, 20000)
- writesDirRaw := ""
- writesDirSummary := ""
-
- readsSummary := metrics.RequestsSummary{TestID: curTS}
- curReadLatencies := make(metrics.Durations, 0, 20000)
- readsDirRaw := ""
- readsDirSummary := ""
-
- writesDirRaw, err = aws_s3.DownloadDir(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Dir(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesJSONS3Key),
- )
- if err == nil {
- ts.cfg.Logger.Info("reading writes results raw",
- zap.String("writes-dir", writesDirRaw),
- zap.String("s3-dir", path.Dir(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesJSONS3Key)),
- )
- cnt := 0
- err = filepath.Walk(writesDirRaw, func(fpath string, info os.FileInfo, werr error) error {
- if werr != nil {
- return werr
- }
- if info.IsDir() {
- return nil
- }
- cnt++
- switch {
- case strings.HasSuffix(fpath, "-writes-raw.json"):
- b, err := ioutil.ReadFile(fpath)
- if err != nil {
- return fmt.Errorf("failed to open %q (%v)", fpath, err)
- }
- var r metrics.Durations
- if err = json.Unmarshal(b, &r); err != nil {
- return fmt.Errorf("failed to unmarshal %q (%s, %v)", fpath, string(b), err)
- }
- curWriteLatencies = append(curWriteLatencies, r...)
- }
- return nil
- })
- if err != nil || cnt == 0 {
- ts.cfg.Logger.Warn("failed to read writes results", zap.Int("file-count", cnt), zap.Error(err))
- os.RemoveAll(writesDirRaw)
- writesDirRaw = ""
- }
- }
- writesDirSummary, err = aws_s3.DownloadDir(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Dir(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesJSONS3Key),
- )
- if err == nil {
- ts.cfg.Logger.Info("reading writes results summary",
- zap.String("writes-dir", writesDirSummary),
- zap.String("s3-dir", path.Dir(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesJSONS3Key)),
- )
- cnt := 0
- err = filepath.Walk(writesDirSummary, func(fpath string, info os.FileInfo, werr error) error {
- if werr != nil {
- return werr
- }
- if info.IsDir() {
- return nil
- }
- cnt++
- switch {
- case strings.HasSuffix(fpath, "-writes-summary.json"):
- b, err := ioutil.ReadFile(fpath)
- if err != nil {
- return fmt.Errorf("failed to open %q (%v)", fpath, err)
- }
- var r metrics.RequestsSummary
- if err = json.Unmarshal(b, &r); err != nil {
- return fmt.Errorf("failed to unmarshal %q (%s, %v)", fpath, string(b), err)
- }
- writesSummary.SuccessTotal += r.SuccessTotal
- writesSummary.FailureTotal += r.FailureTotal
- if writesSummary.LatencyHistogram == nil || len(writesSummary.LatencyHistogram) == 0 {
- writesSummary.LatencyHistogram = r.LatencyHistogram
- } else {
- writesSummary.LatencyHistogram, err = metrics.MergeHistograms(writesSummary.LatencyHistogram, r.LatencyHistogram)
- if err != nil {
- return fmt.Errorf("failed to merge histograms (%v)", err)
- }
- }
- }
- return nil
- })
- if err != nil || cnt == 0 {
- ts.cfg.Logger.Warn("failed to read writes results", zap.Int("file-count", cnt), zap.Error(err))
- os.RemoveAll(writesDirSummary)
- writesDirSummary = ""
- }
- }
-
- readsDirRaw, err = aws_s3.DownloadDir(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Dir(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawReadsJSONS3Key),
- )
- if err == nil {
- ts.cfg.Logger.Info("reading reads results raw",
- zap.String("reads-dir", readsDirRaw),
- zap.String("s3-dir", path.Dir(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawReadsJSONS3Key)),
- )
- cnt := 0
- err = filepath.Walk(readsDirRaw, func(fpath string, info os.FileInfo, werr error) error {
- if werr != nil {
- return werr
- }
- if info.IsDir() {
- return nil
- }
- cnt++
- switch {
- case strings.HasSuffix(fpath, "-reads-raw.json"):
- b, err := ioutil.ReadFile(fpath)
- if err != nil {
- return fmt.Errorf("failed to open %q (%v)", fpath, err)
- }
- var r metrics.Durations
- if err = json.Unmarshal(b, &r); err != nil {
- return fmt.Errorf("failed to unmarshal %q (%s, %v)", fpath, string(b), err)
- }
- curReadLatencies = append(curReadLatencies, r...)
- }
- return nil
- })
- if err != nil || cnt == 0 {
- ts.cfg.Logger.Warn("failed to read reads results", zap.Int("file-count", cnt), zap.Error(err))
- os.RemoveAll(readsDirRaw)
- readsDirRaw = ""
- }
- }
- readsDirSummary, err = aws_s3.DownloadDir(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Dir(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsJSONS3Key),
- )
- if err == nil {
- ts.cfg.Logger.Info("reading reads results summary",
- zap.String("reads-dir", readsDirSummary),
- zap.String("s3-dir", path.Dir(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsJSONS3Key)),
- )
- cnt := 0
- err = filepath.Walk(readsDirSummary, func(fpath string, info os.FileInfo, werr error) error {
- if werr != nil {
- return werr
- }
- if info.IsDir() {
- return nil
- }
- cnt++
- switch {
- case strings.HasSuffix(fpath, "-reads-summary.json"):
- b, err := ioutil.ReadFile(fpath)
- if err != nil {
- return fmt.Errorf("failed to open %q (%v)", fpath, err)
- }
- var r metrics.RequestsSummary
- if err = json.Unmarshal(b, &r); err != nil {
- return fmt.Errorf("failed to unmarshal %q (%s, %v)", fpath, string(b), err)
- }
- readsSummary.SuccessTotal += r.SuccessTotal
- readsSummary.FailureTotal += r.FailureTotal
- if readsSummary.LatencyHistogram == nil || len(readsSummary.LatencyHistogram) == 0 {
- readsSummary.LatencyHistogram = r.LatencyHistogram
- } else {
- readsSummary.LatencyHistogram, err = metrics.MergeHistograms(readsSummary.LatencyHistogram, r.LatencyHistogram)
- if err != nil {
- return fmt.Errorf("failed to merge histograms (%v)", err)
- }
- }
- }
- return nil
- })
- if err != nil || cnt == 0 {
- ts.cfg.Logger.Warn("failed to read reads results", zap.Int("file-count", cnt), zap.Error(err))
- os.RemoveAll(readsDirSummary)
- readsDirSummary = ""
- }
- }
-
- sortStart := time.Now()
- ts.cfg.Logger.Info("sorting write latencies", zap.Int("data", len(curWriteLatencies)))
- sort.Sort(curWriteLatencies)
- ts.cfg.Logger.Info("sorted write latencies", zap.String("took", time.Since(sortStart).String()))
- writesSummary.LantencyP50 = curWriteLatencies.PickLantencyP50()
- writesSummary.LantencyP90 = curWriteLatencies.PickLantencyP90()
- writesSummary.LantencyP99 = curWriteLatencies.PickLantencyP99()
- writesSummary.LantencyP999 = curWriteLatencies.PickLantencyP999()
- writesSummary.LantencyP9999 = curWriteLatencies.PickLantencyP9999()
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWrites = writesSummary
- ts.cfg.EKSConfig.Sync()
-
- sortStart = time.Now()
- ts.cfg.Logger.Info("sorting read latencies", zap.Int("data", len(curReadLatencies)))
- sort.Sort(curReadLatencies)
- ts.cfg.Logger.Info("sorted read latencies", zap.String("took", time.Since(sortStart).String()))
- readsSummary.LantencyP50 = curReadLatencies.PickLantencyP50()
- readsSummary.LantencyP90 = curReadLatencies.PickLantencyP90()
- readsSummary.LantencyP99 = curReadLatencies.PickLantencyP99()
- readsSummary.LantencyP999 = curReadLatencies.PickLantencyP999()
- readsSummary.LantencyP9999 = curReadLatencies.PickLantencyP9999()
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReads = readsSummary
- ts.cfg.EKSConfig.Sync()
-
- wb, err := json.Marshal(curWriteLatencies)
- if err != nil {
- ts.cfg.Logger.Warn("failed to encode JSON", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesJSONPath, wb, 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesJSONS3Key,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesJSONPath, []byte(writesSummary.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesJSONS3Key,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesTablePath, []byte(writesSummary.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesTableS3Key,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesTablePath,
- ); err != nil {
- return err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryWrites:\n%s\n", writesSummary.Table())
-
- rb, err := json.Marshal(curReadLatencies)
- if err != nil {
- ts.cfg.Logger.Warn("failed to encode JSON", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawReadsJSONPath, rb, 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawReadsJSONS3Key,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawReadsJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsJSONPath, []byte(readsSummary.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsJSONS3Key,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsTablePath, []byte(readsSummary.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsTableS3Key,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsTablePath,
- ); err != nil {
- return err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryReads:\n%s\n", readsSummary.Table())
-
- s3Objects := make([]*s3.Object, 0)
- if ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesCompareS3Dir != "" {
- s3Objects, err = aws_s3.ListInDescendingLastModified(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Clean(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesCompareS3Dir)+"/",
- )
- }
- canCompare := len(s3Objects) > 0 && err == nil
- if canCompare {
- reqSummaryS3Key := aws.StringValue(s3Objects[0].Key)
- durRawS3Key := path.Join(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesCompareS3Dir, path.Base(reqSummaryS3Key))
-
- var prevSummary metrics.RequestsSummary
- prevSummary, err = metrics.DownloadRequestsSummaryFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, reqSummaryS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesCompare, err = metrics.CompareRequestsSummary(prevSummary, ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWrites)
- if err != nil {
- ts.cfg.Logger.Warn("failed to compare results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesCompareJSONPath, []byte(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesCompare.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesCompareJSONS3Key,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesCompareJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesCompareTablePath, []byte(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesCompare.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesCompareTableS3Key,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesCompareTablePath,
- ); err != nil {
- return err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryWritesCompare:\n%s\n", ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesCompare.Table())
-
- var prevDurations metrics.Durations
- prevDurations, err = metrics.DownloadDurationsFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, durRawS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- prevDurationsWithLabels := metrics.LabelDurations(prevDurations, prevSummary.TestID)
- curDurationsWithLabels := metrics.LabelDurations(curWriteLatencies, ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWrites.TestID)
- allDurationsWithLabels := append(prevDurationsWithLabels, curDurationsWithLabels...)
- sortStart := time.Now()
- ts.cfg.Logger.Info("sorting before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- )
- sort.Sort(allDurationsWithLabels)
- ts.cfg.Logger.Info("sorted before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- zap.String("took", time.Since(sortStart).String()),
- )
- allDataJSON, err := json.Marshal(allDurationsWithLabels)
- if err != nil {
- ts.cfg.Logger.Warn("failed to marshal results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesCompareAllJSONPath, []byte(allDataJSON), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesCompareAllJSONS3Key,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesCompareAllJSONPath,
- ); err != nil {
- return err
- }
- if err = allDurationsWithLabels.CSV(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesCompareAllCSVPath); err != nil {
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesCompareAllCSVS3Key,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesCompareAllCSVPath,
- ); err != nil {
- return err
- }
- } else {
- ts.cfg.Logger.Warn("previous writes summary not found; skipping comparison", zap.Error(err))
- }
- ts.cfg.Logger.Info("uploading new writes summary to s3 bucket to overwrite the previous")
- if fileutil.Exist(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesJSONPath) {
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawWritesJSONPath,
- ); err != nil {
- return err
- }
- }
- if fileutil.Exist(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesJSONPath) {
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWritesJSONPath,
- ); err != nil {
- return err
- }
- }
-
- s3Objects = make([]*s3.Object, 0)
- if ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsCompareS3Dir != "" {
- s3Objects, err = aws_s3.ListInDescendingLastModified(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Clean(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsCompareS3Dir)+"/",
- )
- }
- canCompare = len(s3Objects) > 0 && err == nil
- if canCompare {
- reqSummaryS3Key := aws.StringValue(s3Objects[0].Key)
- durRawS3Key := path.Join(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawReadsCompareS3Dir, path.Base(reqSummaryS3Key))
-
- var prevSummary metrics.RequestsSummary
- prevSummary, err = metrics.DownloadRequestsSummaryFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, reqSummaryS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsCompare, err = metrics.CompareRequestsSummary(prevSummary, ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReads)
- if err != nil {
- ts.cfg.Logger.Warn("failed to compare results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsCompareJSONPath, []byte(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsCompare.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsCompareJSONS3Key,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsCompareJSONPath,
- ); err != nil {
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsCompareTablePath, []byte(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsCompare.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsCompareTableS3Key,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsCompareTablePath,
- ); err != nil {
- return err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryReadsCompare:\n%s\n", ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsCompare.Table())
-
- var prevDurations metrics.Durations
- prevDurations, err = metrics.DownloadDurationsFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, durRawS3Key)
- if err != nil {
- ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
- return err
- }
- prevDurationsWithLabels := metrics.LabelDurations(prevDurations, prevSummary.TestID)
- curDurationsWithLabels := metrics.LabelDurations(curWriteLatencies, ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReads.TestID)
- allDurationsWithLabels := append(prevDurationsWithLabels, curDurationsWithLabels...)
- sortStart := time.Now()
- ts.cfg.Logger.Info("sorting before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- )
- sort.Sort(allDurationsWithLabels)
- ts.cfg.Logger.Info("sorted before and after durations with label",
- zap.Int("before-data-points", len(prevDurationsWithLabels)),
- zap.Int("after-data-points", len(curDurationsWithLabels)),
- zap.Int("total-points", len(allDurationsWithLabels)),
- zap.String("took", time.Since(sortStart).String()),
- )
- allDataJSON, err := json.Marshal(allDurationsWithLabels)
- if err != nil {
- ts.cfg.Logger.Warn("failed to marshal results", zap.Error(err))
- return err
- }
- if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawReadsCompareAllJSONPath, []byte(allDataJSON), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawReadsCompareAllJSONS3Key,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawReadsCompareAllJSONPath,
- ); err != nil {
- return err
- }
- if err = allDurationsWithLabels.CSV(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawReadsCompareAllCSVPath); err != nil {
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawReadsCompareAllCSVS3Key,
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawReadsCompareAllCSVPath,
- ); err != nil {
- return err
- }
- } else {
- ts.cfg.Logger.Warn("previous writes summary not found; skipping comparison", zap.Error(err))
- }
- ts.cfg.Logger.Info("uploading new reads summary to s3 bucket to overwrite the previous")
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawReadsCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsRawReadsJSONPath,
- ); err != nil {
- return err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.EKSConfig.S3.BucketName,
- path.Join(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsCompareS3Dir, curTS),
- ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReadsJSONPath,
- ); err != nil {
- return err
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) publishResults() (err error) {
- tv := aws.Time(time.Now().UTC())
- datums := make([]*cloudwatch.MetricDatum, 0)
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-remote-writes-latency-p50"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWrites.LantencyP50.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-remote-writes-latency-p90"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWrites.LantencyP90.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-remote-writes-latency-p99"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWrites.LantencyP99.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-remote-writes-latency-p999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWrites.LantencyP999.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-remote-writes-latency-p9999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryWrites.LantencyP9999.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-remote-reads-latency-p50"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReads.LantencyP50.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-remote-reads-latency-p90"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReads.LantencyP90.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-remote-reads-latency-p99"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReads.LantencyP99.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-remote-reads-latency-p999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReads.LantencyP999.Milliseconds())),
- })
- datums = append(datums, &cloudwatch.MetricDatum{
- Timestamp: tv,
- MetricName: aws.String("add-on-stresser-remote-reads-latency-p9999"),
- Unit: aws.String(cloudwatch.StandardUnitMilliseconds),
- Value: aws.Float64(float64(ts.cfg.EKSConfig.AddOnStresserRemote.RequestsSummaryReads.LantencyP9999.Milliseconds())),
- })
- return cw.PutData(ts.cfg.Logger, ts.cfg.CWAPI, ts.cfg.EKSConfig.CWNamespace, 20, datums...)
-}
diff --git a/eks/stresser/stresser.go b/eks/stresser/stresser.go
deleted file mode 100644
index b0d688944..000000000
--- a/eks/stresser/stresser.go
+++ /dev/null
@@ -1,755 +0,0 @@
-// Package stresser implements cluster load tests.
-// ref. https://github.com/kubernetes/perf-tests
-package stresser
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "io"
- "io/ioutil"
- "sort"
- "sync"
- "time"
-
- aws_s3 "github.com/aws/aws-k8s-tester/pkg/aws/s3"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/metrics"
- "github.com/aws/aws-k8s-tester/pkg/randutil"
- "github.com/aws/aws-sdk-go/service/s3/s3iface"
- "github.com/prometheus/client_golang/prometheus"
- "go.uber.org/zap"
- v1 "k8s.io/api/core/v1"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/client-go/kubernetes"
-)
-
-var (
- writeRequestsSuccessTotal = prometheus.NewGauge(
- prometheus.GaugeOpts{
- Namespace: "stresser",
- Subsystem: "client",
- Name: "write_requests_success_total",
- Help: "Total number of successful write requests.",
- })
- writeRequestsFailureTotal = prometheus.NewGauge(
- prometheus.GaugeOpts{
- Namespace: "stresser",
- Subsystem: "client",
- Name: "write_requests_failure_total",
- Help: "Total number of successful write requests.",
- })
- writeRequestLatencyMs = prometheus.NewHistogram(
- prometheus.HistogramOpts{
- Namespace: "stresser",
- Subsystem: "client",
- Name: "write_request_latency_milliseconds",
- Help: "Bucketed histogram of client-side write request and response latency.",
-
- // lowest bucket start of upper bound 0.5 ms with factor 2
- // highest bucket start of 0.5 ms * 2^13 == 4.096 sec
- Buckets: prometheus.ExponentialBuckets(0.5, 2, 14),
- })
-
- readRequestsSuccessTotal = prometheus.NewGauge(
- prometheus.GaugeOpts{
- Namespace: "stresser",
- Subsystem: "client",
- Name: "read_requests_success_total",
- Help: "Total number of successful read requests.",
- })
- readRequestsFailureTotal = prometheus.NewGauge(
- prometheus.GaugeOpts{
- Namespace: "stresser",
- Subsystem: "client",
- Name: "read_requests_failure_total",
- Help: "Total number of successful read requests.",
- })
- readRequestLatencyMs = prometheus.NewHistogram(
- prometheus.HistogramOpts{
- Namespace: "stresser",
- Subsystem: "client",
- Name: "read_request_latency_milliseconds",
- Help: "Bucketed histogram of client-side read request and response latency.",
-
- // lowest bucket start of upper bound 0.5 ms with factor 2
- // highest bucket start of 0.5 ms * 2^13 == 4.096 sec
- Buckets: prometheus.ExponentialBuckets(0.5, 2, 14),
- })
-)
-
-func init() {
- prometheus.MustRegister(writeRequestsSuccessTotal)
- prometheus.MustRegister(writeRequestsFailureTotal)
- prometheus.MustRegister(writeRequestLatencyMs)
- prometheus.MustRegister(readRequestsSuccessTotal)
- prometheus.MustRegister(readRequestsFailureTotal)
- prometheus.MustRegister(readRequestLatencyMs)
-}
-
-// Config configures cluster loader.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
-
- Stopc chan struct{}
-
- S3API s3iface.S3API
- S3BucketName string
-
- Client k8s_client.EKS
- ClientTimeout time.Duration
-
- Deadline time.Time
-
- NamespaceWrite string
- NamespacesRead []string
-
- ObjectSize int
- ListLimit int64
-
- RequestsRawWritesJSONPath string
- RequestsRawWritesJSONS3Key string
- RequestsSummaryWritesJSONPath string
- RequestsSummaryWritesJSONS3Key string
- RequestsSummaryWritesTablePath string
- RequestsSummaryWritesTableS3Key string
-
- RequestsRawReadsJSONPath string
- RequestsRawReadsJSONS3Key string
- RequestsSummaryReadsJSONPath string
- RequestsSummaryReadsJSONS3Key string
- RequestsSummaryReadsTablePath string
- RequestsSummaryReadsTableS3Key string
-}
-
-// Loader defines cluster loader operations.
-type Loader interface {
- Start()
- Stop()
- CollectMetrics() (writeLatencies metrics.Durations, writesSummary metrics.RequestsSummary, readLatencies metrics.Durations, readsSummary metrics.RequestsSummary, err error)
-}
-
-type loader struct {
- cfg Config
- donec chan struct{}
- donecCloseOnce *sync.Once
-
- writeLatencies chan metrics.Durations
- readLatencies chan metrics.Durations
-}
-
-func New(cfg Config) Loader {
- return &loader{
- cfg: cfg,
- donec: make(chan struct{}),
- donecCloseOnce: new(sync.Once),
- writeLatencies: make(chan metrics.Durations, 1), // buffer to not block send
- readLatencies: make(chan metrics.Durations, 1), // buffer to not block send
- }
-}
-
-func (ld *loader) Start() {
- ld.cfg.Logger.Info("starting load functions", zap.String("namespace-write", ld.cfg.NamespaceWrite), zap.Strings("namespaces-read", ld.cfg.NamespacesRead))
- if ld.cfg.ObjectSize > 0 {
- go startWrites(
- ld.cfg.Logger,
- ld.cfg.Client.KubernetesClientSet(),
- ld.cfg.ClientTimeout,
- ld.cfg.Deadline,
- ld.cfg.NamespaceWrite,
- ld.cfg.ObjectSize,
- ld.cfg.Stopc,
- ld.donec,
- ld.writeLatencies,
- )
- }
- go startReads(
- ld.cfg.Logger,
- ld.cfg.Client.KubernetesClientSet(),
- ld.cfg.ClientTimeout,
- ld.cfg.Deadline,
- ld.cfg.NamespacesRead,
- ld.cfg.ListLimit,
- ld.cfg.Stopc,
- ld.donec,
- ld.readLatencies,
- )
- ld.cfg.Logger.Info("started load functions", zap.String("namespace-write", ld.cfg.NamespaceWrite), zap.Strings("namespaces-read", ld.cfg.NamespacesRead))
-}
-
-func (ld *loader) Stop() {
- ld.cfg.Logger.Info("stopping and waiting for load functions")
- ld.donecCloseOnce.Do(func() {
- close(ld.donec)
- })
- time.Sleep(5 * time.Second) // enough time to stop goroutines
- ld.cfg.Logger.Info("stopped and waited for load functions")
-}
-
-// GetMetrics locally fetches output from registered metrics.
-// ref. https://pkg.go.dev/github.com/prometheus/client_golang@v1.6.0/prometheus/promhttp?tab=doc#Handler
-func (ts *loader) CollectMetrics() (writeLatencies metrics.Durations, writesSummary metrics.RequestsSummary, readLatencies metrics.Durations, readsSummary metrics.RequestsSummary, err error) {
- curTS := time.Now().UTC().Format(time.RFC3339Nano)
- writesSummary = metrics.RequestsSummary{TestID: curTS}
- readsSummary = metrics.RequestsSummary{TestID: curTS}
-
- // https://pkg.go.dev/github.com/prometheus/client_golang/prometheus?tab=doc#Gatherer
- mfs, err := prometheus.DefaultGatherer.Gather()
- if err != nil {
- ts.cfg.Logger.Warn("failed to gather prometheus metrics", zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- for _, mf := range mfs {
- if mf == nil {
- continue
- }
- switch *mf.Name {
- case "stresser_client_write_requests_success_total":
- gg := mf.Metric[0].GetGauge()
- writesSummary.SuccessTotal = gg.GetValue()
- case "stresser_client_write_requests_failure_total":
- gg := mf.Metric[0].GetGauge()
- writesSummary.FailureTotal = gg.GetValue()
- case "stresser_client_write_request_latency_milliseconds":
- writesSummary.LatencyHistogram, err = metrics.ParseHistogram("milliseconds", mf.Metric[0].GetHistogram())
- if err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
-
- case "stresser_client_read_requests_success_total":
- gg := mf.Metric[0].GetGauge()
- readsSummary.SuccessTotal = gg.GetValue()
- case "stresser_client_read_requests_failure_total":
- gg := mf.Metric[0].GetGauge()
- readsSummary.FailureTotal = gg.GetValue()
- case "stresser_client_read_request_latency_milliseconds":
- readsSummary.LatencyHistogram, err = metrics.ParseHistogram("milliseconds", mf.Metric[0].GetHistogram())
- if err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- }
- }
-
- ts.cfg.Logger.Info("receiving write latency results")
- select {
- case writeLatencies = <-ts.writeLatencies:
- ts.cfg.Logger.Info("received and sorting write latency results", zap.Int("total-data-points", writeLatencies.Len()))
- now := time.Now()
- sort.Sort(writeLatencies)
- ts.cfg.Logger.Info("sorted write latency results", zap.Int("total-data-points", writeLatencies.Len()), zap.String("took", time.Since(now).String()))
- writesSummary.LantencyP50 = writeLatencies.PickLantencyP50()
- writesSummary.LantencyP90 = writeLatencies.PickLantencyP90()
- writesSummary.LantencyP99 = writeLatencies.PickLantencyP99()
- writesSummary.LantencyP999 = writeLatencies.PickLantencyP999()
- writesSummary.LantencyP9999 = writeLatencies.PickLantencyP9999()
-
- ts.cfg.Logger.Info("writing latency results in JSON to disk", zap.String("path", ts.cfg.RequestsRawWritesJSONPath))
- wb, err := json.Marshal(writeLatencies)
- if err != nil {
- ts.cfg.Logger.Warn("failed to encode latency results in JSON", zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = ioutil.WriteFile(ts.cfg.RequestsRawWritesJSONPath, wb, 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write latency results in JSON to disk", zap.String("path", ts.cfg.RequestsRawWritesJSONPath), zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.S3BucketName,
- ts.cfg.RequestsRawWritesJSONS3Key,
- ts.cfg.RequestsRawWritesJSONPath,
- ); err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
-
- case <-time.After(2 * time.Minute):
- ts.cfg.Logger.Warn("took too long to receive write latency results")
- }
-
- ts.cfg.Logger.Info("receiving read latency results")
- select {
- case readLatencies = <-ts.readLatencies:
- ts.cfg.Logger.Info("received and sorting read latency results", zap.Int("total-data-points", readLatencies.Len()))
- now := time.Now()
- sort.Sort(readLatencies)
- ts.cfg.Logger.Info("sorted read latency results", zap.Int("total-data-points", readLatencies.Len()), zap.String("took", time.Since(now).String()))
- readsSummary.LantencyP50 = readLatencies.PickLantencyP50()
- readsSummary.LantencyP90 = readLatencies.PickLantencyP90()
- readsSummary.LantencyP99 = readLatencies.PickLantencyP99()
- readsSummary.LantencyP999 = readLatencies.PickLantencyP999()
- readsSummary.LantencyP9999 = readLatencies.PickLantencyP9999()
-
- ts.cfg.Logger.Info("writing latency results in JSON to disk", zap.String("path", ts.cfg.RequestsRawReadsJSONPath))
- wb, err := json.Marshal(readLatencies)
- if err != nil {
- ts.cfg.Logger.Warn("failed to encode latency results in JSON", zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = ioutil.WriteFile(ts.cfg.RequestsRawReadsJSONPath, wb, 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write latency results in JSON to disk", zap.String("path", ts.cfg.RequestsRawReadsJSONPath), zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.S3BucketName,
- ts.cfg.RequestsRawReadsJSONS3Key,
- ts.cfg.RequestsRawReadsJSONPath,
- ); err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
-
- case <-time.After(2 * time.Minute):
- ts.cfg.Logger.Warn("took too long to receive read latency results")
- }
-
- if err = ioutil.WriteFile(ts.cfg.RequestsSummaryWritesJSONPath, []byte(writesSummary.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.S3BucketName,
- ts.cfg.RequestsSummaryWritesJSONS3Key,
- ts.cfg.RequestsSummaryWritesJSONPath,
- ); err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = ioutil.WriteFile(ts.cfg.RequestsSummaryWritesTablePath, []byte(writesSummary.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.S3BucketName,
- ts.cfg.RequestsSummaryWritesTableS3Key,
- ts.cfg.RequestsSummaryWritesTablePath,
- ); err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nSummaryWritesTable:\n%s\n", writesSummary.Table())
-
- if err = ioutil.WriteFile(ts.cfg.RequestsSummaryReadsJSONPath, []byte(readsSummary.JSON()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.S3BucketName,
- ts.cfg.RequestsSummaryReadsJSONS3Key,
- ts.cfg.RequestsSummaryReadsJSONPath,
- ); err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = ioutil.WriteFile(ts.cfg.RequestsSummaryReadsTablePath, []byte(readsSummary.Table()), 0600); err != nil {
- ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- if err = aws_s3.Upload(
- ts.cfg.Logger,
- ts.cfg.S3API,
- ts.cfg.S3BucketName,
- ts.cfg.RequestsSummaryReadsTableS3Key,
- ts.cfg.RequestsSummaryReadsTablePath,
- ); err != nil {
- return nil, metrics.RequestsSummary{}, nil, metrics.RequestsSummary{}, err
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryReadsTable:\n%s\n", readsSummary.Table())
-
- return writeLatencies, writesSummary, readLatencies, readsSummary, nil
-}
-
-func startWrites(
- lg *zap.Logger,
- cli *kubernetes.Clientset,
- timeout time.Duration,
- deadline time.Time,
- namespace string,
- objectSize int,
- stopc chan struct{},
- donec chan struct{},
- writeLatencies chan<- metrics.Durations,
-) {
- lg.Info("starting writes")
- ds := make(metrics.Durations, 0, 20000)
- defer func() {
- lg.Info("sending write latency results", zap.Int("total-results", len(ds)))
- select {
- case writeLatencies <- ds:
- lg.Info("sent write latency results", zap.Int("total-results", len(ds)))
- case <-time.After(2 * time.Minute):
- lg.Warn("took to long to send write latency results")
- // in case, receiving takes long...
- select {
- case <-stopc:
- return
- case <-donec:
- return
- }
- }
- }()
-
- val := randutil.String(objectSize)
- cnt := 0
- for {
- cnt++
- select {
- case <-stopc:
- lg.Warn("writes stopped")
- return
- case <-donec:
- lg.Info("writes done")
- return
- default:
- }
-
- key := fmt.Sprintf("secret%d%s", cnt, randutil.String(7))
-
- start := time.Now()
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- _, err := cli.
- CoreV1().
- ConfigMaps(namespace).
- Create(ctx, &v1.ConfigMap{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "v1",
- Kind: "ConfigMap",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: key,
- Namespace: namespace,
- Labels: map[string]string{
- "name": key,
- },
- },
- Data: map[string]string{key: val},
- }, metav1.CreateOptions{})
- cancel()
- took := time.Since(start)
- tookMS := float64(took / time.Millisecond)
- writeRequestLatencyMs.Observe(tookMS)
- ds = append(ds, took)
- if err != nil {
- writeRequestsFailureTotal.Inc()
- lg.Warn("write configmap failed", zap.String("namespace", namespace), zap.Error(err))
- } else {
- writeRequestsSuccessTotal.Inc()
- if cnt%20 == 0 {
- lg.Info("wrote configmap", zap.String("time-left", deadline.Sub(time.Now()).String()), zap.Int("iteration", cnt), zap.String("namespace", namespace))
- }
- }
- select {
- case <-stopc:
- lg.Warn("write configmap stopped")
- return
- case <-donec:
- lg.Info("write configmap done")
- return
- default:
- }
-
- key = fmt.Sprintf("configmap%d%s", cnt, randutil.String(7))
- start = time.Now()
- ctx, cancel = context.WithTimeout(context.Background(), timeout)
- _, err = cli.
- CoreV1().
- Secrets(namespace).
- Create(ctx, &v1.Secret{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "v1",
- Kind: "Secret",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: key,
- Namespace: namespace,
- Labels: map[string]string{
- "name": key,
- },
- },
- Type: v1.SecretTypeOpaque,
- Data: map[string][]byte{key: []byte(val)},
- }, metav1.CreateOptions{})
- cancel()
- took = time.Since(start)
- tookMS = float64(took / time.Millisecond)
- writeRequestLatencyMs.Observe(tookMS)
- ds = append(ds, took)
- if err != nil {
- writeRequestsFailureTotal.Inc()
- lg.Warn("write secret failed", zap.String("namespace", namespace), zap.Error(err))
- } else {
- writeRequestsSuccessTotal.Inc()
- if cnt%20 == 0 {
- lg.Info("wrote secret", zap.String("time-left", deadline.Sub(time.Now()).String()), zap.Int("iteration", cnt), zap.String("namespace", namespace))
- }
- }
- select {
- case <-stopc:
- lg.Warn("write secret stopped")
- return
- case <-donec:
- lg.Info("write secret done")
- return
- default:
- }
- }
-}
-
-func startReads(
- lg *zap.Logger,
- cli *kubernetes.Clientset,
- timeout time.Duration,
- deadline time.Time,
- ns []string,
- listLimit int64,
- stopc chan struct{},
- donec chan struct{},
- readLatencies chan<- metrics.Durations,
-) {
- lg.Info("starting reads", zap.Strings("namespaces", ns))
- ds := make(metrics.Durations, 0, 20000)
- defer func() {
- lg.Info("sending read latency results", zap.Int("total-results", len(ds)))
- select {
- case readLatencies <- ds:
- lg.Info("sent read latency results", zap.Int("total-results", len(ds)))
- case <-time.After(2 * time.Minute):
- lg.Warn("took to long to send read latency results")
- // in case, receiving takes long...
- select {
- case <-stopc:
- return
- case <-donec:
- return
- }
- }
- }()
-
- cnt := 0
- for {
- cnt++
- select {
- case <-stopc:
- lg.Warn("reads stopped")
- return
- case <-donec:
- lg.Info("reads done")
- return
- default:
- }
-
- start := time.Now()
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- rs, err := cli.CoreV1().Nodes().List(ctx, metav1.ListOptions{Limit: listLimit})
- cancel()
- took := time.Since(start)
- tookMS := float64(took / time.Millisecond)
- readRequestLatencyMs.Observe(tookMS)
- ds = append(ds, took)
- if err != nil {
- readRequestsFailureTotal.Inc()
- lg.Warn("list nodes failed", zap.Error(err))
- } else {
- readRequestsSuccessTotal.Inc()
- if cnt%20 == 0 {
- lg.Info("listed nodes", zap.String("time-left", deadline.Sub(time.Now()).String()), zap.Int("iteration", cnt), zap.Int("nodes", len(rs.Items)))
- }
- }
-
- for _, nv := range ns {
- start := time.Now()
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- pods, err := cli.CoreV1().Pods(nv).List(ctx, metav1.ListOptions{Limit: listLimit})
- cancel()
- took := time.Since(start)
- tookMS := float64(took / time.Millisecond)
- readRequestLatencyMs.Observe(tookMS)
- ds = append(ds, took)
- if err != nil {
- readRequestsFailureTotal.Inc()
- lg.Warn("list pods failed", zap.String("namespace", nv), zap.Error(err))
- } else {
- readRequestsSuccessTotal.Inc()
- if cnt%20 == 0 {
- lg.Info("listed pods", zap.String("time-left", deadline.Sub(time.Now()).String()), zap.Int("iteration", cnt), zap.String("namespace", nv), zap.Int("pods", len(pods.Items)))
- }
- }
- select {
- case <-stopc:
- lg.Warn("list pods stopped")
- return
- case <-donec:
- lg.Info("list pods done")
- return
- default:
- }
-
- start = time.Now()
- ctx, cancel = context.WithTimeout(context.Background(), timeout)
- svcs, err := cli.CoreV1().Services(nv).List(ctx, metav1.ListOptions{Limit: listLimit})
- cancel()
- took = time.Since(start)
- tookMS = float64(took / time.Millisecond)
- readRequestLatencyMs.Observe(tookMS)
- ds = append(ds, took)
- if err != nil {
- readRequestsFailureTotal.Inc()
- lg.Warn("list services failed", zap.String("namespace", nv), zap.Error(err))
- } else {
- readRequestsSuccessTotal.Inc()
- if cnt%20 == 0 {
- lg.Info("listed services", zap.String("time-left", deadline.Sub(time.Now()).String()), zap.Int("iteration", cnt), zap.String("namespace", nv), zap.Int("services", len(svcs.Items)))
- }
- }
- select {
- case <-stopc:
- lg.Warn("list services stopped")
- return
- case <-donec:
- lg.Info("list services done")
- return
- default:
- }
-
- start = time.Now()
- ctx, cancel = context.WithTimeout(context.Background(), timeout)
- eps, err := cli.CoreV1().Endpoints(nv).List(ctx, metav1.ListOptions{Limit: listLimit})
- cancel()
- took = time.Since(start)
- tookMS = float64(took / time.Millisecond)
- readRequestLatencyMs.Observe(tookMS)
- ds = append(ds, took)
- if err != nil {
- readRequestsFailureTotal.Inc()
- lg.Warn("list endpoints failed", zap.String("namespace", nv), zap.Error(err))
- } else {
- readRequestsSuccessTotal.Inc()
- if cnt%20 == 0 {
- lg.Info("listed endpoints", zap.String("time-left", deadline.Sub(time.Now()).String()), zap.Int("iteration", cnt), zap.String("namespace", nv), zap.Int("endpoints", len(eps.Items)))
- }
- }
- select {
- case <-stopc:
- lg.Warn("list endpoints stopped")
- return
- case <-donec:
- lg.Info("list endpoints done")
- return
- default:
- }
-
- start = time.Now()
- ctx, cancel = context.WithTimeout(context.Background(), timeout)
- cms, err := cli.CoreV1().ConfigMaps(nv).List(ctx, metav1.ListOptions{Limit: listLimit})
- cancel()
- took = time.Since(start)
- tookMS = float64(took / time.Millisecond)
- readRequestLatencyMs.Observe(tookMS)
- ds = append(ds, took)
- if err != nil {
- readRequestsFailureTotal.Inc()
- lg.Warn("list configmaps failed", zap.String("namespace", nv), zap.Error(err))
- } else {
- readRequestsSuccessTotal.Inc()
- if cnt%20 == 0 {
- lg.Info("listed configmaps", zap.String("time-left", deadline.Sub(time.Now()).String()), zap.Int("iteration", cnt), zap.String("namespace", nv), zap.Int("configmaps", len(cms.Items)))
- }
- }
- select {
- case <-stopc:
- lg.Warn("list configmaps stopped")
- return
- case <-donec:
- lg.Info("list configmaps done")
- return
- default:
- }
-
- start = time.Now()
- ctx, cancel = context.WithTimeout(context.Background(), timeout)
- ss, err := cli.CoreV1().Secrets(nv).List(ctx, metav1.ListOptions{Limit: listLimit})
- cancel()
- took = time.Since(start)
- tookMS = float64(took / time.Millisecond)
- readRequestLatencyMs.Observe(tookMS)
- ds = append(ds, took)
- if err != nil {
- readRequestsFailureTotal.Inc()
- lg.Warn("list secrets failed", zap.String("namespace", nv), zap.Error(err))
- } else {
- readRequestsSuccessTotal.Inc()
- if cnt%20 == 0 {
- lg.Info("listed secrets", zap.String("time-left", deadline.Sub(time.Now()).String()), zap.Int("iteration", cnt), zap.String("namespace", nv), zap.Int("secrets", len(ss.Items)))
- }
- }
- select {
- case <-stopc:
- lg.Warn("list secrets stopped")
- return
- case <-donec:
- lg.Info("list secrets done")
- return
- default:
- }
-
- start = time.Now()
- ctx, cancel = context.WithTimeout(context.Background(), timeout)
- jobs, err := cli.BatchV1().Jobs(nv).List(ctx, metav1.ListOptions{Limit: listLimit})
- cancel()
- took = time.Since(start)
- tookMS = float64(took / time.Millisecond)
- readRequestLatencyMs.Observe(tookMS)
- ds = append(ds, took)
- if err != nil {
- readRequestsFailureTotal.Inc()
- lg.Warn("list jobs failed", zap.String("namespace", nv), zap.Error(err))
- } else {
- readRequestsSuccessTotal.Inc()
- if cnt%20 == 0 {
- lg.Info("listed jobs", zap.String("time-left", deadline.Sub(time.Now()).String()), zap.Int("iteration", cnt), zap.String("namespace", nv), zap.Int("jobs", len(jobs.Items)))
- }
- }
- select {
- case <-stopc:
- lg.Warn("list jobs stopped")
- return
- case <-donec:
- lg.Info("list jobs done")
- return
- default:
- }
-
- start = time.Now()
- ctx, cancel = context.WithTimeout(context.Background(), timeout)
- cjbs, err := cli.BatchV1beta1().CronJobs(nv).List(ctx, metav1.ListOptions{Limit: listLimit})
- cancel()
- took = time.Since(start)
- tookMS = float64(took / time.Millisecond)
- readRequestLatencyMs.Observe(tookMS)
- ds = append(ds, took)
- if err != nil {
- readRequestsFailureTotal.Inc()
- lg.Warn("list cronjobs failed", zap.String("namespace", nv), zap.Error(err))
- } else {
- readRequestsSuccessTotal.Inc()
- if cnt%20 == 0 {
- lg.Info("listed cronjobs", zap.String("time-left", deadline.Sub(time.Now()).String()), zap.Int("iteration", cnt), zap.String("namespace", nv), zap.Int("cronjobs", len(cjbs.Items)))
- }
- }
- select {
- case <-stopc:
- lg.Warn("list cronjobs stopped")
- return
- case <-donec:
- lg.Info("list cronjobs done")
- return
- default:
- }
- }
- }
-}
diff --git a/eks/stresser/stresser_test.go b/eks/stresser/stresser_test.go
deleted file mode 100644
index b52fa0993..000000000
--- a/eks/stresser/stresser_test.go
+++ /dev/null
@@ -1,62 +0,0 @@
-package stresser
-
-import (
- "bytes"
- "fmt"
- "strings"
- "testing"
- "time"
-
- "github.com/prometheus/client_golang/prometheus"
- "github.com/prometheus/common/expfmt"
-)
-
-func TestMetrics(t *testing.T) {
- start := time.Now().Add(-10 * time.Minute)
- readRequestLatencyMs.Observe(float64(time.Since(start) / time.Millisecond))
- readRequestLatencyMs.Observe(float64(time.Since(start) / time.Millisecond))
- readRequestLatencyMs.Observe(float64(time.Since(time.Now().Add(-time.Millisecond)) / time.Millisecond))
-
- readRequestsSuccessTotal.Inc()
- readRequestsSuccessTotal.Inc()
-
- buf := bytes.NewBuffer(nil)
- enc := expfmt.NewEncoder(buf, expfmt.FmtText)
-
- mfs, err := prometheus.DefaultGatherer.Gather()
- if err != nil {
- t.Fatal(err)
- }
-
- for _, mf := range mfs {
- if mf == nil {
- continue
- }
- switch *mf.Name {
- case "stresser_client_read_request_latency_milliseconds":
- for _, bucket := range mf.Metric[0].Histogram.Bucket {
- fmt.Println(*bucket.UpperBound, *bucket.CumulativeCount)
- }
- fmt.Println(*mf.Metric[0].Histogram.SampleCount)
- case "stresser_client_read_requests_success_total":
- gg := *mf.Metric[0].GetGauge()
- cnt := gg.GetValue()
- fmt.Println("count:", cnt)
- if cnt != 2 {
- t.Fatalf("stresser_client_read_requests_success_total expected 2, got %v", cnt)
- }
- }
-
- err := enc.Encode(mf)
- if err != nil {
- t.Fatal(err)
- }
- }
-
- body := buf.String()
- if !strings.Contains(body, `stresser_client_read_request_latency_milliseconds_bucket{le="+Inf"} 3`) {
- t.Fatalf("unexpected output:\n\n%s\n\n", body)
- }
-
- fmt.Println(body)
-}
diff --git a/eks/stresser2/stresser.go b/eks/stresser2/stresser.go
deleted file mode 100644
index 8a47d0144..000000000
--- a/eks/stresser2/stresser.go
+++ /dev/null
@@ -1,555 +0,0 @@
-package stresser2
-
-import (
- "context"
- "errors"
- "fmt"
- "io"
- "reflect"
- "strings"
- "time"
-
- eks_tester "github.com/aws/aws-k8s-tester/eks/tester"
- "github.com/aws/aws-k8s-tester/eksconfig"
- aws_ecr "github.com/aws/aws-k8s-tester/pkg/aws/ecr"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/service/ecr/ecriface"
- "github.com/dustin/go-humanize"
- "go.uber.org/zap"
- batch_v1 "k8s.io/api/batch/v1"
- batch_v1beta1 "k8s.io/api/batch/v1beta1"
- v1 "k8s.io/api/core/v1"
- rbacv1 "k8s.io/api/rbac/v1"
- apierrs "k8s.io/apimachinery/pkg/api/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/utils/exec"
- "sigs.k8s.io/yaml"
-)
-
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
-
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
- ECRAPI ecriface.ECRAPI
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-func New(cfg Config) eks_tester.Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
- ecrImage string
-}
-
-const (
- stresserV2AppName = "stresser2-app"
- stresserV2ServiceAccountName = "stresser2-remote-service-account"
- stresserV2RBACRoleName = "stresser2-remote-rbac-role"
- stresserV2RBACClusterRoleBindingName = "stresser2-remote-rbac-role-binding"
-)
-
-func (ts *tester) Create() (err error) {
- if !ts.cfg.EKSConfig.IsEnabledAddOnStresserRemoteV2() {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
- if ts.cfg.EKSConfig.AddOnStresserRemoteV2.Created {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Create", zap.String("tester", pkgName))
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.Created = true
- ts.cfg.EKSConfig.Sync()
- createStart := time.Now()
- defer func() {
- createEnd := time.Now()
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- if ts.ecrImage, _, err = aws_ecr.Check(
- ts.cfg.Logger,
- ts.cfg.ECRAPI,
- ts.cfg.EKSConfig.Partition,
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.RepositoryAccountID,
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.RepositoryRegion,
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.RepositoryName,
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.RepositoryImageTag,
- ); err != nil {
- return err
- }
-
- if err = k8s_client.CreateNamespace(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.Namespace,
- ); err != nil {
- return err
- }
- if err = ts.createServiceAccount(); err != nil {
- return err
- }
- if err = ts.createRBACClusterRole(); err != nil {
- return err
- }
- if err = ts.createRBACClusterRoleBinding(); err != nil {
- return err
- }
- if err = ts.createCronJob(); err != nil {
- return err
- }
-
- // TODO waits for all the job spawned by cronJob up and running
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) Delete() (err error) {
- if !ts.cfg.EKSConfig.IsEnabledAddOnStresserRemoteV2() {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnStresserRemoteV2.Created {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Delete", zap.String("tester", pkgName))
- deleteStart := time.Now()
- defer func() {
- deleteEnd := time.Now()
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.TimeFrameDelete = timeutil.NewTimeFrame(deleteStart, deleteEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- var errs []string
-
- if err := ts.deleteCronJob(); err != nil {
- errs = append(errs, err.Error())
- }
- time.Sleep(2 * time.Minute)
-
- if err := ts.deleteRBACClusterRoleBinding(); err != nil {
- errs = append(errs, err.Error())
- }
- if err := ts.deleteRBACClusterRole(); err != nil {
- errs = append(errs, err.Error())
- }
- if err := ts.deleteServiceAccount(); err != nil {
- errs = append(errs, err.Error())
- }
-
- getAllArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnStresserRemoteV2.Namespace,
- "get",
- "all",
- }
- getAllCmd := strings.Join(getAllArgs, " ")
-
- if err := k8s_client.DeleteNamespaceAndWait(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.Namespace,
- k8s_client.DefaultNamespaceDeletionInterval,
- k8s_client.DefaultNamespaceDeletionTimeout,
- k8s_client.WithQueryFunc(func() {
- fmt.Fprintf(ts.cfg.LogWriter, "\n")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, getAllArgs[0], getAllArgs[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl get all' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getAllCmd, out)
- }),
- k8s_client.WithForceDelete(true),
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete remote stresser2 namespace (%v)", err))
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.Created = false
- ts.cfg.EKSConfig.Sync()
- return
-}
-
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-// ref. https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/#foreground-cascading-deletion
-func (ts *tester) createServiceAccount() (err error) {
- ts.cfg.Logger.Info("creating stresser2 ServiceAccount")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err = ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- ServiceAccounts(ts.cfg.EKSConfig.AddOnStresserRemoteV2.Namespace).
- Create(
- ctx,
- &v1.ServiceAccount{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "v1",
- Kind: "ServiceAccount",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: stresserV2ServiceAccountName,
- Namespace: ts.cfg.EKSConfig.AddOnStresserRemoteV2.Namespace,
- Labels: map[string]string{
- "app.kubernetes.io/name": stresserV2AppName,
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create stresser2 ServiceAccount (%v)", err)
- }
-
- ts.cfg.Logger.Info("created stresser2 ServiceAccount")
- ts.cfg.EKSConfig.Sync()
- return
-}
-func (ts *tester) deleteServiceAccount() (err error) {
- ts.cfg.Logger.Info("deleting stresser2 ServiceAccount")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err = ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- ServiceAccounts(ts.cfg.EKSConfig.AddOnStresserRemoteV2.Namespace).
- Delete(
- ctx,
- stresserV2ServiceAccountName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete stresser2 ServiceAccount (%v)", err)
- }
- ts.cfg.Logger.Info("deleted stresser2 ServiceAccount", zap.Error(err))
-
- ts.cfg.EKSConfig.Sync()
- return
-}
-func (ts *tester) createRBACClusterRole() (err error) {
- ts.cfg.Logger.Info("creating stresser RBAC ClusterRole")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err = ts.cfg.K8SClient.KubernetesClientSet().
- RbacV1().
- ClusterRoles().
- Create(
- ctx,
- &rbacv1.ClusterRole{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "rbac.authorization.k8s.io/v1",
- Kind: "ClusterRole",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: stresserV2RBACRoleName,
- Namespace: "default",
- Labels: map[string]string{
- "app.kubernetes.io/name": stresserV2AppName,
- },
- },
- Rules: []rbacv1.PolicyRule{
- {
- APIGroups: []string{
- "*",
- },
- Resources: []string{
- "configmaps",
- "leases",
- "nodes",
- "pods",
- "secrets",
- "services",
- "namespaces",
- "stresser",
- "endpoints",
- "events",
- "ingresses",
- "ingresses/status",
- "services",
- "jobs",
- "cronjobs",
- },
- Verbs: []string{
- "create",
- "get",
- "list",
- "update",
- "watch",
- "patch",
- },
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create stresser2 RBAC ClusterRole (%v)", err)
- }
-
- ts.cfg.Logger.Info("created stresser2 RBAC ClusterRole")
- ts.cfg.EKSConfig.Sync()
- return
-}
-func (ts *tester) deleteRBACClusterRole() (err error) {
- ts.cfg.Logger.Info("deleting stresser2 RBAC ClusterRole")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err = ts.cfg.K8SClient.KubernetesClientSet().
- RbacV1().
- ClusterRoles().
- Delete(
- ctx,
- stresserV2RBACRoleName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete stresser2 RBAC ClusterRole (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted stresser2 RBAC ClusterRole", zap.Error(err))
- ts.cfg.EKSConfig.Sync()
- return
-}
-
-// https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) createRBACClusterRoleBinding() (err error) {
- ts.cfg.Logger.Info("creating stresser2 RBAC ClusterRoleBinding")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err = ts.cfg.K8SClient.KubernetesClientSet().
- RbacV1().
- ClusterRoleBindings().
- Create(
- ctx,
- &rbacv1.ClusterRoleBinding{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "rbac.authorization.k8s.io/v1",
- Kind: "ClusterRoleBinding",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: stresserV2RBACClusterRoleBindingName,
- Namespace: "default",
- Labels: map[string]string{
- "app.kubernetes.io/name": stresserV2AppName,
- },
- },
- RoleRef: rbacv1.RoleRef{
- APIGroup: "rbac.authorization.k8s.io",
- Kind: "ClusterRole",
- Name: stresserV2RBACRoleName,
- },
- Subjects: []rbacv1.Subject{
- {
- APIGroup: "",
- Kind: "ServiceAccount",
- Name: stresserV2ServiceAccountName,
- Namespace: ts.cfg.EKSConfig.AddOnStresserRemoteV2.Namespace,
- },
- {
- APIGroup: "rbac.authorization.k8s.io",
- Kind: "User",
- Name: "system:node",
- },
- },
- },
- metav1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create stresser2 RBAC ClusterRoleBinding (%v)", err)
- }
-
- ts.cfg.Logger.Info("created stresser2 RBAC ClusterRoleBinding")
- ts.cfg.EKSConfig.Sync()
- return
-}
-func (ts *tester) deleteRBACClusterRoleBinding() (err error) {
- ts.cfg.Logger.Info("deleting stresser2 RBAC ClusterRoleBinding")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err = ts.cfg.K8SClient.KubernetesClientSet().
- RbacV1().
- ClusterRoleBindings().
- Delete(
- ctx,
- stresserV2RBACClusterRoleBindingName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete stresser2 RBAC ClusterRoleBinding (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted stresser2 RBAC ClusterRoleBinding", zap.Error(err))
- ts.cfg.EKSConfig.Sync()
- return
-}
-
-func (ts *tester) createCronJob() (err error) {
- obj, b, err := ts.createObject()
- if err != nil {
- return err
- }
-
- ts.cfg.Logger.Info("creating CronJob",
- zap.String("name", stresserV2AppName),
- zap.Int("completes", ts.cfg.EKSConfig.AddOnStresserRemoteV2.Completes),
- zap.Int("parallels", ts.cfg.EKSConfig.AddOnStresserRemoteV2.Parallels),
- zap.String("object-size", humanize.Bytes(uint64(len(b)))),
- )
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err = ts.cfg.K8SClient.KubernetesClientSet().
- BatchV1beta1().
- CronJobs(ts.cfg.EKSConfig.AddOnStresserRemoteV2.Namespace).
- Create(ctx, &obj, metav1.CreateOptions{})
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create CronJob (%v)", err)
- }
-
- ts.cfg.Logger.Info("created CronJob")
- return nil
-}
-func (ts *tester) createObject() (batch_v1beta1.CronJob, string, error) {
- testerCmd := fmt.Sprintf("/aws-k8s-tester eks create stresser2 --number=%d --duration=%s --object-size=%d --secrets=%d --busybox-image=%s",
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.Coroutines,
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.Duration,
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.ObjectSize,
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.Secrets,
- fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s:%s",
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.RepositoryAccountID,
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.RepositoryRegion,
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.RepositoryBusyBoxName,
- ts.cfg.EKSConfig.AddOnStresserRemoteV2.RepositoryBusyBoxImageTag),
- )
-
- dirOrCreate := v1.HostPathDirectoryOrCreate
- podSpec := v1.PodTemplateSpec{
- ObjectMeta: metav1.ObjectMeta{
- Labels: map[string]string{
- "app.kubernetes.io/name": stresserV2AppName,
- },
- },
- Spec: v1.PodSpec{
- RestartPolicy: v1.RestartPolicyOnFailure,
- Containers: []v1.Container{
- {
- Name: stresserV2AppName,
- Image: ts.ecrImage,
- ImagePullPolicy: v1.PullAlways,
- Command: []string{
- "/bin/sh",
- "-ec",
- testerCmd,
- },
- // ref. https://kubernetes.io/docs/concepts/cluster-administration/logging/
- VolumeMounts: []v1.VolumeMount{
- {
- Name: "logging",
- MountPath: "/var/log",
- ReadOnly: false,
- },
- },
- },
- },
- // ref. https://kubernetes.io/docs/concepts/cluster-administration/logging/
- Volumes: []v1.Volume{
- {
- Name: "logging",
- VolumeSource: v1.VolumeSource{
- HostPath: &v1.HostPathVolumeSource{
- Path: "/var/log",
- Type: &dirOrCreate,
- },
- },
- },
- },
- },
- }
- jobSpec := batch_v1beta1.JobTemplateSpec{
- ObjectMeta: metav1.ObjectMeta{
- Name: stresserV2AppName,
- Namespace: ts.cfg.EKSConfig.AddOnStresserRemoteV2.Namespace,
- },
- Spec: batch_v1.JobSpec{
- Completions: aws.Int32(int32(ts.cfg.EKSConfig.AddOnStresserRemoteV2.Completes)),
- Parallelism: aws.Int32(int32(ts.cfg.EKSConfig.AddOnStresserRemoteV2.Parallels)),
- Template: podSpec,
- },
- }
- cronObj := batch_v1beta1.CronJob{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "batch/v1beta1",
- Kind: "CronJob",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: stresserV2AppName,
- Namespace: ts.cfg.EKSConfig.AddOnStresserRemoteV2.Namespace,
- },
- Spec: batch_v1beta1.CronJobSpec{
- Schedule: ts.cfg.EKSConfig.AddOnStresserRemoteV2.Schedule,
- SuccessfulJobsHistoryLimit: aws.Int32(ts.cfg.EKSConfig.AddOnStresserRemoteV2.SuccessfulJobsHistoryLimit),
- FailedJobsHistoryLimit: aws.Int32(ts.cfg.EKSConfig.AddOnStresserRemoteV2.FailedJobsHistoryLimit),
- JobTemplate: jobSpec,
- ConcurrencyPolicy: batch_v1beta1.ReplaceConcurrent,
- },
- }
- b, err := yaml.Marshal(cronObj)
- return cronObj, string(b), err
-}
-func (ts *tester) deleteCronJob() (err error) {
- propagationBackground := metav1.DeletePropagationBackground
- ts.cfg.Logger.Info("deleting CronJob", zap.String("name", stresserV2AppName))
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err = ts.cfg.
- K8SClient.KubernetesClientSet().
- BatchV1beta1().
- CronJobs(ts.cfg.EKSConfig.AddOnCronJobs.Namespace).
- Delete(
- ctx,
- stresserV2AppName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &propagationBackground,
- },
- )
- cancel()
- if err != nil {
- return err
- }
- ts.cfg.Logger.Info("deleted CronJob", zap.String("name", stresserV2AppName))
- return nil
-}
diff --git a/eks/tester/tester.go b/eks/tester/tester.go
deleted file mode 100644
index 67788936c..000000000
--- a/eks/tester/tester.go
+++ /dev/null
@@ -1,24 +0,0 @@
-// Package tester defines EKS tester interface.
-package tester
-
-// Tester defines tester.
-type Tester interface {
- // Name returns the name of the tester.
- Name() string
- // Create creates test objects, and waits for completion.
- Create() error
- // Delete deletes all test objects.
- Delete() error
-}
-
-// Addon is a new interface similar to tester.
-// Instead of Name() reflection is used for object names
-// IsEnabled() allows for generic addon skipping.
-type Addon interface {
- // Apply idempotently creates test objects
- Apply() error
- // Delete idempotently deletes test objects
- Delete() error
- // IsEnabled automatically skips the addon if false
- IsEnabled() bool
-}
diff --git a/eks/trainium/trainium.go b/eks/trainium/trainium.go
deleted file mode 100644
index 65e05f3e1..000000000
--- a/eks/trainium/trainium.go
+++ /dev/null
@@ -1,392 +0,0 @@
-package trainium
-
-import (
- "bytes"
- "context"
- "errors"
- "fmt"
- "html/template"
- "io"
- "reflect"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/eksconfig"
- "github.com/aws/aws-k8s-tester/pkg/fileutil"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "go.uber.org/zap"
- "k8s.io/utils/exec"
-)
-
-// Config defines Trainium configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
-}
-
-// Tester defines Trainium tester.
-type Tester interface {
- // Name returns the name of the tester.
- Name() string
- InstallNeuronDriver() error
- CreateTrainiumJob() error
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-// New creates a new Job tester.
-func New(cfg Config) Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
-}
-
-// neuron device plugin for Kubernetes from
-// https://github.com/aws/aws-neuron-sdk/blob/master/docs/neuron-container-tools/k8s-neuron-device-plugin.yml
-const neuronDriverTemplate = `
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
- name: neuron-device-plugin
-rules:
-- apiGroups:
- - ""
- resources:
- - nodes
- verbs:
- - get
- - list
- - watch
-- apiGroups:
- - ""
- resources:
- - events
- verbs:
- - create
- - patch
-- apiGroups:
- - ""
- resources:
- - pods
- verbs:
- - update
- - patch
- - get
- - list
- - watch
-- apiGroups:
- - ""
- resources:
- - nodes/status
- verbs:
- - patch
- - update
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: neuron-device-plugin
- namespace: kube-system
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
- name: neuron-device-plugin
- namespace: kube-system
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: neuron-device-plugin
-subjects:
-- kind: ServiceAccount
- name: neuron-device-plugin
- namespace: kube-system
----
-# https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
- name: neuron-device-plugin-daemonset
- namespace: kube-system
-spec:
- selector:
- matchLabels:
- name: neuron-device-plugin-ds
- updateStrategy:
- type: RollingUpdate
- template:
- metadata:
- annotations:
- scheduler.alpha.kubernetes.io/critical-pod: ""
- labels:
- name: neuron-device-plugin-ds
- spec:
- serviceAccount: neuron-device-plugin
- tolerations:
- - key: CriticalAddonsOnly
- operator: Exists
- - key: aws.amazon.com/neuron
- operator: Exists
- effect: NoSchedule
- # Mark this pod as a critical add-on; when enabled, the critical add-on
- # scheduler reserves resources for critical add-on pods so that they can
- # be rescheduled after a failure.
- # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
- priorityClassName: "system-node-critical"
- affinity:
- nodeAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- nodeSelectorTerms:
- - matchExpressions:
- - key: "beta.kubernetes.io/instance-type"
- operator: In
- values:
- - inf1.xlarge
- - inf1.2xlarge
- - inf1.6xlarge
- - inf1.24xlarge
- - trn1.2xlarge
- - trn1.32xlarge
- - trn2.48xlarge
- - matchExpressions:
- - key: "node.kubernetes.io/instance-type"
- operator: In
- values:
- - inf1.xlarge
- - inf1.2xlarge
- - inf1.6xlarge
- - inf1.24xlarge
- - trn1.2xlarge
- - trn1.32xlarge
- - trn2.48xlarge
- containers:
- - image: public.ecr.aws/neuron/neuron-device-plugin:1.9.3.0
- imagePullPolicy: IfNotPresent
- name: k8s-neuron-device-plugin-ctr
- env:
- - name: KUBECONFIG
- value: /etc/kubernetes/kubelet.conf
- - name: NODE_NAME
- valueFrom:
- fieldRef:
- fieldPath: spec.nodeName
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop: ["ALL"]
- volumeMounts:
- - name: device-plugin
- mountPath: /var/lib/kubelet/device-plugins
- volumes:
- - name: device-plugin
- hostPath:
- path: /var/lib/kubelet/device-plugins
-
-`
-
-func (ts *tester) InstallNeuronDriver() (err error) {
- ts.cfg.Logger.Info("starting tester.InstallNeuronDriver", zap.String("tester", pkgName))
- fpath, err := fileutil.WriteTempFile([]byte(neuronDriverTemplate))
- if err != nil {
- return err
- }
- applyArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "apply",
- "-f",
- fpath,
- }
- applyCmd := strings.Join(applyArgs, " ")
-
- applied := false
- retryStart, waitDur := time.Now(), 3*time.Minute
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("install neuron driver stopped")
- return nil
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
- output, err := exec.New().CommandContext(ctx, applyArgs[0], applyArgs[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", applyCmd, out)
- if err != nil {
- ts.cfg.Logger.Warn("failed to install Neuron driver", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
-
- applied = true
- ts.cfg.Logger.Info("installed neuron driver")
- break
- }
- if !applied {
- return errors.New("failed to install neuron driver")
- }
-
- ts.cfg.Logger.Info("checking neuron driver")
-
- descArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=kube-system",
- "describe",
- "daemonset.apps/neuron-device-plugin-daemonset",
- }
- descCmd := strings.Join(descArgs, " ")
-
- installed := false
- for time.Since(retryStart) < waitDur {
- ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
- out, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
- cancel()
- output := strings.TrimSpace(string(out))
- if err != nil {
- ts.cfg.Logger.Warn("failed to kubectl describe daemonset.apps/neuron-device-plugin-daemonset", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmd, output)
-
- if strings.Contains(output, "SuccessfulCreate") {
- installed = true
- break
- }
- }
-
- if installed {
- ts.cfg.Logger.Info("checked neuron driver")
- return ts.cfg.EKSConfig.Sync()
- }
- ts.cfg.Logger.Warn("failed to install neuron driver")
- return errors.New("neuron driver installation failed")
-}
-
-const trainiumJobTemplate = `
-apiVersion: batch/v1
-kind: Job
-metadata:
- labels:
- k8s-app: neuron-trainium-test
- name: trainium-client
-spec:
- template:
- metadata:
- name: neuron-trainium-test
- spec:
- containers:
- - name: neuron-trainium-container
- image: {{ .Account }}.dkr.ecr.{{ .Region }}.amazonaws.com/neuron-trainium-test:1.0
- imagePullPolicy: IfNotPresent
- resources:
- limits:
- aws.amazon.com/neuron: 1
- requests:
- aws.amazon.com/neuron: 1
- # Do not restart containers after they exit
- restartPolicy: Never
- # of retries before marking as failed.
- backoffLimit: 3
-`
-
-func (ts *tester) CreateTrainiumJob() error {
- ts.cfg.Logger.Info("starting tester.CreateTrainiumJob", zap.String("tester", pkgName))
- tpl := template.Must(template.New("tmplTrainiumJobTemplate").Parse(trainiumJobTemplate))
- buf := bytes.NewBuffer(nil)
- if err := tpl.Execute(buf, struct {
- Account string
- Region string
- }{
- Account: ts.cfg.EKSConfig.Status.AWSAccountID,
- Region: ts.cfg.EKSConfig.Region,
- }); err != nil {
- return err
- }
- fpath, err := fileutil.WriteTempFile(buf.Bytes())
- if err != nil {
- return err
- }
- applyArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "apply",
- "-f",
- fpath,
- }
- applyCmd := strings.Join(applyArgs, " ")
-
- applied := false
- retryStart, waitDur := time.Now(), 10*time.Minute
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("create trainium job stopped")
- return nil
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
- output, err := exec.New().CommandContext(ctx, applyArgs[0], applyArgs[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", applyCmd, out)
- if err != nil {
- ts.cfg.Logger.Warn("failed to create trainium job", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
-
- applied = true
- ts.cfg.Logger.Info("created trainium job")
- break
- }
- if !applied {
- return errors.New("failed to create trainium job")
- }
-
- ts.cfg.Logger.Info("checking trainium job")
-
- getArgs := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "get",
- "pods",
- "--selector=job-name=trainium-client",
- }
- getCmd := strings.Join(getArgs, " ")
-
- completed := false
- for time.Since(retryStart) < waitDur {
- ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
- out, err := exec.New().CommandContext(ctx, getArgs[0], getArgs[1:]...).CombinedOutput()
- cancel()
- output := strings.TrimSpace(string(out))
- if err != nil {
- ts.cfg.Logger.Warn("failed to kubectl get pods --selector=job-name=trainium-client", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getCmd, output)
-
- if strings.Contains(output, "Completed") {
- completed = true
- break
- }
- }
-
- if completed {
- ts.cfg.Logger.Info("checked trainium job")
- return ts.cfg.EKSConfig.Sync()
- }
- ts.cfg.Logger.Warn("failed to test trainium job")
- return errors.New("trainium job failed")
-}
diff --git a/eks/wordpress/wordpress.go b/eks/wordpress/wordpress.go
deleted file mode 100644
index cdd852d1c..000000000
--- a/eks/wordpress/wordpress.go
+++ /dev/null
@@ -1,417 +0,0 @@
-// Package wordpress implements wordpress add-on.
-package wordpress
-
-import (
- "context"
- "errors"
- "fmt"
- "io"
- "io/ioutil"
- "reflect"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/eks/helm"
- eks_tester "github.com/aws/aws-k8s-tester/eks/tester"
- "github.com/aws/aws-k8s-tester/eksconfig"
- "github.com/aws/aws-k8s-tester/pkg/aws/elb"
- "github.com/aws/aws-k8s-tester/pkg/httputil"
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface"
- "go.uber.org/zap"
- apierrs "k8s.io/apimachinery/pkg/api/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/utils/exec"
-)
-
-// Config defines Wordpress configuration.
-type Config struct {
- Logger *zap.Logger
- LogWriter io.Writer
- Stopc chan struct{}
- EKSConfig *eksconfig.Config
- K8SClient k8s_client.EKS
- ELB2API elbv2iface.ELBV2API
-}
-
-var pkgName = reflect.TypeOf(tester{}).PkgPath()
-
-func (ts *tester) Name() string { return pkgName }
-
-func New(cfg Config) eks_tester.Tester {
- cfg.Logger.Info("creating tester", zap.String("tester", pkgName))
- return &tester{cfg: cfg}
-}
-
-type tester struct {
- cfg Config
-}
-
-const (
- chartRepoName = "bitnami"
- chartRepoURL = "https://charts.bitnami.com/bitnami"
- chartName = "wordpress"
-)
-
-func (ts *tester) Create() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnWordpress() {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
- if ts.cfg.EKSConfig.AddOnWordpress.Created {
- ts.cfg.Logger.Info("skipping tester.Create", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Create", zap.String("tester", pkgName))
- ts.cfg.EKSConfig.AddOnWordpress.Created = true
- ts.cfg.EKSConfig.Sync()
- createStart := time.Now()
- defer func() {
- createEnd := time.Now()
- ts.cfg.EKSConfig.AddOnWordpress.TimeFrameCreate = timeutil.NewTimeFrame(createStart, createEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- if err := k8s_client.CreateNamespace(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnWordpress.Namespace,
- ); err != nil {
- return err
- }
- if err := helm.RepoAdd(ts.cfg.Logger, chartRepoName, chartRepoURL); err != nil {
- return err
- }
- if err := ts.createHelmWordpress(); err != nil {
- return err
- }
- if err := ts.waitService(); err != nil {
- return err
- }
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-func (ts *tester) Delete() error {
- if !ts.cfg.EKSConfig.IsEnabledAddOnWordpress() {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
- if !ts.cfg.EKSConfig.AddOnWordpress.Created {
- ts.cfg.Logger.Info("skipping tester.Delete", zap.String("tester", pkgName))
- return nil
- }
-
- ts.cfg.Logger.Info("starting tester.Delete", zap.String("tester", pkgName))
- deleteStart := time.Now()
- defer func() {
- deleteEnd := time.Now()
- ts.cfg.EKSConfig.AddOnWordpress.TimeFrameDelete = timeutil.NewTimeFrame(deleteStart, deleteEnd)
- ts.cfg.EKSConfig.Sync()
- }()
-
- var errs []string
-
- if err := ts.deleteHelmWordpress(); err != nil {
- errs = append(errs, err.Error())
- }
-
- if err := ts.deleteService(); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete WordPress Service (%v)", err))
- }
- ts.cfg.Logger.Info("wait for 3-minute after deleting Service")
- time.Sleep(3 * time.Minute)
-
- /*
- # NLB tags
- kubernetes.io/service-name
- leegyuho-test-prod-nlb-hello-world/hello-world-service
-
- kubernetes.io/cluster/leegyuho-test-prod
- owned
- */
- if err := elb.DeleteELBv2(
- ts.cfg.Logger,
- ts.cfg.ELB2API,
- ts.cfg.EKSConfig.AddOnWordpress.NLBARN,
- ts.cfg.EKSConfig.VPC.ID,
- map[string]string{
- "kubernetes.io/cluster/" + ts.cfg.EKSConfig.Name: "owned",
- "kubernetes.io/service-name": ts.cfg.EKSConfig.AddOnWordpress.Namespace + "/" + wordpressServiceName,
- },
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete WordPress (%v)", err))
- }
-
- if err := k8s_client.DeleteNamespaceAndWait(
- ts.cfg.Logger,
- ts.cfg.K8SClient.KubernetesClientSet(),
- ts.cfg.EKSConfig.AddOnWordpress.Namespace,
- k8s_client.DefaultNamespaceDeletionInterval,
- k8s_client.DefaultNamespaceDeletionTimeout,
- k8s_client.WithForceDelete(true),
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete Wordpress namespace (%v)", err))
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- ts.cfg.EKSConfig.AddOnWordpress.Created = false
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-// https://github.com/helm/charts/blob/master/stable/wordpress/values.yaml
-// https://github.com/helm/charts/blob/master/stable/mariadb/values.yaml
-func (ts *tester) createHelmWordpress() error {
- ngType := "managed"
- if ts.cfg.EKSConfig.IsEnabledAddOnNodeGroups() {
- // TODO: test in MNG
- ngType = "custom"
- }
-
- // https://github.com/helm/charts/blob/master/stable/wordpress/values.yaml
- values := map[string]interface{}{
- "nodeSelector": map[string]interface{}{
- // do not deploy in bottlerocket; PVC not working
- "AMIType": ec2config.AMITypeAL2X8664,
- "NGType": ngType,
- },
- "wordpressUsername": ts.cfg.EKSConfig.AddOnWordpress.UserName,
- "wordpressPassword": ts.cfg.EKSConfig.AddOnWordpress.Password,
- "persistence": map[string]interface{}{
- "enabled": true,
- // use CSI driver with volume type "gp2", as in launch configuration
- "storageClassName": "gp2",
- },
- // https://github.com/helm/charts/blob/master/stable/mariadb/values.yaml
- "mariadb": map[string]interface{}{
- "enabled": true,
- "rootUser": map[string]interface{}{
- "password": ts.cfg.EKSConfig.AddOnWordpress.Password,
- "forcePassword": false,
- },
- "db": map[string]interface{}{
- "name": "wordpress",
- "user": ts.cfg.EKSConfig.AddOnWordpress.UserName,
- "password": ts.cfg.EKSConfig.AddOnWordpress.Password,
- },
- "master": map[string]interface{}{
- "nodeSelector": map[string]interface{}{
- // do not deploy in bottlerocket; PVC not working
- "AMIType": ec2config.AMITypeAL2X8664,
- "NGType": ngType,
- },
- "persistence": map[string]interface{}{
- "enabled": true,
- // use CSI driver with volume type "gp2", as in launch configuration
- "storageClassName": "gp2",
- },
- },
- "slave": map[string]interface{}{
- "nodeSelector": map[string]interface{}{
- // do not deploy in bottlerocket; PVC not working
- "AMIType": ec2config.AMITypeAL2X8664,
- "NGType": ngType,
- },
- },
- },
- }
-
- return helm.Install(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Stopc: ts.cfg.Stopc,
- Timeout: 15 * time.Minute,
- KubeConfigPath: ts.cfg.EKSConfig.KubeConfigPath,
- Namespace: ts.cfg.EKSConfig.AddOnWordpress.Namespace,
- ChartRepoURL: chartRepoURL,
- ChartName: chartName,
- ReleaseName: chartName,
- Values: values,
- QueryFunc: nil,
- QueryInterval: 30 * time.Second,
- })
-}
-
-func (ts *tester) deleteHelmWordpress() error {
- return helm.Uninstall(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Timeout: 15 * time.Minute,
- KubeConfigPath: ts.cfg.EKSConfig.KubeConfigPath,
- Namespace: ts.cfg.EKSConfig.AddOnWordpress.Namespace,
- ChartName: chartName,
- ReleaseName: chartName,
- })
-}
-
-func (ts *tester) waitService() error {
- ts.cfg.Logger.Info("waiting for WordPress service")
-
- waitDur := 2 * time.Minute
- ts.cfg.Logger.Info("waiting for WordPress service", zap.Duration("wait", waitDur))
- select {
- case <-ts.cfg.Stopc:
- return errors.New("WordPress service creation aborted")
- case <-time.After(waitDur):
- }
-
- args := []string{
- ts.cfg.EKSConfig.KubectlPath,
- "--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
- "--namespace=" + ts.cfg.EKSConfig.AddOnWordpress.Namespace,
- "describe",
- "svc",
- wordpressServiceName,
- }
- argsCmd := strings.Join(args, " ")
- hostName := ""
- retryStart := time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("WordPress service creation aborted")
- case <-time.After(5 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- cmdOut, err := exec.New().CommandContext(ctx, args[0], args[1:]...).CombinedOutput()
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe svc' failed", zap.String("command", argsCmd), zap.Error(err))
- } else {
- out := string(cmdOut)
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\" output:\n%s\n\n", argsCmd, out)
- }
-
- ts.cfg.Logger.Info("querying WordPress service for HTTP endpoint")
- ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
- so, err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(ts.cfg.EKSConfig.AddOnWordpress.Namespace).
- Get(ctx, wordpressServiceName, metav1.GetOptions{})
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("failed to get WordPress service; retrying", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
-
- ts.cfg.Logger.Info(
- "WordPress service has been linked to LoadBalancer",
- zap.String("load-balancer", fmt.Sprintf("%+v", so.Status.LoadBalancer)),
- )
- for _, ing := range so.Status.LoadBalancer.Ingress {
- ts.cfg.Logger.Info(
- "WordPress service has been linked to LoadBalancer.Ingress",
- zap.String("ingress", fmt.Sprintf("%+v", ing)),
- )
- hostName = ing.Hostname
- break
- }
-
- if hostName != "" {
- ts.cfg.Logger.Info("found host name", zap.String("host-name", hostName))
- break
- }
- }
-
- if hostName == "" {
- return errors.New("failed to find host name")
- }
-
- ts.cfg.EKSConfig.AddOnWordpress.URL = "http://" + hostName
-
- // TODO: is there any better way to find out the NLB name?
- ts.cfg.EKSConfig.AddOnWordpress.NLBName = strings.Split(hostName, "-")[0]
- ss := strings.Split(hostName, ".")[0]
- ss = strings.Replace(ss, "-", "/", -1)
- ts.cfg.EKSConfig.AddOnWordpress.NLBARN = fmt.Sprintf(
- "arn:aws:elasticloadbalancing:%s:%s:loadbalancer/net/%s",
- ts.cfg.EKSConfig.Region,
- ts.cfg.EKSConfig.Status.AWSAccountID,
- ss,
- )
-
- fmt.Fprintf(ts.cfg.LogWriter, "\nNLB WordPress ARN: %s\n", ts.cfg.EKSConfig.AddOnWordpress.NLBARN)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB WordPress Name: %s\n", ts.cfg.EKSConfig.AddOnWordpress.NLBName)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB WordPress URL: %s\n", ts.cfg.EKSConfig.AddOnWordpress.URL)
- fmt.Fprintf(ts.cfg.LogWriter, "WordPress UserName: %s\n", ts.cfg.EKSConfig.AddOnWordpress.UserName)
- fmt.Fprintf(ts.cfg.LogWriter, "WordPress Password: %d characters\n\n", len(ts.cfg.EKSConfig.AddOnWordpress.Password))
-
- ts.cfg.Logger.Info("waiting before testing WordPress Service")
- time.Sleep(20 * time.Second)
-
- retryStart = time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("WordPress Service creation aborted")
- case <-time.After(5 * time.Second):
- }
-
- out, err := httputil.ReadInsecure(ts.cfg.Logger, ioutil.Discard, ts.cfg.EKSConfig.AddOnWordpress.URL)
- if err != nil {
- ts.cfg.Logger.Warn("failed to read NLB WordPress Service; retrying", zap.Error(err))
- time.Sleep(5 * time.Second)
- continue
- }
- httpOutput := string(out)
- fmt.Fprintf(ts.cfg.LogWriter, "\nNLB WordPress Service output:\n%s\n", httpOutput)
-
- if strings.Contains(httpOutput, `Welcome to WordPress. This is your first post`) {
- ts.cfg.Logger.Info(
- "read WordPress Service; exiting",
- zap.String("host-name", hostName),
- )
- break
- }
-
- ts.cfg.Logger.Warn("unexpected WordPress Service output; retrying")
- }
-
- fmt.Fprintf(ts.cfg.LogWriter, "\nNLB WordPress ARN: %s\n", ts.cfg.EKSConfig.AddOnWordpress.NLBARN)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB WordPress Name: %s\n", ts.cfg.EKSConfig.AddOnWordpress.NLBName)
- fmt.Fprintf(ts.cfg.LogWriter, "NLB WordPress URL: %s\n", ts.cfg.EKSConfig.AddOnWordpress.URL)
- fmt.Fprintf(ts.cfg.LogWriter, "WordPress UserName: %s\n", ts.cfg.EKSConfig.AddOnWordpress.UserName)
- fmt.Fprintf(ts.cfg.LogWriter, "WordPress Password: %d characters\n\n", len(ts.cfg.EKSConfig.AddOnWordpress.Password))
-
- ts.cfg.EKSConfig.Sync()
- return nil
-}
-
-const wordpressServiceName = "wordpress"
-
-func (ts *tester) deleteService() error {
- ts.cfg.Logger.Info("deleting wordpress Service")
- foreground := metav1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.K8SClient.KubernetesClientSet().
- CoreV1().
- Services(ts.cfg.EKSConfig.AddOnWordpress.Namespace).
- Delete(
- ctx,
- wordpressServiceName,
- metav1.DeleteOptions{
- GracePeriodSeconds: aws.Int64(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !apierrs.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete wordpress Service (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted wordpress Service", zap.Error(err))
- ts.cfg.EKSConfig.Sync()
- return nil
-}
diff --git a/eksconfig/README.md b/eksconfig/README.md
deleted file mode 100644
index 374b41df7..000000000
--- a/eksconfig/README.md
+++ /dev/null
@@ -1,1090 +0,0 @@
-
-```
-# total 36 add-ons
-# set the following *_ENABLE env vars to enable add-ons, rest are set with default values
-AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_CW_AGENT_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_METRICS_SERVER_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_CSI_EBS_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_KUBERNETES_DASHBOARD_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_IRSA_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_CUDA_VECTOR_ADD_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_ENABLE=true \
-AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_VERSION_UPGRADE_ENABLE=true \
-
-
-
-*----------------------------------------------------------------*-------------------*----------------------------------------------------------*-------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*----------------------------------------------------------------*-------------------*----------------------------------------------------------*-------------------*
-| AWS_K8S_TESTER_EKS_,INLINE | read-only "false" | *eksconfig.Config.TypeMeta | v1.TypeMeta |
-| AWS_K8S_TESTER_EKS_METADATA | read-only "false" | *eksconfig.Config.ObjectMeta | v1.ObjectMeta |
-| AWS_K8S_TESTER_EKS_NAME | read-only "false" | *eksconfig.Config.Name | string |
-| AWS_K8S_TESTER_EKS_PARTITION | read-only "false" | *eksconfig.Config.Partition | string |
-| AWS_K8S_TESTER_EKS_REGION | read-only "false" | *eksconfig.Config.Region | string |
-| AWS_K8S_TESTER_EKS_AVAILABILITY_ZONE_NAMES | read-only "true" | *eksconfig.Config.AvailabilityZoneNames | []string |
-| AWS_K8S_TESTER_EKS_CONFIG_PATH | read-only "false" | *eksconfig.Config.ConfigPath | string |
-| AWS_K8S_TESTER_EKS_KUBECTL_COMMANDS_OUTPUT_PATH | read-only "false" | *eksconfig.Config.KubectlCommandsOutputPath | string |
-| AWS_K8S_TESTER_EKS_REMOTE_ACCESS_COMMANDS_OUTPUT_PATH | read-only "false" | *eksconfig.Config.RemoteAccessCommandsOutputPath | string |
-| AWS_K8S_TESTER_EKS_LOG_COLOR | read-only "false" | *eksconfig.Config.LogColor | bool |
-| AWS_K8S_TESTER_EKS_LOG_COLOR_OVERRIDE | read-only "false" | *eksconfig.Config.LogColorOverride | string |
-| AWS_K8S_TESTER_EKS_LOG_LEVEL | read-only "false" | *eksconfig.Config.LogLevel | string |
-| AWS_K8S_TESTER_EKS_LOG_OUTPUTS | read-only "false" | *eksconfig.Config.LogOutputs | []string |
-| AWS_K8S_TESTER_EKS_AWS_CLI_PATH | read-only "false" | *eksconfig.Config.AWSCLIPath | string |
-| AWS_K8S_TESTER_EKS_KUBECTL_PATH | read-only "false" | *eksconfig.Config.KubectlPath | string |
-| AWS_K8S_TESTER_EKS_KUBECTL_DOWNLOAD_URL | read-only "false" | *eksconfig.Config.KubectlDownloadURL | string |
-| AWS_K8S_TESTER_EKS_KUBECONFIG_PATH | read-only "false" | *eksconfig.Config.KubeConfigPath | string |
-| AWS_K8S_TESTER_EKS_AWS_IAM_AUTHENTICATOR_PATH | read-only "false" | *eksconfig.Config.AWSIAMAuthenticatorPath | string |
-| AWS_K8S_TESTER_EKS_AWS_IAM_AUTHENTICATOR_DOWNLOAD_URL | read-only "false" | *eksconfig.Config.AWSIAMAuthenticatorDownloadURL | string |
-| AWS_K8S_TESTER_EKS_ON_FAILURE_DELETE | read-only "false" | *eksconfig.Config.OnFailureDelete | bool |
-| AWS_K8S_TESTER_EKS_ON_FAILURE_DELETE_WAIT_SECONDS | read-only "false" | *eksconfig.Config.OnFailureDeleteWaitSeconds | uint64 |
-| AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_CLUSTER | read-only "false" | *eksconfig.Config.CommandAfterCreateCluster | string |
-| AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_CLUSTER_OUTPUT_PATH | read-only "true" | *eksconfig.Config.CommandAfterCreateClusterOutputPath | string |
-| AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_CLUSTER_TIMEOUT | read-only "false" | *eksconfig.Config.CommandAfterCreateClusterTimeout | time.Duration |
-| AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_CLUSTER_TIMEOUT_STRING | read-only "true" | *eksconfig.Config.CommandAfterCreateClusterTimeoutString | string |
-| AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_ADD_ONS | read-only "false" | *eksconfig.Config.CommandAfterCreateAddOns | string |
-| AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_ADD_ONS_OUTPUT_PATH | read-only "true" | *eksconfig.Config.CommandAfterCreateAddOnsOutputPath | string |
-| AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_ADD_ONS_TIMEOUT | read-only "false" | *eksconfig.Config.CommandAfterCreateAddOnsTimeout | time.Duration |
-| AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_ADD_ONS_TIMEOUT_STRING | read-only "true" | *eksconfig.Config.CommandAfterCreateAddOnsTimeoutString | string |
-| AWS_K8S_TESTER_EKS_CW_NAMESPACE | read-only "false" | *eksconfig.Config.CWNamespace | string |
-| AWS_K8S_TESTER_EKS_SKIP_DELETE_CLUSTER_AND_NODES | read-only "false" | *eksconfig.Config.SkipDeleteClusterAndNodes | bool |
-| AWS_K8S_TESTER_EKS_TAGS | read-only "false" | *eksconfig.Config.Tags | map[string]string |
-| AWS_K8S_TESTER_EKS_REQUEST_HEADER_KEY | read-only "false" | *eksconfig.Config.RequestHeaderKey | string |
-| AWS_K8S_TESTER_EKS_REQUEST_HEADER_VALUE | read-only "false" | *eksconfig.Config.RequestHeaderValue | string |
-| AWS_K8S_TESTER_EKS_RESOLVER_URL | read-only "false" | *eksconfig.Config.ResolverURL | string |
-| AWS_K8S_TESTER_EKS_SIGNING_NAME | read-only "false" | *eksconfig.Config.SigningName | string |
-| AWS_K8S_TESTER_EKS_VERSION | read-only "false" | *eksconfig.Config.Version | string |
-| AWS_K8S_TESTER_EKS_VERSION_VALUE | read-only "true" | *eksconfig.Config.VersionValue | float64 |
-| AWS_K8S_TESTER_EKS_KUBE_APISERVER_MAX_REQUESTS_INFLIGHT | read-only "false" | *eksconfig.Config.KubeAPIServerMaxRequestsInflight | string |
-| AWS_K8S_TESTER_EKS_KUBE_CONTROLLER_MANAGER_QPS | read-only "false" | *eksconfig.Config.KubeControllerManagerQPS | string |
-| AWS_K8S_TESTER_EKS_KUBE_CONTROLLER_MANAGER_BURST | read-only "false" | *eksconfig.Config.KubeControllerManagerBurst | string |
-| AWS_K8S_TESTER_EKS_KUBE_SCHEDULER_QPS | read-only "false" | *eksconfig.Config.KubeSchedulerQPS | string |
-| AWS_K8S_TESTER_EKS_KUBE_SCHEDULER_BURST | read-only "false" | *eksconfig.Config.KubeSchedulerBurst | string |
-| AWS_K8S_TESTER_EKS_FE_UPDATE_MASTER_FLAGS_URL | read-only "false" | *eksconfig.Config.FEUpdateMasterFlagsURL | string |
-| AWS_K8S_TESTER_EKS_REMOTE_ACCESS_KEY_CREATE | read-only "false" | *eksconfig.Config.RemoteAccessKeyCreate | bool |
-| AWS_K8S_TESTER_EKS_REMOTE_ACCESS_KEY_NAME | read-only "false" | *eksconfig.Config.RemoteAccessKeyName | string |
-| AWS_K8S_TESTER_EKS_REMOTE_ACCESS_PRIVATE_KEY_PATH | read-only "false" | *eksconfig.Config.RemoteAccessPrivateKeyPath | string |
-| AWS_K8S_TESTER_EKS_CLIENTS | read-only "false" | *eksconfig.Config.Clients | int |
-| AWS_K8S_TESTER_EKS_CLIENT_QPS | read-only "false" | *eksconfig.Config.ClientQPS | float32 |
-| AWS_K8S_TESTER_EKS_CLIENT_BURST | read-only "false" | *eksconfig.Config.ClientBurst | int |
-| AWS_K8S_TESTER_EKS_CLIENT_TIMEOUT | read-only "false" | *eksconfig.Config.ClientTimeout | time.Duration |
-| AWS_K8S_TESTER_EKS_CLIENT_TIMEOUT_STRING | read-only "true" | *eksconfig.Config.ClientTimeoutString | string |
-| AWS_K8S_TESTER_EKS_TOTAL_NODES | read-only "true" | *eksconfig.Config.TotalNodes | int32 |
-| AWS_K8S_TESTER_EKS_SPEC | read-only "false" | *eksconfig.Config.Spec | eksconfig.Spec |
-*----------------------------------------------------------------*-------------------*----------------------------------------------------------*-------------------*
-
-
-*--------------------------------------------------------*-------------------*---------------------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*--------------------------------------------------------*-------------------*---------------------------------------------*---------*
-| AWS_K8S_TESTER_EKS_S3_BUCKET_CREATE | read-only "false" | *eksconfig.S3.BucketCreate | bool |
-| AWS_K8S_TESTER_EKS_S3_BUCKET_CREATE_KEEP | read-only "false" | *eksconfig.S3.BucketCreateKeep | bool |
-| AWS_K8S_TESTER_EKS_S3_BUCKET_NAME | read-only "false" | *eksconfig.S3.BucketName | string |
-| AWS_K8S_TESTER_EKS_S3_BUCKET_LIFECYCLE_EXPIRATION_DAYS | read-only "false" | *eksconfig.S3.BucketLifecycleExpirationDays | int64 |
-*--------------------------------------------------------*-------------------*---------------------------------------------*---------*
-
-
-*------------------------------------------*-------------------*---------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*------------------------------------------*-------------------*---------------------------------*---------*
-| AWS_K8S_TESTER_EKS_ENCRYPTION_CMK_CREATE | read-only "false" | *eksconfig.Encryption.CMKCreate | bool |
-| AWS_K8S_TESTER_EKS_ENCRYPTION_CMK_ARN | read-only "false" | *eksconfig.Encryption.CMKARN | string |
-*------------------------------------------*-------------------*---------------------------------*---------*
-
-
-*-----------------------------------------------*-------------------*-------------------------------------*----------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-----------------------------------------------*-------------------*-------------------------------------*----------*
-| AWS_K8S_TESTER_EKS_ROLE_NAME | read-only "false" | *eksconfig.Role.Name | string |
-| AWS_K8S_TESTER_EKS_ROLE_CREATE | read-only "false" | *eksconfig.Role.Create | bool |
-| AWS_K8S_TESTER_EKS_ROLE_ARN | read-only "false" | *eksconfig.Role.ARN | string |
-| AWS_K8S_TESTER_EKS_ROLE_SERVICE_PRINCIPALS | read-only "false" | *eksconfig.Role.ServicePrincipals | []string |
-| AWS_K8S_TESTER_EKS_ROLE_MANAGED_POLICY_ARNS | read-only "false" | *eksconfig.Role.ManagedPolicyARNs | []string |
-| AWS_K8S_TESTER_EKS_ROLE_POLICY_NAME | read-only "true" | *eksconfig.Role.PolicyName | string |
-| AWS_K8S_TESTER_EKS_ROLE_POLICY_ARN | read-only "true" | *eksconfig.Role.PolicyARN | string |
-| AWS_K8S_TESTER_EKS_ROLE_INSTANCE_PROFILE_NAME | read-only "true" | *eksconfig.Role.InstanceProfileName | string |
-| AWS_K8S_TESTER_EKS_ROLE_INSTANCE_PROFILE_ARN | read-only "true" | *eksconfig.Role.InstanceProfileARN | string |
-*-----------------------------------------------*-------------------*-------------------------------------*----------*
-
-
-*-------------------------------------------------------------------*-------------------*------------------------------------------------------*----------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-------------------------------------------------------------------*-------------------*------------------------------------------------------*----------*
-| AWS_K8S_TESTER_EKS_VPC_CREATE | read-only "false" | *eksconfig.VPC.Create | bool |
-| AWS_K8S_TESTER_EKS_VPC_ID | read-only "false" | *eksconfig.VPC.ID | string |
-| AWS_K8S_TESTER_EKS_VPC_SECURITY_GROUP_ID | read-only "true" | *eksconfig.VPC.SecurityGroupID | string |
-| AWS_K8S_TESTER_EKS_VPC_CIDRS | read-only "false" | *eksconfig.VPC.CIDRs | []string |
-| AWS_K8S_TESTER_EKS_VPC_PUBLIC_SUBNET_CIDRS | read-only "false" | *eksconfig.VPC.PublicSubnetCIDRs | []string |
-| AWS_K8S_TESTER_EKS_VPC_PUBLIC_SUBNET_IDS | read-only "true" | *eksconfig.VPC.PublicSubnetIDs | []string |
-| AWS_K8S_TESTER_EKS_VPC_INTERNET_GATEWAY_ID | read-only "true" | *eksconfig.VPC.InternetGatewayID | string |
-| AWS_K8S_TESTER_EKS_VPC_PUBLIC_ROUTE_TABLE_ID | read-only "true" | *eksconfig.VPC.PublicRouteTableID | string |
-| AWS_K8S_TESTER_EKS_VPC_PUBLIC_SUBNET_ROUTE_TABLE_ASSOCIATION_IDS | read-only "true" | *eksconfig.VPC.PublicSubnetRouteTableAssociationIDs | []string |
-| AWS_K8S_TESTER_EKS_VPC_EIP_ALLOCATION_IDS | read-only "true" | *eksconfig.VPC.EIPAllocationIDs | []string |
-| AWS_K8S_TESTER_EKS_VPC_NAT_GATEWAY_IDS | read-only "true" | *eksconfig.VPC.NATGatewayIDs | []string |
-| AWS_K8S_TESTER_EKS_VPC_PRIVATE_SUBNET_CIDRS | read-only "false" | *eksconfig.VPC.PrivateSubnetCIDRs | []string |
-| AWS_K8S_TESTER_EKS_VPC_PRIVATE_SUBNET_IDS | read-only "true" | *eksconfig.VPC.PrivateSubnetIDs | []string |
-| AWS_K8S_TESTER_EKS_VPC_PRIVATE_ROUTE_TABLE_IDS | read-only "true" | *eksconfig.VPC.PrivateRouteTableIDs | []string |
-| AWS_K8S_TESTER_EKS_VPC_PRIVATE_SUBNET_ROUTE_TABLE_ASSOCIATION_IDS | read-only "true" | *eksconfig.VPC.PrivateSubnetRouteTableAssociationIDs | []string |
-| AWS_K8S_TESTER_EKS_VPC_DHCP_OPTIONS_DOMAIN_NAME | read-only "false" | *eksconfig.VPC.DHCPOptionsDomainName | string |
-| AWS_K8S_TESTER_EKS_VPC_DHCP_OPTIONS_DOMAIN_NAME_SERVERS | read-only "false" | *eksconfig.VPC.DHCPOptionsDomainNameServers | []string |
-| AWS_K8S_TESTER_EKS_VPC_DHCP_OPTIONS_ID | read-only "true" | *eksconfig.VPC.DHCPOptionsID | string |
-| AWS_K8S_TESTER_EKS_VPC_NODE_GROUP_SECURITY_GROUP_NAME | read-only "true" | *eksconfig.VPC.NodeGroupSecurityGroupName | string |
-| AWS_K8S_TESTER_EKS_VPC_NODE_GROUP_SECURITY_GROUP_ID | read-only "true" | *eksconfig.VPC.NodeGroupSecurityGroupID | string |
-*-------------------------------------------------------------------*-------------------*------------------------------------------------------*----------*
-
-
-*--------------------------------------------------------------*-------------------*------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*--------------------------------------------------------------*-------------------*------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_ENABLE | read-only "false" | *eksconfig.AddOnCNIVPC.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_CREATED | read-only "true" | *eksconfig.AddOnCNIVPC.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnCNIVPC.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnCNIVPC.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_VERSION | read-only "false" | *eksconfig.AddOnCNIVPC.Version | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_INIT_ACCOUNT_ID | read-only "false" | *eksconfig.AddOnCNIVPC.RepositoryInitAccountID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_INIT_REGION | read-only "false" | *eksconfig.AddOnCNIVPC.RepositoryInitRegion | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_INIT_NAME | read-only "false" | *eksconfig.AddOnCNIVPC.RepositoryInitName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_INIT_IMAGE_TAG | read-only "false" | *eksconfig.AddOnCNIVPC.RepositoryInitImageTag | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_ACCOUNT_ID | read-only "false" | *eksconfig.AddOnCNIVPC.RepositoryAccountID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_REGION | read-only "false" | *eksconfig.AddOnCNIVPC.RepositoryRegion | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_NAME | read-only "false" | *eksconfig.AddOnCNIVPC.RepositoryName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_IMAGE_TAG | read-only "false" | *eksconfig.AddOnCNIVPC.RepositoryImageTag | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_MINIMUM_IP_TARGET | read-only "false" | *eksconfig.AddOnCNIVPC.MinimumIPTarget | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_WARM_IP_TARGET | read-only "false" | *eksconfig.AddOnCNIVPC.WarmIPTarget | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_NODE_SELECTOR | read-only "false" | *eksconfig.AddOnCNIVPC.NodeSelector | map[string]string |
-*--------------------------------------------------------------*-------------------*------------------------------------------------*--------------------*
-
-
-*---------------------------------------------------------*-------------------*--------------------------------------------*--------------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*---------------------------------------------------------*-------------------*--------------------------------------------*--------------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ENABLE | read-only "false" | *eksconfig.AddOnNodeGroups.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_CREATED | read-only "true" | *eksconfig.AddOnNodeGroups.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnNodeGroups.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnNodeGroups.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_FETCH_LOGS | read-only "false" | *eksconfig.AddOnNodeGroups.FetchLogs | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_LOGS_DIR | read-only "false" | *eksconfig.AddOnNodeGroups.LogsDir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_LOGS_TAR_GZ_PATH | read-only "false" | *eksconfig.AddOnNodeGroups.LogsTarGzPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS | read-only "false" | *eksconfig.AddOnNodeGroups.ASGs | map[string]eksconfig.ASG |
-*---------------------------------------------------------*-------------------*--------------------------------------------*--------------------------*
-
-
-*------------------------------------------------------------------*-------------------*-------------------------------------*----------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*------------------------------------------------------------------*-------------------*-------------------------------------*----------*
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_NAME | read-only "false" | *eksconfig.Role.Name | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_CREATE | read-only "false" | *eksconfig.Role.Create | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_ARN | read-only "false" | *eksconfig.Role.ARN | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_SERVICE_PRINCIPALS | read-only "false" | *eksconfig.Role.ServicePrincipals | []string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_MANAGED_POLICY_ARNS | read-only "false" | *eksconfig.Role.ManagedPolicyARNs | []string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_POLICY_NAME | read-only "true" | *eksconfig.Role.PolicyName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_POLICY_ARN | read-only "true" | *eksconfig.Role.PolicyARN | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_INSTANCE_PROFILE_NAME | read-only "true" | *eksconfig.Role.InstanceProfileName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_INSTANCE_PROFILE_ARN | read-only "true" | *eksconfig.Role.InstanceProfileARN | string |
-*------------------------------------------------------------------*-------------------*-------------------------------------*----------*
-
-
-*--------------------------------------------------------------------*-------------------*------------------------------------------------------*--------------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*--------------------------------------------------------------------*-------------------*------------------------------------------------------*--------------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE | read-only "false" | *eksconfig.AddOnManagedNodeGroups.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_CREATED | read-only "true" | *eksconfig.AddOnManagedNodeGroups.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnManagedNodeGroups.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnManagedNodeGroups.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_FETCH_LOGS | read-only "false" | *eksconfig.AddOnManagedNodeGroups.FetchLogs | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_REQUEST_HEADER_KEY | read-only "false" | *eksconfig.AddOnManagedNodeGroups.RequestHeaderKey | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_REQUEST_HEADER_VALUE | read-only "false" | *eksconfig.AddOnManagedNodeGroups.RequestHeaderValue | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_RESOLVER_URL | read-only "false" | *eksconfig.AddOnManagedNodeGroups.ResolverURL | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_SIGNING_NAME | read-only "false" | *eksconfig.AddOnManagedNodeGroups.SigningName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_LOGS_DIR | read-only "false" | *eksconfig.AddOnManagedNodeGroups.LogsDir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_LOGS_TAR_GZ_PATH | read-only "false" | *eksconfig.AddOnManagedNodeGroups.LogsTarGzPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS | read-only "false" | *eksconfig.AddOnManagedNodeGroups.MNGs | map[string]eksconfig.MNG |
-*--------------------------------------------------------------------*-------------------*------------------------------------------------------*--------------------------*
-
-
-*--------------------------------------------------------------------------*-------------------*-------------------------------------*----------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*--------------------------------------------------------------------------*-------------------*-------------------------------------*----------*
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_NAME | read-only "false" | *eksconfig.Role.Name | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_CREATE | read-only "false" | *eksconfig.Role.Create | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_ARN | read-only "false" | *eksconfig.Role.ARN | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_SERVICE_PRINCIPALS | read-only "false" | *eksconfig.Role.ServicePrincipals | []string |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_MANAGED_POLICY_ARNS | read-only "false" | *eksconfig.Role.ManagedPolicyARNs | []string |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_POLICY_NAME | read-only "true" | *eksconfig.Role.PolicyName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_POLICY_ARN | read-only "true" | *eksconfig.Role.PolicyARN | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_INSTANCE_PROFILE_NAME | read-only "true" | *eksconfig.Role.InstanceProfileName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_INSTANCE_PROFILE_ARN | read-only "true" | *eksconfig.Role.InstanceProfileARN | string |
-*--------------------------------------------------------------------------*-------------------*-------------------------------------*----------*
-
-
-*------------------------------------------------------*-------------------*-----------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*------------------------------------------------------*-------------------*-----------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_CW_AGENT_ENABLE | read-only "false" | *eksconfig.AddOnCWAgent.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CW_AGENT_CREATED | read-only "true" | *eksconfig.AddOnCWAgent.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CW_AGENT_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnCWAgent.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CW_AGENT_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnCWAgent.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CW_AGENT_NAMESPACE | read-only "false" | *eksconfig.AddOnCWAgent.Namespace | string |
-*------------------------------------------------------*-------------------*-----------------------------------------*--------------------*
-
-
-*--------------------------------------------------------------------*-------------------*-------------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*--------------------------------------------------------------------*-------------------*-------------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_ENABLE | read-only "false" | *eksconfig.AddOnFluentd.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_CREATED | read-only "true" | *eksconfig.AddOnFluentd.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnFluentd.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnFluentd.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_NAMESPACE | read-only "false" | *eksconfig.AddOnFluentd.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_REPOSITORY_BUSYBOX_ACCOUNT_ID | read-only "false" | *eksconfig.AddOnFluentd.RepositoryBusyboxAccountID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_REPOSITORY_BUSYBOX_REGION | read-only "false" | *eksconfig.AddOnFluentd.RepositoryBusyboxRegion | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_REPOSITORY_BUSYBOX_NAME | read-only "false" | *eksconfig.AddOnFluentd.RepositoryBusyboxName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_REPOSITORY_BUSYBOX_IMAGE_TAG | read-only "false" | *eksconfig.AddOnFluentd.RepositoryBusyboxImageTag | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_THREADS | read-only "false" | *eksconfig.AddOnFluentd.Threads | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_METADATA_LOG_LEVEL | read-only "false" | *eksconfig.AddOnFluentd.MetadataLogLevel | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_METADATA_CACHE_SIZE | read-only "false" | *eksconfig.AddOnFluentd.MetadataCacheSize | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_METADATA_WATCH | read-only "false" | *eksconfig.AddOnFluentd.MetadataWatch | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_METADATA_SKIP_LABELS | read-only "false" | *eksconfig.AddOnFluentd.MetadataSkipLabels | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_METADATA_SKIP_MASTER_URL | read-only "false" | *eksconfig.AddOnFluentd.MetadataSkipMasterURL | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_METADATA_SKIP_CONTAINER_METADATA | read-only "false" | *eksconfig.AddOnFluentd.MetadataSkipContainerMetadata | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_METADATA_SKIP_NAMESPACE_METADATA | read-only "false" | *eksconfig.AddOnFluentd.MetadataSkipNamespaceMetadata | bool |
-*--------------------------------------------------------------------*-------------------*-------------------------------------------------------*--------------------*
-
-
-*------------------------------------------------------------*-------------------*-----------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*------------------------------------------------------------*-------------------*-----------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_METRICS_SERVER_ENABLE | read-only "false" | *eksconfig.AddOnMetricsServer.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_METRICS_SERVER_CREATED | read-only "true" | *eksconfig.AddOnMetricsServer.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_METRICS_SERVER_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnMetricsServer.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_METRICS_SERVER_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnMetricsServer.TimeFrameDelete | timeutil.TimeFrame |
-*------------------------------------------------------------*-------------------*-----------------------------------------------*--------------------*
-
-
-*---------------------------------------------------------------------------*-------------------*-------------------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*---------------------------------------------------------------------------*-------------------*-------------------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_ENABLE | read-only "false" | *eksconfig.AddOnConformance.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_CREATED | read-only "true" | *eksconfig.AddOnConformance.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnConformance.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnConformance.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_S3_DIR | read-only "false" | *eksconfig.AddOnConformance.S3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_NAMESPACE | read-only "false" | *eksconfig.AddOnConformance.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_PATH | read-only "false" | *eksconfig.AddOnConformance.SonobuoyPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_DOWNLOAD_URL | read-only "false" | *eksconfig.AddOnConformance.SonobuoyDownloadURL | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_E2E_REPO_CONFIG | read-only "false" | *eksconfig.AddOnConformance.SonobuoyE2eRepoConfig | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_IMAGE | read-only "false" | *eksconfig.AddOnConformance.SonobuoyImage | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SYSTEMD_LOGS_IMAGE | read-only "false" | *eksconfig.AddOnConformance.SystemdLogsImage | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_DELETE_TIMEOUT | read-only "false" | *eksconfig.AddOnConformance.SonobuoyDeleteTimeout | time.Duration |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_DELETE_TIMEOUT_STRING | read-only "true" | *eksconfig.AddOnConformance.SonobuoyDeleteTimeoutString | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_TIMEOUT | read-only "false" | *eksconfig.AddOnConformance.SonobuoyRunTimeout | time.Duration |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_TIMEOUT_STRING | read-only "true" | *eksconfig.AddOnConformance.SonobuoyRunTimeoutString | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_MODE | read-only "false" | *eksconfig.AddOnConformance.SonobuoyRunMode | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_KUBE_CONFORMANCE_IMAGE | read-only "false" | *eksconfig.AddOnConformance.SonobuoyRunKubeConformanceImage | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_E2E_FOCUS | read-only "false" | *eksconfig.AddOnConformance.SonobuoyRunE2eFocus | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_E2E_SKIP | read-only "false" | *eksconfig.AddOnConformance.SonobuoyRunE2eSkip | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RESULT_TAR_GZ_PATH | read-only "true" | *eksconfig.AddOnConformance.SonobuoyResultTarGzPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RESULT_TAR_GZ_S3_KEY | read-only "true" | *eksconfig.AddOnConformance.SonobuoyResultTarGzS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RESULT_DIR | read-only "true" | *eksconfig.AddOnConformance.SonobuoyResultDir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RESULT_E2E_LOG_PATH | read-only "true" | *eksconfig.AddOnConformance.SonobuoyResultE2eLogPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RESULT_E2E_LOG_S3_KEY | read-only "true" | *eksconfig.AddOnConformance.SonobuoyResultE2eLogS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RESULT_JUNIT_XML_PATH | read-only "true" | *eksconfig.AddOnConformance.SonobuoyResultJunitXMLPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RESULT_JUNIT_XML_S3_KEY | read-only "true" | *eksconfig.AddOnConformance.SonobuoyResultJunitXMLS3Key | string |
-*---------------------------------------------------------------------------*-------------------*-------------------------------------------------------------*--------------------*
-
-
-*-----------------------------------------------------------------*-------------------*-------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-----------------------------------------------------------------*-------------------*-------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_ENABLE | read-only "false" | *eksconfig.AddOnAppMesh.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_CREATED | read-only "true" | *eksconfig.AddOnAppMesh.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnAppMesh.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnAppMesh.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_S3_DIR | read-only "false" | *eksconfig.AddOnAppMesh.S3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_NAMESPACE | read-only "false" | *eksconfig.AddOnAppMesh.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_CONTROLLER_IMAGE | read-only "false" | *eksconfig.AddOnAppMesh.ControllerImage | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_INJECTOR_IMAGE | read-only "false" | *eksconfig.AddOnAppMesh.InjectorImage | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_POLICY_CFN_STACK_ID | read-only "true" | *eksconfig.AddOnAppMesh.PolicyCFNStackID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_POLICY_CFN_STACK_YAML_PATH | read-only "true" | *eksconfig.AddOnAppMesh.PolicyCFNStackYAMLPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_POLICY_CFN_STACK_YAML_S3_KEY | read-only "true" | *eksconfig.AddOnAppMesh.PolicyCFNStackYAMLS3Key | string |
-*-----------------------------------------------------------------*-------------------*-------------------------------------------------*--------------------*
-
-
-*-----------------------------------------------------*-------------------*----------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-----------------------------------------------------*-------------------*----------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_CSI_EBS_ENABLE | read-only "false" | *eksconfig.AddOnCSIEBS.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSI_EBS_CREATED | read-only "true" | *eksconfig.AddOnCSIEBS.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSI_EBS_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnCSIEBS.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSI_EBS_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnCSIEBS.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSI_EBS_CHART_REPO_URL | read-only "false" | *eksconfig.AddOnCSIEBS.ChartRepoURL | string |
-*-----------------------------------------------------*-------------------*----------------------------------------*--------------------*
-
-
-*---------------------------------------------------------------------*-------------------*---------------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*---------------------------------------------------------------------*-------------------*---------------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBERNETES_DASHBOARD_ENABLE | read-only "false" | *eksconfig.AddOnKubernetesDashboard.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBERNETES_DASHBOARD_CREATED | read-only "true" | *eksconfig.AddOnKubernetesDashboard.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBERNETES_DASHBOARD_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnKubernetesDashboard.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBERNETES_DASHBOARD_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnKubernetesDashboard.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBERNETES_DASHBOARD_AUTHENTICATION_TOKEN | read-only "true" | *eksconfig.AddOnKubernetesDashboard.AuthenticationToken | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBERNETES_DASHBOARD_URL | read-only "true" | *eksconfig.AddOnKubernetesDashboard.URL | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBERNETES_DASHBOARD_KUBECTL_PROXY_PID | read-only "true" | *eksconfig.AddOnKubernetesDashboard.KubectlProxyPID | int |
-*---------------------------------------------------------------------*-------------------*---------------------------------------------------------*--------------------*
-
-
-*----------------------------------------------------------------------*-------------------*--------------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*----------------------------------------------------------------------*-------------------*--------------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_ENABLE | read-only "false" | *eksconfig.AddOnPrometheusGrafana.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_CREATED | read-only "true" | *eksconfig.AddOnPrometheusGrafana.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnPrometheusGrafana.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnPrometheusGrafana.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_GRAFANA_ADMIN_USER_NAME | read-only "false" | *eksconfig.AddOnPrometheusGrafana.GrafanaAdminUserName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_GRAFANA_ADMIN_PASSWORD | read-only "false" | *eksconfig.AddOnPrometheusGrafana.GrafanaAdminPassword | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_GRAFANA_NLB_ARN | read-only "true" | *eksconfig.AddOnPrometheusGrafana.GrafanaNLBARN | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_GRAFANA_NLB_NAME | read-only "true" | *eksconfig.AddOnPrometheusGrafana.GrafanaNLBName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_GRAFANA_URL | read-only "true" | *eksconfig.AddOnPrometheusGrafana.GrafanaURL | string |
-*----------------------------------------------------------------------*-------------------*--------------------------------------------------------*--------------------*
-
-
-*---------------------------------------------------------------*-------------------*--------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*---------------------------------------------------------------*-------------------*--------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_ENABLE | read-only "false" | *eksconfig.AddOnPHPApache.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_CREATED | read-only "true" | *eksconfig.AddOnPHPApache.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnPHPApache.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnPHPApache.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_NAMESPACE | read-only "false" | *eksconfig.AddOnPHPApache.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_REPOSITORY_ACCOUNT_ID | read-only "false" | *eksconfig.AddOnPHPApache.RepositoryAccountID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_REPOSITORY_REGION | read-only "false" | *eksconfig.AddOnPHPApache.RepositoryRegion | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_REPOSITORY_NAME | read-only "false" | *eksconfig.AddOnPHPApache.RepositoryName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_REPOSITORY_IMAGE_TAG | read-only "false" | *eksconfig.AddOnPHPApache.RepositoryImageTag | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_DEPLOYMENT_REPLICAS | read-only "false" | *eksconfig.AddOnPHPApache.DeploymentReplicas | int32 |
-| AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_DEPLOYMENT_NODE_SELECTOR | read-only "false" | *eksconfig.AddOnPHPApache.DeploymentNodeSelector | map[string]string |
-*---------------------------------------------------------------*-------------------*--------------------------------------------------*--------------------*
-
-
-*--------------------------------------------------------------------*-------------------*------------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*--------------------------------------------------------------------*-------------------*------------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE | read-only "false" | *eksconfig.AddOnNLBHelloWorld.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_CREATED | read-only "true" | *eksconfig.AddOnNLBHelloWorld.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnNLBHelloWorld.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnNLBHelloWorld.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_NAMESPACE | read-only "false" | *eksconfig.AddOnNLBHelloWorld.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_DEPLOYMENT_REPLICAS | read-only "false" | *eksconfig.AddOnNLBHelloWorld.DeploymentReplicas | int32 |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_DEPLOYMENT_NODE_SELECTOR | read-only "false" | *eksconfig.AddOnNLBHelloWorld.DeploymentNodeSelector | map[string]string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_NLB_ARN | read-only "true" | *eksconfig.AddOnNLBHelloWorld.NLBARN | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_NLB_NAME | read-only "true" | *eksconfig.AddOnNLBHelloWorld.NLBName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_URL | read-only "true" | *eksconfig.AddOnNLBHelloWorld.URL | string |
-*--------------------------------------------------------------------*-------------------*------------------------------------------------------*--------------------*
-
-
-*------------------------------------------------------------------*-------------------*-----------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*------------------------------------------------------------------*-------------------*-----------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_ENABLE | read-only "false" | *eksconfig.AddOnNLBGuestbook.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_CREATED | read-only "true" | *eksconfig.AddOnNLBGuestbook.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnNLBGuestbook.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnNLBGuestbook.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_NAMESPACE | read-only "false" | *eksconfig.AddOnNLBGuestbook.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_DEPLOYMENT_REPLICAS | read-only "false" | *eksconfig.AddOnNLBGuestbook.DeploymentReplicas | int32 |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_DEPLOYMENT_NODE_SELECTOR | read-only "false" | *eksconfig.AddOnNLBGuestbook.DeploymentNodeSelector | map[string]string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_NLB_ARN | read-only "true" | *eksconfig.AddOnNLBGuestbook.NLBARN | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_NLB_NAME | read-only "true" | *eksconfig.AddOnNLBGuestbook.NLBName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_URL | read-only "true" | *eksconfig.AddOnNLBGuestbook.URL | string |
-*------------------------------------------------------------------*-------------------*-----------------------------------------------------*--------------------*
-
-
-*------------------------------------------------------------------*-------------------*----------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*------------------------------------------------------------------*-------------------*----------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_ENABLE | read-only "false" | *eksconfig.AddOnALB2048.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_CREATED | read-only "true" | *eksconfig.AddOnALB2048.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnALB2048.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnALB2048.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_NAMESPACE | read-only "false" | *eksconfig.AddOnALB2048.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_DEPLOYMENT_REPLICAS_ALB | read-only "false" | *eksconfig.AddOnALB2048.DeploymentReplicasALB | int32 |
-| AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_DEPLOYMENT_REPLICAS_2048 | read-only "false" | *eksconfig.AddOnALB2048.DeploymentReplicas2048 | int32 |
-| AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_DEPLOYMENT_NODE_SELECTOR_2048 | read-only "false" | *eksconfig.AddOnALB2048.DeploymentNodeSelector2048 | map[string]string |
-| AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_ALB_ARN | read-only "true" | *eksconfig.AddOnALB2048.ALBARN | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_ALB_NAME | read-only "true" | *eksconfig.AddOnALB2048.ALBName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_URL | read-only "true" | *eksconfig.AddOnALB2048.URL | string |
-*------------------------------------------------------------------*-------------------*----------------------------------------------------*--------------------*
-
-
-*-----------------------------------------------------*-------------------*----------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-----------------------------------------------------*-------------------*----------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_ENABLE | read-only "false" | *eksconfig.AddOnJobsPi.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_CREATED | read-only "true" | *eksconfig.AddOnJobsPi.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnJobsPi.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnJobsPi.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_NAMESPACE | read-only "false" | *eksconfig.AddOnJobsPi.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_COMPLETES | read-only "false" | *eksconfig.AddOnJobsPi.Completes | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_PARALLELS | read-only "false" | *eksconfig.AddOnJobsPi.Parallels | int |
-*-----------------------------------------------------*-------------------*----------------------------------------*--------------------*
-
-
-*-------------------------------------------------------------------*-------------------*-----------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-------------------------------------------------------------------*-------------------*-----------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_ENABLE | read-only "false" | *eksconfig.AddOnJobsEcho.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_CREATED | read-only "true" | *eksconfig.AddOnJobsEcho.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnJobsEcho.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnJobsEcho.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_NAMESPACE | read-only "false" | *eksconfig.AddOnJobsEcho.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_REPOSITORY_BUSYBOX_ACCOUNT_ID | read-only "false" | *eksconfig.AddOnJobsEcho.RepositoryBusyboxAccountID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_REPOSITORY_BUSYBOX_REGION | read-only "false" | *eksconfig.AddOnJobsEcho.RepositoryBusyboxRegion | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_REPOSITORY_BUSYBOX_NAME | read-only "false" | *eksconfig.AddOnJobsEcho.RepositoryBusyboxName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_REPOSITORY_BUSYBOX_IMAGE_TAG | read-only "false" | *eksconfig.AddOnJobsEcho.RepositoryBusyboxImageTag | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_COMPLETES | read-only "false" | *eksconfig.AddOnJobsEcho.Completes | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_PARALLELS | read-only "false" | *eksconfig.AddOnJobsEcho.Parallels | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_ECHO_SIZE | read-only "false" | *eksconfig.AddOnJobsEcho.EchoSize | int |
-*-------------------------------------------------------------------*-------------------*-----------------------------------------------------*--------------------*
-
-
-*-------------------------------------------------------------------*-------------------*-----------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-------------------------------------------------------------------*-------------------*-----------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_ENABLE | read-only "false" | *eksconfig.AddOnCronJobs.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_CREATED | read-only "true" | *eksconfig.AddOnCronJobs.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnCronJobs.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnCronJobs.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_NAMESPACE | read-only "false" | *eksconfig.AddOnCronJobs.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_REPOSITORY_BUSYBOX_ACCOUNT_ID | read-only "false" | *eksconfig.AddOnCronJobs.RepositoryBusyboxAccountID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_REPOSITORY_BUSYBOX_REGION | read-only "false" | *eksconfig.AddOnCronJobs.RepositoryBusyboxRegion | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_REPOSITORY_BUSYBOX_NAME | read-only "false" | *eksconfig.AddOnCronJobs.RepositoryBusyboxName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_REPOSITORY_BUSYBOX_IMAGE_TAG | read-only "false" | *eksconfig.AddOnCronJobs.RepositoryBusyboxImageTag | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_SCHEDULE | read-only "false" | *eksconfig.AddOnCronJobs.Schedule | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_COMPLETES | read-only "false" | *eksconfig.AddOnCronJobs.Completes | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_PARALLELS | read-only "false" | *eksconfig.AddOnCronJobs.Parallels | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_SUCCESSFUL_JOBS_HISTORY_LIMIT | read-only "false" | *eksconfig.AddOnCronJobs.SuccessfulJobsHistoryLimit | int32 |
-| AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_FAILED_JOBS_HISTORY_LIMIT | read-only "false" | *eksconfig.AddOnCronJobs.FailedJobsHistoryLimit | int32 |
-| AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_ECHO_SIZE | read-only "false" | *eksconfig.AddOnCronJobs.EchoSize | int |
-*-------------------------------------------------------------------*-------------------*-----------------------------------------------------*--------------------*
-
-
-*------------------------------------------------------------------------------------*-------------------*------------------------------------------------------------------*-------------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*------------------------------------------------------------------------------------*-------------------*------------------------------------------------------------------*-------------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_ENABLE | read-only "false" | *eksconfig.AddOnCSRsLocal.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_CREATED | read-only "true" | *eksconfig.AddOnCSRsLocal.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnCSRsLocal.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnCSRsLocal.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_S3_DIR | read-only "false" | *eksconfig.AddOnCSRsLocal.S3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_OBJECTS | read-only "false" | *eksconfig.AddOnCSRsLocal.Objects | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_INITIAL_REQUEST_CONDITION_TYPE | read-only "false" | *eksconfig.AddOnCSRsLocal.InitialRequestConditionType | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_CREATED_NAMES | read-only "true" | *eksconfig.AddOnCSRsLocal.CreatedNames | []string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_RAW_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsRawWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_RAW_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsRawWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnCSRsLocal.RequestsRawWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_PATH | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsRawWritesCompareAllJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsRawWritesCompareAllJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_PATH | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsRawWritesCompareAllCSVPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_S3_KEY | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsRawWritesCompareAllCSVS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_SUMMARY_WRITES | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsSummaryWrites | metrics.RequestsSummary |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_SUMMARY_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsSummaryWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_SUMMARY_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsSummaryWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_SUMMARY_WRITES_TABLE_PATH | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsSummaryWritesTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_SUMMARY_WRITES_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsSummaryWritesTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnCSRsLocal.RequestsSummaryWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsSummaryWritesCompare | metrics.RequestsCompare |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_PATH | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsSummaryWritesCompareJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsSummaryWritesCompareJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_PATH | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsSummaryWritesCompareTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnCSRsLocal.RequestsSummaryWritesCompareTableS3Key | string |
-*------------------------------------------------------------------------------------*-------------------*------------------------------------------------------------------*-------------------------*
-
-
-*-------------------------------------------------------------------------------------*-------------------*-------------------------------------------------------------------*-------------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-------------------------------------------------------------------------------------*-------------------*-------------------------------------------------------------------*-------------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_ENABLE | read-only "false" | *eksconfig.AddOnCSRsRemote.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_CREATED | read-only "true" | *eksconfig.AddOnCSRsRemote.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnCSRsRemote.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnCSRsRemote.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_S3_DIR | read-only "false" | *eksconfig.AddOnCSRsRemote.S3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_NAMESPACE | read-only "false" | *eksconfig.AddOnCSRsRemote.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REPOSITORY_ACCOUNT_ID | read-only "false" | *eksconfig.AddOnCSRsRemote.RepositoryAccountID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REPOSITORY_REGION | read-only "false" | *eksconfig.AddOnCSRsRemote.RepositoryRegion | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REPOSITORY_NAME | read-only "false" | *eksconfig.AddOnCSRsRemote.RepositoryName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REPOSITORY_IMAGE_TAG | read-only "false" | *eksconfig.AddOnCSRsRemote.RepositoryImageTag | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_COMPLETES | read-only "false" | *eksconfig.AddOnCSRsRemote.Completes | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_PARALLELS | read-only "false" | *eksconfig.AddOnCSRsRemote.Parallels | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_OBJECTS | read-only "false" | *eksconfig.AddOnCSRsRemote.Objects | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_INITIAL_REQUEST_CONDITION_TYPE | read-only "false" | *eksconfig.AddOnCSRsRemote.InitialRequestConditionType | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_SUMMARY_WRITES_OUTPUT_NAME_PREFIX | read-only "false" | *eksconfig.AddOnCSRsRemote.RequestsSummaryWritesOutputNamePrefix | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_RAW_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsRawWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_RAW_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsRawWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_RAW_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnCSRsRemote.RequestsRawWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_PATH | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsRawWritesCompareAllJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsRawWritesCompareAllJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_PATH | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsRawWritesCompareAllCSVPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_S3_KEY | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsRawWritesCompareAllCSVS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_SUMMARY_WRITES | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsSummaryWrites | metrics.RequestsSummary |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_SUMMARY_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsSummaryWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_SUMMARY_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsSummaryWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_SUMMARY_WRITES_TABLE_PATH | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsSummaryWritesTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_SUMMARY_WRITES_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsSummaryWritesTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnCSRsRemote.RequestsSummaryWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsSummaryWritesCompare | metrics.RequestsCompare |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_PATH | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsSummaryWritesCompareJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsSummaryWritesCompareJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_PATH | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsSummaryWritesCompareTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnCSRsRemote.RequestsSummaryWritesCompareTableS3Key | string |
-*-------------------------------------------------------------------------------------*-------------------*-------------------------------------------------------------------*-------------------------*
-
-
-*------------------------------------------------------------------------------------------*-------------------*------------------------------------------------------------------------*-------------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*------------------------------------------------------------------------------------------*-------------------*------------------------------------------------------------------------*-------------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_ENABLE | read-only "false" | *eksconfig.AddOnConfigmapsLocal.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_CREATED | read-only "true" | *eksconfig.AddOnConfigmapsLocal.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnConfigmapsLocal.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnConfigmapsLocal.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_S3_DIR | read-only "false" | *eksconfig.AddOnConfigmapsLocal.S3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_NAMESPACE | read-only "false" | *eksconfig.AddOnConfigmapsLocal.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_OBJECTS | read-only "false" | *eksconfig.AddOnConfigmapsLocal.Objects | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_OBJECT_SIZE | read-only "false" | *eksconfig.AddOnConfigmapsLocal.ObjectSize | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_CREATED_NAMES | read-only "true" | *eksconfig.AddOnConfigmapsLocal.CreatedNames | []string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_RAW_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsRawWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_RAW_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsRawWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnConfigmapsLocal.RequestsRawWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_PATH | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsRawWritesCompareAllJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsRawWritesCompareAllJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_PATH | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsRawWritesCompareAllCSVPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_S3_KEY | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsRawWritesCompareAllCSVS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_SUMMARY_WRITES | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsSummaryWrites | metrics.RequestsSummary |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_SUMMARY_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsSummaryWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_SUMMARY_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsSummaryWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_SUMMARY_WRITES_TABLE_PATH | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsSummaryWritesTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_SUMMARY_WRITES_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsSummaryWritesTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnConfigmapsLocal.RequestsSummaryWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsSummaryWritesCompare | metrics.RequestsCompare |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_PATH | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsSummaryWritesCompareJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsSummaryWritesCompareJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_PATH | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsSummaryWritesCompareTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnConfigmapsLocal.RequestsSummaryWritesCompareTableS3Key | string |
-*------------------------------------------------------------------------------------------*-------------------*------------------------------------------------------------------------*-------------------------*
-
-
-*-------------------------------------------------------------------------------------------*-------------------*-------------------------------------------------------------------------*-------------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-------------------------------------------------------------------------------------------*-------------------*-------------------------------------------------------------------------*-------------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_ENABLE | read-only "false" | *eksconfig.AddOnConfigmapsRemote.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_CREATED | read-only "true" | *eksconfig.AddOnConfigmapsRemote.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnConfigmapsRemote.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnConfigmapsRemote.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_S3_DIR | read-only "false" | *eksconfig.AddOnConfigmapsRemote.S3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_NAMESPACE | read-only "false" | *eksconfig.AddOnConfigmapsRemote.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REPOSITORY_ACCOUNT_ID | read-only "false" | *eksconfig.AddOnConfigmapsRemote.RepositoryAccountID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REPOSITORY_REGION | read-only "false" | *eksconfig.AddOnConfigmapsRemote.RepositoryRegion | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REPOSITORY_NAME | read-only "false" | *eksconfig.AddOnConfigmapsRemote.RepositoryName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REPOSITORY_IMAGE_TAG | read-only "false" | *eksconfig.AddOnConfigmapsRemote.RepositoryImageTag | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_COMPLETES | read-only "false" | *eksconfig.AddOnConfigmapsRemote.Completes | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_PARALLELS | read-only "false" | *eksconfig.AddOnConfigmapsRemote.Parallels | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_OBJECTS | read-only "false" | *eksconfig.AddOnConfigmapsRemote.Objects | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_OBJECT_SIZE | read-only "false" | *eksconfig.AddOnConfigmapsRemote.ObjectSize | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_CREATED_NAMES | read-only "true" | *eksconfig.AddOnConfigmapsRemote.CreatedNames | []string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_SUMMARY_WRITES_OUTPUT_NAME_PREFIX | read-only "false" | *eksconfig.AddOnConfigmapsRemote.RequestsSummaryWritesOutputNamePrefix | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_RAW_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsRawWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_RAW_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsRawWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_RAW_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnConfigmapsRemote.RequestsRawWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_PATH | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsRawWritesCompareAllJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsRawWritesCompareAllJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_PATH | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsRawWritesCompareAllCSVPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_S3_KEY | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsRawWritesCompareAllCSVS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_SUMMARY_WRITES | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsSummaryWrites | metrics.RequestsSummary |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_SUMMARY_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsSummaryWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_SUMMARY_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsSummaryWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_SUMMARY_WRITES_TABLE_PATH | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsSummaryWritesTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_SUMMARY_WRITES_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsSummaryWritesTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompare | metrics.RequestsCompare |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_PATH | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompareJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompareJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_PATH | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompareTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompareTableS3Key | string |
-*-------------------------------------------------------------------------------------------*-------------------*-------------------------------------------------------------------------*-------------------------*
-
-
-*---------------------------------------------------------------------------------------*-------------------*---------------------------------------------------------------------*-------------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*---------------------------------------------------------------------------------------*-------------------*---------------------------------------------------------------------*-------------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_ENABLE | read-only "false" | *eksconfig.AddOnSecretsLocal.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_CREATED | read-only "true" | *eksconfig.AddOnSecretsLocal.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnSecretsLocal.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnSecretsLocal.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_S3_DIR | read-only "false" | *eksconfig.AddOnSecretsLocal.S3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_NAMESPACE | read-only "false" | *eksconfig.AddOnSecretsLocal.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_OBJECTS | read-only "false" | *eksconfig.AddOnSecretsLocal.Objects | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_OBJECT_SIZE | read-only "false" | *eksconfig.AddOnSecretsLocal.ObjectSize | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_NAME_PREFIX | read-only "false" | *eksconfig.AddOnSecretsLocal.NamePrefix | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_RAW_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsRawWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_RAW_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsRawWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnSecretsLocal.RequestsRawWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsRawWritesCompareAllJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsRawWritesCompareAllJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsRawWritesCompareAllCSVPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsRawWritesCompareAllCSVS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_WRITES | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryWrites | metrics.RequestsSummary |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_WRITES_TABLE_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryWritesTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_WRITES_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryWritesTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnSecretsLocal.RequestsSummaryWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryWritesCompare | metrics.RequestsCompare |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryWritesCompareJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryWritesCompareJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryWritesCompareTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryWritesCompareTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_RAW_READS_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsRawReadsJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_RAW_READS_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsRawReadsJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_RAW_READS_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnSecretsLocal.RequestsRawReadsCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_RAW_READS_COMPARE_ALL_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsRawReadsCompareAllJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_RAW_READS_COMPARE_ALL_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsRawReadsCompareAllJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_RAW_READS_COMPARE_ALL_CSV_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsRawReadsCompareAllCSVPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_RAW_READS_COMPARE_ALL_CSV_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsRawReadsCompareAllCSVS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_READS | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryReads | metrics.RequestsSummary |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_READS_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryReadsJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_READS_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryReadsJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_READS_TABLE_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryReadsTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_READS_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryReadsTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_READS_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnSecretsLocal.RequestsSummaryReadsCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_READS_COMPARE | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryReadsCompare | metrics.RequestsCompare |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_READS_COMPARE_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryReadsCompareJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_READS_COMPARE_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryReadsCompareJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_READS_COMPARE_TABLE_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryReadsCompareTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_REQUESTS_SUMMARY_READS_COMPARE_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnSecretsLocal.RequestsSummaryReadsCompareTableS3Key | string |
-*---------------------------------------------------------------------------------------*-------------------*---------------------------------------------------------------------*-------------------------*
-
-
-*----------------------------------------------------------------------------------------*-------------------*----------------------------------------------------------------------*-------------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*----------------------------------------------------------------------------------------*-------------------*----------------------------------------------------------------------*-------------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_ENABLE | read-only "false" | *eksconfig.AddOnSecretsRemote.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_CREATED | read-only "true" | *eksconfig.AddOnSecretsRemote.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnSecretsRemote.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnSecretsRemote.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_S3_DIR | read-only "false" | *eksconfig.AddOnSecretsRemote.S3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_NAMESPACE | read-only "false" | *eksconfig.AddOnSecretsRemote.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REPOSITORY_ACCOUNT_ID | read-only "false" | *eksconfig.AddOnSecretsRemote.RepositoryAccountID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REPOSITORY_REGION | read-only "false" | *eksconfig.AddOnSecretsRemote.RepositoryRegion | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REPOSITORY_NAME | read-only "false" | *eksconfig.AddOnSecretsRemote.RepositoryName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REPOSITORY_IMAGE_TAG | read-only "false" | *eksconfig.AddOnSecretsRemote.RepositoryImageTag | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_COMPLETES | read-only "false" | *eksconfig.AddOnSecretsRemote.Completes | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_PARALLELS | read-only "false" | *eksconfig.AddOnSecretsRemote.Parallels | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_OBJECTS | read-only "false" | *eksconfig.AddOnSecretsRemote.Objects | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_OBJECT_SIZE | read-only "false" | *eksconfig.AddOnSecretsRemote.ObjectSize | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_NAME_PREFIX | read-only "false" | *eksconfig.AddOnSecretsRemote.NamePrefix | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_WRITES_OUTPUT_NAME_PREFIX | read-only "false" | *eksconfig.AddOnSecretsRemote.RequestsSummaryWritesOutputNamePrefix | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_READS_OUTPUT_NAME_PREFIX | read-only "false" | *eksconfig.AddOnSecretsRemote.RequestsSummaryReadsOutputNamePrefix | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_RAW_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsRawWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_RAW_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsRawWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_RAW_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnSecretsRemote.RequestsRawWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsRawWritesCompareAllJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsRawWritesCompareAllJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsRawWritesCompareAllCSVPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsRawWritesCompareAllCSVS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_WRITES | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryWrites | metrics.RequestsSummary |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_WRITES_TABLE_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryWritesTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_WRITES_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryWritesTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnSecretsRemote.RequestsSummaryWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryWritesCompare | metrics.RequestsCompare |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryWritesCompareJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryWritesCompareJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryWritesCompareTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryWritesCompareTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_RAW_READS_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsRawReadsJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_RAW_READS_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsRawReadsJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_RAW_READS_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnSecretsRemote.RequestsRawReadsCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_RAW_READS_COMPARE_ALL_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsRawReadsCompareAllJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_RAW_READS_COMPARE_ALL_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsRawReadsCompareAllJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_RAW_READS_COMPARE_ALL_CSV_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsRawReadsCompareAllCSVPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_RAW_READS_COMPARE_ALL_CSV_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsRawReadsCompareAllCSVS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_READS | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryReads | metrics.RequestsSummary |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_READS_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryReadsJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_READS_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryReadsJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_READS_TABLE_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryReadsTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_READS_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryReadsTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_READS_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnSecretsRemote.RequestsSummaryReadsCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_READS_COMPARE | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryReadsCompare | metrics.RequestsCompare |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_READS_COMPARE_JSON_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryReadsCompareJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_READS_COMPARE_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryReadsCompareJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_READS_COMPARE_TABLE_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryReadsCompareTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REQUESTS_SUMMARY_READS_COMPARE_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnSecretsRemote.RequestsSummaryReadsCompareTableS3Key | string |
-*----------------------------------------------------------------------------------------*-------------------*----------------------------------------------------------------------*-------------------------*
-
-
-*--------------------------------------------------------------*-------------------*-----------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*--------------------------------------------------------------*-------------------*-----------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ENABLE | read-only "false" | *eksconfig.AddOnFargate.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_CREATED | read-only "true" | *eksconfig.AddOnFargate.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnFargate.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnFargate.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_S3_DIR | read-only "false" | *eksconfig.AddOnFargate.S3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_NAMESPACE | read-only "false" | *eksconfig.AddOnFargate.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_REPOSITORY_ACCOUNT_ID | read-only "false" | *eksconfig.AddOnFargate.RepositoryAccountID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_REPOSITORY_REGION | read-only "false" | *eksconfig.AddOnFargate.RepositoryRegion | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_REPOSITORY_NAME | read-only "false" | *eksconfig.AddOnFargate.RepositoryName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_REPOSITORY_IMAGE_TAG | read-only "false" | *eksconfig.AddOnFargate.RepositoryImageTag | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ROLE_NAME | read-only "false" | *eksconfig.AddOnFargate.RoleName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ROLE_CREATE | read-only "false" | *eksconfig.AddOnFargate.RoleCreate | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ROLE_ARN | read-only "false" | *eksconfig.AddOnFargate.RoleARN | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ROLE_SERVICE_PRINCIPALS | read-only "false" | *eksconfig.AddOnFargate.RoleServicePrincipals | []string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ROLE_MANAGED_POLICY_ARNS | read-only "false" | *eksconfig.AddOnFargate.RoleManagedPolicyARNs | []string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ROLE_CFN_STACK_ID | read-only "true" | *eksconfig.AddOnFargate.RoleCFNStackID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ROLE_CFN_STACK_YAML_PATH | read-only "true" | *eksconfig.AddOnFargate.RoleCFNStackYAMLPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ROLE_CFN_STACK_YAML_S3_KEY | read-only "true" | *eksconfig.AddOnFargate.RoleCFNStackYAMLS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_PROFILE_NAME | read-only "false" | *eksconfig.AddOnFargate.ProfileName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_SECRET_NAME | read-only "false" | *eksconfig.AddOnFargate.SecretName | string |
-*--------------------------------------------------------------*-------------------*-----------------------------------------------*--------------------*
-
-
-*-----------------------------------------------------------*-------------------*--------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-----------------------------------------------------------*-------------------*--------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_ENABLE | read-only "false" | *eksconfig.AddOnIRSA.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_CREATED | read-only "true" | *eksconfig.AddOnIRSA.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnIRSA.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnIRSA.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_S3_DIR | read-only "false" | *eksconfig.AddOnIRSA.S3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_NAMESPACE | read-only "false" | *eksconfig.AddOnIRSA.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_REPOSITORY_ACCOUNT_ID | read-only "false" | *eksconfig.AddOnIRSA.RepositoryAccountID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_REPOSITORY_REGION | read-only "false" | *eksconfig.AddOnIRSA.RepositoryRegion | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_REPOSITORY_NAME | read-only "false" | *eksconfig.AddOnIRSA.RepositoryName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_REPOSITORY_IMAGE_TAG | read-only "false" | *eksconfig.AddOnIRSA.RepositoryImageTag | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_ROLE_NAME | read-only "false" | *eksconfig.AddOnIRSA.RoleName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_ROLE_ARN | read-only "false" | *eksconfig.AddOnIRSA.RoleARN | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_ROLE_CFN_STACK_ID | read-only "true" | *eksconfig.AddOnIRSA.RoleCFNStackID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_ROLE_CFN_STACK_YAML_PATH | read-only "true" | *eksconfig.AddOnIRSA.RoleCFNStackYAMLPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_ROLE_CFN_STACK_YAML_S3_KEY | read-only "true" | *eksconfig.AddOnIRSA.RoleCFNStackYAMLS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_S3_KEY | read-only "false" | *eksconfig.AddOnIRSA.S3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_DEPLOYMENT_REPLICAS | read-only "false" | *eksconfig.AddOnIRSA.DeploymentReplicas | int32 |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_DEPLOYMENT_RESULT_PATH | read-only "false" | *eksconfig.AddOnIRSA.DeploymentResultPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_DEPLOYMENT_TOOK | read-only "true" | *eksconfig.AddOnIRSA.DeploymentTook | time.Duration |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_DEPLOYMENT_TOOK_STRING | read-only "true" | *eksconfig.AddOnIRSA.DeploymentTookString | string |
-*-----------------------------------------------------------*-------------------*--------------------------------------------*--------------------*
-
-
-*-------------------------------------------------------------------*-------------------*---------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-------------------------------------------------------------------*-------------------*---------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ENABLE | read-only "false" | *eksconfig.AddOnIRSAFargate.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_CREATED | read-only "true" | *eksconfig.AddOnIRSAFargate.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnIRSAFargate.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnIRSAFargate.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_S3_DIR | read-only "false" | *eksconfig.AddOnIRSAFargate.S3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_NAMESPACE | read-only "false" | *eksconfig.AddOnIRSAFargate.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_REPOSITORY_ACCOUNT_ID | read-only "false" | *eksconfig.AddOnIRSAFargate.RepositoryAccountID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_REPOSITORY_REGION | read-only "false" | *eksconfig.AddOnIRSAFargate.RepositoryRegion | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_REPOSITORY_NAME | read-only "false" | *eksconfig.AddOnIRSAFargate.RepositoryName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_REPOSITORY_IMAGE_TAG | read-only "false" | *eksconfig.AddOnIRSAFargate.RepositoryImageTag | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ROLE_NAME | read-only "false" | *eksconfig.AddOnIRSAFargate.RoleName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ROLE_ARN | read-only "false" | *eksconfig.AddOnIRSAFargate.RoleARN | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ROLE_SERVICE_PRINCIPALS | read-only "false" | *eksconfig.AddOnIRSAFargate.RoleServicePrincipals | []string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ROLE_MANAGED_POLICY_ARNS | read-only "false" | *eksconfig.AddOnIRSAFargate.RoleManagedPolicyARNs | []string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ROLE_CFN_STACK_ID | read-only "true" | *eksconfig.AddOnIRSAFargate.RoleCFNStackID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ROLE_CFN_STACK_YAML_PATH | read-only "true" | *eksconfig.AddOnIRSAFargate.RoleCFNStackYAMLPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ROLE_CFN_STACK_YAML_S3_KEY | read-only "true" | *eksconfig.AddOnIRSAFargate.RoleCFNStackYAMLS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_S3_KEY | read-only "false" | *eksconfig.AddOnIRSAFargate.S3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_PROFILE_NAME | read-only "false" | *eksconfig.AddOnIRSAFargate.ProfileName | string |
-*-------------------------------------------------------------------*-------------------*---------------------------------------------------*--------------------*
-
-
-*-------------------------------------------------------*-------------------*-------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-------------------------------------------------------*-------------------*-------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_ENABLE | read-only "false" | *eksconfig.AddOnWordpress.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_CREATED | read-only "true" | *eksconfig.AddOnWordpress.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnWordpress.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnWordpress.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_NAMESPACE | read-only "false" | *eksconfig.AddOnWordpress.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_USER_NAME | read-only "false" | *eksconfig.AddOnWordpress.UserName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_PASSWORD | read-only "false" | *eksconfig.AddOnWordpress.Password | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_NLB_ARN | read-only "true" | *eksconfig.AddOnWordpress.NLBARN | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_NLB_NAME | read-only "true" | *eksconfig.AddOnWordpress.NLBName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_URL | read-only "true" | *eksconfig.AddOnWordpress.URL | string |
-*-------------------------------------------------------*-------------------*-------------------------------------------*--------------------*
-
-
-*----------------------------------------------------------*-------------------*---------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*----------------------------------------------------------*-------------------*---------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_ENABLE | read-only "false" | *eksconfig.AddOnJupyterHub.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_CREATED | read-only "true" | *eksconfig.AddOnJupyterHub.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnJupyterHub.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnJupyterHub.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_NAMESPACE | read-only "false" | *eksconfig.AddOnJupyterHub.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_PROXY_SECRET_TOKEN | read-only "false" | *eksconfig.AddOnJupyterHub.ProxySecretToken | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_NLB_ARN | read-only "true" | *eksconfig.AddOnJupyterHub.NLBARN | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_NLB_NAME | read-only "true" | *eksconfig.AddOnJupyterHub.NLBName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_URL | read-only "true" | *eksconfig.AddOnJupyterHub.URL | string |
-*----------------------------------------------------------*-------------------*---------------------------------------------*--------------------*
-
-
-# NOT WORKING...
-*-------------------------------------------------------*-------------------*-------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-------------------------------------------------------*-------------------*-------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_ENABLE | read-only "false" | *eksconfig.AddOnKubeflow.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_CREATED | read-only "true" | *eksconfig.AddOnKubeflow.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnKubeflow.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnKubeflow.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_KFCTL_PATH | read-only "false" | *eksconfig.AddOnKubeflow.KfctlPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_KFCTL_DOWNLOAD_URL | read-only "false" | *eksconfig.AddOnKubeflow.KfctlDownloadURL | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_BASE_DIR | read-only "false" | *eksconfig.AddOnKubeflow.BaseDir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_KF_DIR | read-only "true" | *eksconfig.AddOnKubeflow.KfDir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_KFCTL_CONFIG_PATH | read-only "true" | *eksconfig.AddOnKubeflow.KfctlConfigPath | string |
-*-------------------------------------------------------*-------------------*-------------------------------------------*--------------------*
-
-
-*-------------------------------------------------------------*-------------------*-----------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-------------------------------------------------------------*-------------------*-----------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_CUDA_VECTOR_ADD_ENABLE | read-only "false" | *eksconfig.AddOnCUDAVectorAdd.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CUDA_VECTOR_ADD_CREATED | read-only "true" | *eksconfig.AddOnCUDAVectorAdd.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CUDA_VECTOR_ADD_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnCUDAVectorAdd.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CUDA_VECTOR_ADD_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnCUDAVectorAdd.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CUDA_VECTOR_ADD_NAMESPACE | read-only "false" | *eksconfig.AddOnCUDAVectorAdd.Namespace | string |
-*-------------------------------------------------------------*-------------------*-----------------------------------------------*--------------------*
-
-
-*-----------------------------------------------------------------------------------*-------------------*--------------------------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-----------------------------------------------------------------------------------*-------------------*--------------------------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_ENABLE | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_CREATED | read-only "true" | *eksconfig.AddOnClusterLoaderLocal.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnClusterLoaderLocal.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnClusterLoaderLocal.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_S3_DIR | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.S3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_CLUSTER_LOADER_PATH | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.ClusterLoaderPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_CLUSTER_LOADER_DOWNLOAD_URL | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.ClusterLoaderDownloadURL | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_TEST_CONFIG_PATH | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.TestConfigPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_REPORT_DIR | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.ReportDir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_REPORT_TAR_GZ_PATH | read-only "true" | *eksconfig.AddOnClusterLoaderLocal.ReportTarGzPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_REPORT_TAR_GZ_S3_KEY | read-only "true" | *eksconfig.AddOnClusterLoaderLocal.ReportTarGzS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_LOG_PATH | read-only "true" | *eksconfig.AddOnClusterLoaderLocal.LogPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_LOG_S3_KEY | read-only "true" | *eksconfig.AddOnClusterLoaderLocal.LogS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_POD_STARTUP_LATENCY_PATH | read-only "true" | *eksconfig.AddOnClusterLoaderLocal.PodStartupLatencyPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_POD_STARTUP_LATENCY_S3_KEY | read-only "true" | *eksconfig.AddOnClusterLoaderLocal.PodStartupLatencyS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_RUNS | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.Runs | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_TIMEOUT | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.Timeout | time.Duration |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_NODES | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.Nodes | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_NODES_PER_NAMESPACE | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.NodesPerNamespace | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_PODS_PER_NODE | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.PodsPerNode | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_BIG_GROUP_SIZE | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.BigGroupSize | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_MEDIUM_GROUP_SIZE | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.MediumGroupSize | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_SMALL_GROUP_SIZE | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.SmallGroupSize | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_SMALL_STATEFUL_SETS_PER_NAMESPACE | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.SmallStatefulSetsPerNamespace | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_MEDIUM_STATEFUL_SETS_PER_NAMESPACE | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.MediumStatefulSetsPerNamespace | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_CL2_USE_HOST_NETWORK_PODS | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.CL2UseHostNetworkPods | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_CL2_LOAD_TEST_THROUGHPUT | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.CL2LoadTestThroughput | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_CL2_ENABLE_PVS | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.CL2EnablePVS | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_CL2_SCHEDULER_THROUGHPUT_THRESHOLD | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.CL2SchedulerThroughputThreshold | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_PROMETHEUS_SCRAPE_KUBE_PROXY | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.PrometheusScrapeKubeProxy | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_ENABLE_SYSTEM_POD_METRICS | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.EnableSystemPodMetrics | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_POD_STARTUP_LATENCY | read-only "true" | *eksconfig.AddOnClusterLoaderLocal.PodStartupLatency | util.PerfData |
-*-----------------------------------------------------------------------------------*-------------------*--------------------------------------------------------------------*--------------------*
-
-
-*------------------------------------------------------------------------------------*-------------------*---------------------------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*------------------------------------------------------------------------------------*-------------------*---------------------------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_ENABLE | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_CREATED | read-only "true" | *eksconfig.AddOnClusterLoaderRemote.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnClusterLoaderRemote.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnClusterLoaderRemote.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_S3_DIR | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.S3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_NAMESPACE | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_REPOSITORY_ACCOUNT_ID | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.RepositoryAccountID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_REPOSITORY_REGION | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.RepositoryRegion | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_REPOSITORY_NAME | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.RepositoryName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_REPOSITORY_IMAGE_TAG | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.RepositoryImageTag | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_CLUSTER_LOADER_PATH | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.ClusterLoaderPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_CLUSTER_LOADER_DOWNLOAD_URL | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.ClusterLoaderDownloadURL | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_REPORT_TAR_GZ_PATH | read-only "true" | *eksconfig.AddOnClusterLoaderRemote.ReportTarGzPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_REPORT_TAR_GZ_S3_KEY | read-only "true" | *eksconfig.AddOnClusterLoaderRemote.ReportTarGzS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_LOG_PATH | read-only "true" | *eksconfig.AddOnClusterLoaderRemote.LogPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_LOG_S3_KEY | read-only "true" | *eksconfig.AddOnClusterLoaderRemote.LogS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_POD_STARTUP_LATENCY_PATH | read-only "true" | *eksconfig.AddOnClusterLoaderRemote.PodStartupLatencyPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_POD_STARTUP_LATENCY_S3_KEY | read-only "true" | *eksconfig.AddOnClusterLoaderRemote.PodStartupLatencyS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_RUNS | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.Runs | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_NODES | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.Nodes | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_TIMEOUT | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.Timeout | time.Duration |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_NODES_PER_NAMESPACE | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.NodesPerNamespace | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_PODS_PER_NODE | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.PodsPerNode | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_BIG_GROUP_SIZE | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.BigGroupSize | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_MEDIUM_GROUP_SIZE | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.MediumGroupSize | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_SMALL_GROUP_SIZE | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.SmallGroupSize | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_SMALL_STATEFUL_SETS_PER_NAMESPACE | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.SmallStatefulSetsPerNamespace | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_MEDIUM_STATEFUL_SETS_PER_NAMESPACE | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.MediumStatefulSetsPerNamespace | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_CL2_USE_HOST_NETWORK_PODS | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.CL2UseHostNetworkPods | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_CL2_LOAD_TEST_THROUGHPUT | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.CL2LoadTestThroughput | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_CL2_ENABLE_PVS | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.CL2EnablePVS | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_CL2_SCHEDULER_THROUGHPUT_THRESHOLD | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.CL2SchedulerThroughputThreshold | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_PROMETHEUS_SCRAPE_KUBE_PROXY | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.PrometheusScrapeKubeProxy | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_ENABLE_SYSTEM_POD_METRICS | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.EnableSystemPodMetrics | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_POD_STARTUP_LATENCY | read-only "true" | *eksconfig.AddOnClusterLoaderRemote.PodStartupLatency | util.PerfData |
-*------------------------------------------------------------------------------------*-------------------*---------------------------------------------------------------------*--------------------*
-
-
-*----------------------------------------------------------------------------------------*-------------------*----------------------------------------------------------------------*-------------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*----------------------------------------------------------------------------------------*-------------------*----------------------------------------------------------------------*-------------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_ENABLE | read-only "false" | *eksconfig.AddOnStresserLocal.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_CREATED | read-only "true" | *eksconfig.AddOnStresserLocal.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnStresserLocal.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnStresserLocal.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_S3_DIR | read-only "false" | *eksconfig.AddOnStresserLocal.S3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_NAMESPACE | read-only "false" | *eksconfig.AddOnStresserLocal.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_OBJECT_SIZE | read-only "false" | *eksconfig.AddOnStresserLocal.ObjectSize | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_LIST_LIMIT | read-only "false" | *eksconfig.AddOnStresserLocal.ListLimit | int64 |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_DURATION | read-only "false" | *eksconfig.AddOnStresserLocal.Duration | time.Duration |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_DURATION_STRING | read-only "true" | *eksconfig.AddOnStresserLocal.DurationString | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_RAW_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsRawWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_RAW_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsRawWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_RAW_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnStresserLocal.RequestsRawWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsRawWritesCompareAllJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsRawWritesCompareAllJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsRawWritesCompareAllCSVPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_S3_KEY | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsRawWritesCompareAllCSVS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_WRITES | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryWrites | metrics.RequestsSummary |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_WRITES_TABLE_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryWritesTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_WRITES_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryWritesTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnStresserLocal.RequestsSummaryWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryWritesCompare | metrics.RequestsCompare |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryWritesCompareJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryWritesCompareJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryWritesCompareTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryWritesCompareTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_RAW_READS_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsRawReadsJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_RAW_READS_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsRawReadsJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_RAW_READS_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnStresserLocal.RequestsRawReadsCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_RAW_READS_COMPARE_ALL_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsRawReadsCompareAllJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_RAW_READS_COMPARE_ALL_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsRawReadsCompareAllJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_RAW_READS_COMPARE_ALL_CSV_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsRawReadsCompareAllCSVPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_RAW_READS_COMPARE_ALL_CSV_S3_KEY | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsRawReadsCompareAllCSVS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_READS | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryReads | metrics.RequestsSummary |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_READS_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryReadsJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_READS_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryReadsJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_READS_TABLE_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryReadsTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_READS_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryReadsTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_READS_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnStresserLocal.RequestsSummaryReadsCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_READS_COMPARE | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryReadsCompare | metrics.RequestsCompare |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_READS_COMPARE_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryReadsCompareJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_READS_COMPARE_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryReadsCompareJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_READS_COMPARE_TABLE_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryReadsCompareTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_REQUESTS_SUMMARY_READS_COMPARE_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnStresserLocal.RequestsSummaryReadsCompareTableS3Key | string |
-*----------------------------------------------------------------------------------------*-------------------*----------------------------------------------------------------------*-------------------------*
-
-
-*-----------------------------------------------------------------------------------------*-------------------*-----------------------------------------------------------------------*-------------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*-----------------------------------------------------------------------------------------*-------------------*-----------------------------------------------------------------------*-------------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_ENABLE | read-only "false" | *eksconfig.AddOnStresserRemote.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_CREATED | read-only "true" | *eksconfig.AddOnStresserRemote.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnStresserRemote.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_TIME_FRAME_DELETE | read-only "true" | *eksconfig.AddOnStresserRemote.TimeFrameDelete | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_S3_DIR | read-only "false" | *eksconfig.AddOnStresserRemote.S3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_NAMESPACE | read-only "false" | *eksconfig.AddOnStresserRemote.Namespace | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REPOSITORY_ACCOUNT_ID | read-only "false" | *eksconfig.AddOnStresserRemote.RepositoryAccountID | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REPOSITORY_REGION | read-only "false" | *eksconfig.AddOnStresserRemote.RepositoryRegion | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REPOSITORY_NAME | read-only "false" | *eksconfig.AddOnStresserRemote.RepositoryName | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REPOSITORY_IMAGE_TAG | read-only "false" | *eksconfig.AddOnStresserRemote.RepositoryImageTag | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_COMPLETES | read-only "false" | *eksconfig.AddOnStresserRemote.Completes | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_PARALLELS | read-only "false" | *eksconfig.AddOnStresserRemote.Parallels | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_OBJECT_SIZE | read-only "false" | *eksconfig.AddOnStresserRemote.ObjectSize | int |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_LIST_LIMIT | read-only "false" | *eksconfig.AddOnStresserRemote.ListLimit | int64 |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_DURATION | read-only "false" | *eksconfig.AddOnStresserRemote.Duration | time.Duration |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_DURATION_STRING | read-only "true" | *eksconfig.AddOnStresserRemote.DurationString | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_WRITES_OUTPUT_NAME_PREFIX | read-only "false" | *eksconfig.AddOnStresserRemote.RequestsSummaryWritesOutputNamePrefix | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_READS_OUTPUT_NAME_PREFIX | read-only "false" | *eksconfig.AddOnStresserRemote.RequestsSummaryReadsOutputNamePrefix | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_RAW_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsRawWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_RAW_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsRawWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_RAW_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnStresserRemote.RequestsRawWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsRawWritesCompareAllJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsRawWritesCompareAllJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsRawWritesCompareAllCSVPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_RAW_WRITES_COMPARE_ALL_CSV_S3_KEY | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsRawWritesCompareAllCSVS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_WRITES | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryWrites | metrics.RequestsSummary |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_WRITES_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryWritesJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_WRITES_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryWritesJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_WRITES_TABLE_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryWritesTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_WRITES_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryWritesTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnStresserRemote.RequestsSummaryWritesCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryWritesCompare | metrics.RequestsCompare |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryWritesCompareJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryWritesCompareJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryWritesCompareTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_WRITES_COMPARE_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryWritesCompareTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_RAW_READS_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsRawReadsJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_RAW_READS_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsRawReadsJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_RAW_READS_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnStresserRemote.RequestsRawReadsCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_RAW_READS_COMPARE_ALL_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsRawReadsCompareAllJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_RAW_READS_COMPARE_ALL_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsRawReadsCompareAllJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_RAW_READS_COMPARE_ALL_CSV_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsRawReadsCompareAllCSVPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_RAW_READS_COMPARE_ALL_CSV_S3_KEY | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsRawReadsCompareAllCSVS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_READS | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryReads | metrics.RequestsSummary |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_READS_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryReadsJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_READS_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryReadsJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_READS_TABLE_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryReadsTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_READS_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryReadsTableS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_READS_COMPARE_S3_DIR | read-only "false" | *eksconfig.AddOnStresserRemote.RequestsSummaryReadsCompareS3Dir | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_READS_COMPARE | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryReadsCompare | metrics.RequestsCompare |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_READS_COMPARE_JSON_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryReadsCompareJSONPath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_READS_COMPARE_JSON_S3_KEY | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryReadsCompareJSONS3Key | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_READS_COMPARE_TABLE_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryReadsCompareTablePath | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_READS_COMPARE_TABLE_S3_PATH | read-only "true" | *eksconfig.AddOnStresserRemote.RequestsSummaryReadsCompareTableS3Key | string |
-*-----------------------------------------------------------------------------------------*-------------------*-----------------------------------------------------------------------*-------------------------*
-
-
-*------------------------------------------------------------------------------*-------------------*---------------------------------------------------------------*--------------------*
-| ENVIRONMENTAL VARIABLE | READ ONLY | TYPE | GO TYPE |
-*------------------------------------------------------------------------------*-------------------*---------------------------------------------------------------*--------------------*
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_VERSION_UPGRADE_ENABLE | read-only "false" | *eksconfig.AddOnClusterVersionUpgrade.Enable | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_VERSION_UPGRADE_CREATED | read-only "true" | *eksconfig.AddOnClusterVersionUpgrade.Created | bool |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_VERSION_UPGRADE_TIME_FRAME_CREATE | read-only "true" | *eksconfig.AddOnClusterVersionUpgrade.TimeFrameCreate | timeutil.TimeFrame |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_VERSION_UPGRADE_WAIT_BEFORE_UPGRADE | read-only "false" | *eksconfig.AddOnClusterVersionUpgrade.WaitBeforeUpgrade | time.Duration |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_VERSION_UPGRADE_WAIT_BEFORE_UPGRADE_STRING | read-only "true" | *eksconfig.AddOnClusterVersionUpgrade.WaitBeforeUpgradeString | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_VERSION_UPGRADE_VERSION | read-only "false" | *eksconfig.AddOnClusterVersionUpgrade.Version | string |
-| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_VERSION_UPGRADE_VERSION_VALUE | read-only "true" | *eksconfig.AddOnClusterVersionUpgrade.VersionValue | float64 |
-*------------------------------------------------------------------------------*-------------------*---------------------------------------------------------------*--------------------*
-
-
-```
diff --git a/eksconfig/add-on-alb-2048.go b/eksconfig/add-on-alb-2048.go
deleted file mode 100644
index 3f345ca96..000000000
--- a/eksconfig/add-on-alb-2048.go
+++ /dev/null
@@ -1,75 +0,0 @@
-package eksconfig
-
-import (
- "errors"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnALB2048 defines parameters for EKS cluster
-// add-on ALB 2048 service.
-type AddOnALB2048 struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // DeploymentReplicasALB is the number of ALB replicas to deploy using "Deployment" object.
- DeploymentReplicasALB int32 `json:"deployment-replicas-alb"`
- // DeploymentReplicas2048 is the number of 2048 replicas to deploy using "Deployment" object.
- DeploymentReplicas2048 int32 `json:"deployment-replicas-2048"`
- // DeploymentNodeSelector2048 is configured to overwrite existing node selector
- // for ALB 2048 deployment. If left empty, tester sets default selector.
- DeploymentNodeSelector2048 map[string]string `json:"deployment-node-selector-2048"`
-
- // ALBARN is the ARN of the ALB created from the service.
- ALBARN string `json:"alb-arn" read-only:"true"`
- // ALBName is the name of the ALB created from the service.
- ALBName string `json:"alb-name" read-only:"true"`
- // URL is the URL for ALB 2048 Service.
- URL string `json:"url" read-only:"true"`
-}
-
-// EnvironmentVariablePrefixAddOnALB2048 is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnALB2048 = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_ALB_2048_"
-
-// IsEnabledAddOnALB2048 returns true if "AddOnALB2048" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnALB2048() bool {
- if cfg.AddOnALB2048 == nil {
- return false
- }
- if cfg.AddOnALB2048.Enable {
- return true
- }
- cfg.AddOnALB2048 = nil
- return false
-}
-
-func getDefaultAddOnALB2048() *AddOnALB2048 {
- return &AddOnALB2048{
- Enable: false,
- DeploymentReplicasALB: 3,
- DeploymentReplicas2048: 3,
- DeploymentNodeSelector2048: make(map[string]string),
- }
-}
-
-func (cfg *Config) validateAddOnALB2048() error {
- if !cfg.IsEnabledAddOnALB2048() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnALB2048.Enable true but no node group is enabled")
- }
- if cfg.AddOnALB2048.Namespace == "" {
- cfg.AddOnALB2048.Namespace = cfg.Name + "-alb-2048"
- }
- return nil
-}
diff --git a/eksconfig/add-on-ami-soft-lockup-issue-454.go b/eksconfig/add-on-ami-soft-lockup-issue-454.go
deleted file mode 100644
index 16abcb4f3..000000000
--- a/eksconfig/add-on-ami-soft-lockup-issue-454.go
+++ /dev/null
@@ -1,65 +0,0 @@
-package eksconfig
-
-import (
- "errors"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnAmiSoftLockupIssue454 defines parameters for EKS cluster
-// add-on NLB hello-world service.
-type AddOnAmiSoftLockupIssue454 struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // DeploymentReplicas is the number of replicas to deploy using "Deployment" object.
- DeploymentReplicas int32 `json:"deployment-replicas"`
- // DeploymentNodeSelector is configured to overwrite existing node selector
- // for deployment. If left empty, tester sets default selector.
- DeploymentNodeSelector map[string]string `json:"deployment-node-selector"`
-}
-
-// EnvironmentVariablePrefixAddOnAmiSoftLockupIssue454 is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnAmiSoftLockupIssue454 = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_AMI_SOFT_LOCKUP_ISSUE_454_"
-
-// IsEnabledAddOnAmiSoftLockupIssue454 returns true if "AddOnAmiSoftLockupIssue454" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnAmiSoftLockupIssue454() bool {
- if cfg.AddOnAmiSoftLockupIssue454 == nil {
- return false
- }
- if cfg.AddOnAmiSoftLockupIssue454.Enable {
- return true
- }
- cfg.AddOnAmiSoftLockupIssue454 = nil
- return false
-}
-
-func getDefaultAddOnAmiSoftLockupIssue454() *AddOnAmiSoftLockupIssue454 {
- return &AddOnAmiSoftLockupIssue454{
- Enable: false,
- DeploymentReplicas: 8,
- DeploymentNodeSelector: make(map[string]string),
- }
-}
-
-func (cfg *Config) validateAddOnAmiSoftLockupIssue454() error {
- if !cfg.IsEnabledAddOnAmiSoftLockupIssue454() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnAmiSoftLockupIssue454.Enable true but no node group is enabled")
- }
- if cfg.AddOnAmiSoftLockupIssue454.Namespace == "" {
- cfg.AddOnAmiSoftLockupIssue454.Namespace = cfg.Name + "-ami-soft-lockup-issue-454"
- }
- return nil
-}
diff --git a/eksconfig/add-on-app-mesh.go b/eksconfig/add-on-app-mesh.go
deleted file mode 100644
index 8804214ff..000000000
--- a/eksconfig/add-on-app-mesh.go
+++ /dev/null
@@ -1,90 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "path"
- "path/filepath"
- "strings"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnAppMesh defines parameters for EKS cluster
-// add-on "EKS App Mesh Integration".
-type AddOnAppMesh struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // S3Dir is the S3 directory to store all test results.
- // It is under the bucket "eksconfig.Config.S3BucketName".
- S3Dir string `json:"s3-dir"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // ControllerImage is the image of appMesh controller
- ControllerImage string `json:"controller-image"`
- // InjectorImage is the image of appMesh injector
- InjectorImage string `json:"injector-image"`
-
- // PolicyCFNStackID is the CFN stack ID for policy.
- PolicyCFNStackID string `json:"policy-cfn-stack-id,omitempty" read-only:"true"`
- PolicyCFNStackYAMLPath string `json:"policy-cfn-stack-yaml-path" read-only:"true"`
- PolicyCFNStackYAMLS3Key string `json:"policy-cfn-stack-yaml-s3-key" read-only:"true"`
-}
-
-// EnvironmentVariablePrefixAddOnAppMesh is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnAppMesh = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_APP_MESH_"
-
-// IsEnabledAddOnAppMesh returns true if "AddOnAppMesh" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnAppMesh() bool {
- if cfg.AddOnAppMesh == nil {
- return false
- }
- if cfg.AddOnAppMesh.Enable {
- return true
- }
- cfg.AddOnAppMesh = nil
- return false
-}
-
-func getDefaultAddOnAppMesh() *AddOnAppMesh {
- return &AddOnAppMesh{
- Enable: false,
- }
-}
-
-func (cfg *Config) validateAddOnAppMesh() error {
- if !cfg.IsEnabledAddOnAppMesh() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnAppMesh.Enable true but no node group is enabled")
- }
-
- if cfg.AddOnAppMesh.S3Dir == "" {
- cfg.AddOnAppMesh.S3Dir = path.Join(cfg.Name, "add-on-app-mesh")
- }
-
- if cfg.AddOnAppMesh.Namespace == "" {
- cfg.AddOnAppMesh.Namespace = cfg.Name + "-appmesh"
- }
-
- if cfg.AddOnAppMesh.PolicyCFNStackYAMLPath == "" {
- cfg.AddOnAppMesh.PolicyCFNStackYAMLPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + ".add-on-app-mesh.policy.cfn.yaml"
- }
- if cfg.AddOnAppMesh.PolicyCFNStackYAMLS3Key == "" {
- cfg.AddOnAppMesh.PolicyCFNStackYAMLS3Key = path.Join(
- cfg.AddOnAppMesh.S3Dir,
- filepath.Base(cfg.AddOnAppMesh.PolicyCFNStackYAMLPath),
- )
- }
-
- return nil
-}
diff --git a/eksconfig/add-on-cluster-loader-local.go b/eksconfig/add-on-cluster-loader-local.go
deleted file mode 100644
index 3b8b72c92..000000000
--- a/eksconfig/add-on-cluster-loader-local.go
+++ /dev/null
@@ -1,237 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
- "path"
- "path/filepath"
- "runtime"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/pkg/fileutil"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- measurement_util "k8s.io/perf-tests/clusterloader2/pkg/measurement/util"
-)
-
-/*
-Note: make sure all other test config is copied in the "same" directory as "--testconfig" (in local)
-
-"/var/log/cluster-loader-remote.log" output:
-I0529 18:59:08.745755 27 simple_test_executor.go:162] Step "Scaling and updating objects" ended
-W0529 18:59:08.745762 27 simple_test_executor.go:165] Got errors during step execution: [reading template (job.yaml) for identifier error: reading error: open /job.yaml: no such file or directory
-reading template (statefulset.yaml) for identifier error: reading error: open /statefulset.yaml: no such file or directory
-reading template (daemonset.yaml) for identifier error: reading error: open /daemonset.yaml: no such file or directory
-reading template (deployment.yaml) for identifier error: reading error: open /deployment.yaml: no such file or directory
-reading template (statefulset.yaml) for identifier error: reading error: open /statefulset.yaml: no such file or directory
-reading template (deployment.yaml) for identifier error: reading error: open /deployment.yaml: no such file or directory
-reading template (deployment.yaml) for identifier error: reading error: open /deployment.yaml: no such file or directory
-reading template (job.yaml) for identifier error: reading error: open /job.yaml: no such file or directory
-reading template (job.yaml) for identifier error: reading error: open /job.yaml: no such file or directory]
-I0529 18:59:08.745802 27 simple_test_executor.go:135] Step "Waiting for objects to become scaled" started
-*/
-
-// AddOnClusterLoaderLocal defines parameters for EKS cluster
-// add-on cluster loader local.
-// It generates loads from the local host machine.
-// ref. https://github.com/kubernetes/perf-tests/tree/master/clusterloader2
-type AddOnClusterLoaderLocal struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // S3Dir is the S3 directory to store all test results.
- // It is under the bucket "eksconfig.Config.S3BucketName".
- S3Dir string `json:"s3-dir"`
-
- // ClusterLoaderPath is the clusterloader executable binary path.
- // ref. https://github.com/kubernetes/perf-tests/tree/master/clusterloader2
- ClusterLoaderPath string `json:"cluster-loader-path"`
- ClusterLoaderDownloadURL string `json:"cluster-loader-download-url"`
- // TestConfigPath is the clusterloader2 test configuration file.
- // Set via "--testconfig" flag.
- TestConfigPath string `json:"test-config-path"`
-
- // ReportDir is the clusterloader2 test report directory.
- // Set via "--report-dir" flag.
- ReportDir string `json:"report-dir"`
-
- // ReportTarGzPath is the .tar.gz file path for report directory.
- ReportTarGzPath string `json:"report-tar-gz-path" read-only:"true"`
- ReportTarGzS3Key string `json:"report-tar-gz-s3-key" read-only:"true"`
- // LogPath is the log file path to stream clusterloader binary runs.
- LogPath string `json:"log-path" read-only:"true"`
- LogS3Key string `json:"log-s3-key" read-only:"true"`
- // PodStartupLatencyPath is the JSON file path to store pod startup latency.
- PodStartupLatencyPath string `json:"pod-startup-latency-path" read-only:"true"`
- PodStartupLatencyS3Key string `json:"pod-startup-latency-s3-key" read-only:"true"`
-
- // Runs is the number of "clusterloader2" runs back-to-back.
- Runs int `json:"runs"`
- // Timeout is the timeout for the total test runs.
- Timeout time.Duration `json:"timeout"`
-
- // Nodes is the number of nodes.
- // Set via "--nodes" flag.
- Nodes int `json:"nodes"`
-
- //
- //
- // below are set via "--testoverrides" flag
- // see https://github.com/kubernetes/perf-tests/tree/master/clusterloader2/testing/overrides for more.
-
- NodesPerNamespace int `json:"nodes-per-namespace"`
- PodsPerNode int `json:"pods-per-node"`
-
- BigGroupSize int `json:"big-group-size"`
- MediumGroupSize int `json:"medium-group-size"`
- SmallGroupSize int `json:"small-group-size"`
-
- SmallStatefulSetsPerNamespace int `json:"small-stateful-sets-per-namespace"`
- MediumStatefulSetsPerNamespace int `json:"medium-stateful-sets-per-namespace"`
-
- // ref. https://github.com/kubernetes/perf-tests/pull/1345
- CL2UseHostNetworkPods bool `json:"cl2-use-host-network-pods"`
- CL2LoadTestThroughput int `json:"cl2-load-test-throughput"`
- CL2EnablePVS bool `json:"cl2-enable-pvs"`
- CL2SchedulerThroughputThreshold int `json:"cl2-scheduler-throughput-threshold"`
- PrometheusScrapeKubeProxy bool `json:"prometheus-scrape-kube-proxy"`
- EnableSystemPodMetrics bool `json:"enable-system-pod-metrics"`
-
- PodStartupLatency measurement_util.PerfData `json:"pod-startup-latency" read-only:"true"`
-}
-
-// EnvironmentVariablePrefixAddOnClusterLoaderLocal is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnClusterLoaderLocal = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_CLUSTER_LOADER_LOCAL_"
-
-// IsEnabledAddOnClusterLoaderLocal returns true if "AddOnClusterLoaderLocal" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnClusterLoaderLocal() bool {
- if cfg.AddOnClusterLoaderLocal == nil {
- return false
- }
- if cfg.AddOnClusterLoaderLocal.Enable {
- return true
- }
- cfg.AddOnClusterLoaderLocal = nil
- return false
-}
-
-func getDefaultAddOnClusterLoaderLocal() *AddOnClusterLoaderLocal {
- cfg := &AddOnClusterLoaderLocal{
- Enable: false,
-
- ClusterLoaderPath: "/tmp/clusterloader2",
- ClusterLoaderDownloadURL: "https://github.com/aws/aws-k8s-tester/releases/download/v1.5.2/clusterloader2-linux-amd64",
-
- Runs: 2,
- Timeout: 30 * time.Minute,
-
- Nodes: 10,
-
- NodesPerNamespace: 10,
- PodsPerNode: 10,
-
- BigGroupSize: 25,
- MediumGroupSize: 10,
- SmallGroupSize: 5,
-
- SmallStatefulSetsPerNamespace: 0,
- MediumStatefulSetsPerNamespace: 0,
-
- CL2UseHostNetworkPods: false,
-
- // ref. https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/testing/load/kubemark/throughput_override.yaml
- CL2LoadTestThroughput: 20,
- CL2EnablePVS: false,
- CL2SchedulerThroughputThreshold: 100,
- PrometheusScrapeKubeProxy: false,
- EnableSystemPodMetrics: false,
- }
- if runtime.GOOS == "darwin" {
- cfg.ClusterLoaderDownloadURL = strings.Replace(cfg.ClusterLoaderDownloadURL, "linux", "darwin", -1)
- }
- return cfg
-}
-
-func (cfg *Config) validateAddOnClusterLoaderLocal() error {
- if !cfg.IsEnabledAddOnClusterLoaderLocal() {
- return nil
- }
-
- if cfg.AddOnClusterLoaderLocal.S3Dir == "" {
- cfg.AddOnClusterLoaderLocal.S3Dir = path.Join(cfg.Name, "add-on-cluster-loader-local")
- }
-
- if cfg.AddOnClusterLoaderLocal.ReportTarGzPath == "" {
- cfg.AddOnClusterLoaderLocal.ReportTarGzPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-cluster-loader-local.tar.gz"
- }
- if cfg.AddOnClusterLoaderLocal.ReportTarGzS3Key == "" {
- cfg.AddOnClusterLoaderLocal.ReportTarGzS3Key = path.Join(
- cfg.AddOnClusterLoaderLocal.S3Dir,
- filepath.Base(cfg.AddOnClusterLoaderLocal.ReportTarGzPath),
- )
- }
- if cfg.AddOnClusterLoaderLocal.LogPath == "" {
- cfg.AddOnClusterLoaderLocal.LogPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-cluster-loader-local.log"
- }
- if cfg.AddOnClusterLoaderLocal.LogS3Key == "" {
- cfg.AddOnClusterLoaderLocal.LogS3Key = path.Join(
- cfg.AddOnClusterLoaderLocal.S3Dir,
- filepath.Base(cfg.AddOnClusterLoaderLocal.LogPath),
- )
- }
- if cfg.AddOnClusterLoaderLocal.PodStartupLatencyPath == "" {
- cfg.AddOnClusterLoaderLocal.PodStartupLatencyPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-cluster-loader-remote.log"
- }
- if cfg.AddOnClusterLoaderLocal.PodStartupLatencyS3Key == "" {
- cfg.AddOnClusterLoaderLocal.PodStartupLatencyS3Key = path.Join(
- cfg.AddOnClusterLoaderLocal.S3Dir,
- filepath.Base(cfg.AddOnClusterLoaderLocal.PodStartupLatencyPath),
- )
- }
-
- if cfg.AddOnClusterLoaderLocal.ClusterLoaderPath == "" && cfg.AddOnClusterLoaderLocal.ClusterLoaderDownloadURL == "" {
- return errors.New("empty AddOnClusterLoaderLocal.ClusterLoaderPath and ClusterLoaderDownloadURL")
- }
- if cfg.AddOnClusterLoaderLocal.TestConfigPath == "" {
- return errors.New("empty AddOnClusterLoaderLocal.TestConfigPath")
- }
- if cfg.AddOnClusterLoaderLocal.ReportDir == "" {
- cfg.AddOnClusterLoaderLocal.ReportDir = filepath.Join(filepath.Dir(cfg.ConfigPath), cfg.Name+"-cluster-loader-local-report")
- }
- if err := fileutil.IsDirWriteable(cfg.AddOnClusterLoaderLocal.ReportDir); err != nil {
- return err
- }
-
- if cfg.AddOnClusterLoaderLocal.Runs == 0 {
- return errors.New("unexpected zero AddOnClusterLoaderLocal.Runs")
- }
- if cfg.AddOnClusterLoaderLocal.Timeout == 0 {
- return errors.New("unexpected zero AddOnClusterLoaderLocal.Timeout")
- }
-
- if cfg.AddOnClusterLoaderLocal.Nodes == 0 {
- return errors.New("unexpected zero AddOnClusterLoaderLocal.Nodes")
- }
-
- if cfg.AddOnClusterLoaderLocal.CL2LoadTestThroughput == 0 {
- // ref. https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/testing/load/kubemark/throughput_override.yaml
- cfg.AddOnClusterLoaderLocal.CL2LoadTestThroughput = 20
- }
- if cfg.AddOnClusterLoaderLocal.PrometheusScrapeKubeProxy {
- return fmt.Errorf("unexpected AddOnClusterLoaderLocal.PrometheusScrapeKubeProxy %v; not supported yet", cfg.AddOnClusterLoaderLocal.PrometheusScrapeKubeProxy)
- }
- if cfg.AddOnClusterLoaderLocal.EnableSystemPodMetrics {
- return fmt.Errorf("unexpected AddOnClusterLoaderLocal.EnableSystemPodMetrics %v; not supported yet", cfg.AddOnClusterLoaderLocal.EnableSystemPodMetrics)
- }
- if cfg.AddOnClusterLoaderLocal.CL2SchedulerThroughputThreshold <= 0 {
- cfg.AddOnClusterLoaderLocal.CL2SchedulerThroughputThreshold = 100
- }
-
- return nil
-}
diff --git a/eksconfig/add-on-cluster-loader-remote.go b/eksconfig/add-on-cluster-loader-remote.go
deleted file mode 100644
index bb1473d77..000000000
--- a/eksconfig/add-on-cluster-loader-remote.go
+++ /dev/null
@@ -1,265 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
- "path"
- "path/filepath"
- "runtime"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- measurement_util "k8s.io/perf-tests/clusterloader2/pkg/measurement/util"
-)
-
-/*
-Note: make sure all other test config is copied in the "same" directory as "--testconfig" (in Dockerfile)
-
-"/var/log/cluster-loader-remote.log" output:
-I0529 18:59:08.745755 27 simple_test_executor.go:162] Step "Scaling and updating objects" ended
-W0529 18:59:08.745762 27 simple_test_executor.go:165] Got errors during step execution: [reading template (job.yaml) for identifier error: reading error: open /job.yaml: no such file or directory
-reading template (statefulset.yaml) for identifier error: reading error: open /statefulset.yaml: no such file or directory
-reading template (daemonset.yaml) for identifier error: reading error: open /daemonset.yaml: no such file or directory
-reading template (deployment.yaml) for identifier error: reading error: open /deployment.yaml: no such file or directory
-reading template (statefulset.yaml) for identifier error: reading error: open /statefulset.yaml: no such file or directory
-reading template (deployment.yaml) for identifier error: reading error: open /deployment.yaml: no such file or directory
-reading template (deployment.yaml) for identifier error: reading error: open /deployment.yaml: no such file or directory
-reading template (job.yaml) for identifier error: reading error: open /job.yaml: no such file or directory
-reading template (job.yaml) for identifier error: reading error: open /job.yaml: no such file or directory]
-I0529 18:59:08.745802 27 simple_test_executor.go:135] Step "Waiting for objects to become scaled" started
-*/
-
-// AddOnClusterLoaderRemote defines parameters for EKS cluster
-// add-on cluster loader remote.
-// It generates loads from the remote host machine.
-// ref. https://github.com/kubernetes/perf-tests/pull/1295
-// ref. https://github.com/kubernetes/perf-tests/tree/master/clusterloader2
-type AddOnClusterLoaderRemote struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // S3Dir is the S3 directory to store all test results.
- // It is under the bucket "eksconfig.Config.S3BucketName".
- S3Dir string `json:"s3-dir"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // RepositoryAccountID is the account ID for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryAccountID string `json:"repository-account-id,omitempty"`
- // RepositoryRegion is the ECR repository region to pull from.
- RepositoryRegion string `json:"repository-region,omitempty"`
- // RepositoryName is the repositoryName for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryName string `json:"repository-name,omitempty"`
- // RepositoryImageTag is the image tag for tester ECR image.
- // e.g. "latest" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester:latest"
- RepositoryImageTag string `json:"repository-image-tag,omitempty"`
-
- // ClusterLoaderPath is the clusterloader executable binary path.
- // ref. https://github.com/kubernetes/perf-tests/tree/master/clusterloader2
- ClusterLoaderPath string `json:"cluster-loader-path"`
- ClusterLoaderDownloadURL string `json:"cluster-loader-download-url"`
-
- // ReportTarGzPath is the .tar.gz file path for report directory.
- // This is the local path after downloaded from remote nodes.
- ReportTarGzPath string `json:"report-tar-gz-path" read-only:"true"`
- ReportTarGzS3Key string `json:"report-tar-gz-s3-key" read-only:"true"`
- // LogPath is the log file path to stream clusterloader binary runs.
- LogPath string `json:"log-path" read-only:"true"`
- LogS3Key string `json:"log-s3-key" read-only:"true"`
- // PodStartupLatencyPath is the JSON file path to store pod startup latency.
- PodStartupLatencyPath string `json:"pod-startup-latency-path" read-only:"true"`
- PodStartupLatencyS3Key string `json:"pod-startup-latency-s3-key" read-only:"true"`
-
- // Runs is the number of "clusterloader2" runs back-to-back.
- Runs int `json:"runs"`
-
- // Nodes is the number of nodes.
- // Set via "--nodes" flag.
- Nodes int `json:"nodes"`
- // Timeout is the timeout for the total test runs.
- Timeout time.Duration `json:"timeout"`
-
- //
- //
- // below are set via "--testoverrides" flag
- // see https://github.com/kubernetes/perf-tests/tree/master/clusterloader2/testing/overrides for more.
-
- NodesPerNamespace int `json:"nodes-per-namespace"`
- PodsPerNode int `json:"pods-per-node"`
-
- BigGroupSize int `json:"big-group-size"`
- MediumGroupSize int `json:"medium-group-size"`
- SmallGroupSize int `json:"small-group-size"`
-
- SmallStatefulSetsPerNamespace int `json:"small-stateful-sets-per-namespace"`
- MediumStatefulSetsPerNamespace int `json:"medium-stateful-sets-per-namespace"`
-
- // ref. https://github.com/kubernetes/perf-tests/pull/1345
- CL2UseHostNetworkPods bool `json:"cl2-use-host-network-pods"`
- CL2LoadTestThroughput int `json:"cl2-load-test-throughput"`
- CL2EnablePVS bool `json:"cl2-enable-pvs"`
- CL2SchedulerThroughputThreshold int `json:"cl2-scheduler-throughput-threshold"`
- PrometheusScrapeKubeProxy bool `json:"prometheus-scrape-kube-proxy"`
- EnableSystemPodMetrics bool `json:"enable-system-pod-metrics"`
-
- PodStartupLatency measurement_util.PerfData `json:"pod-startup-latency" read-only:"true"`
-}
-
-// EnvironmentVariablePrefixAddOnClusterLoaderRemote is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnClusterLoaderRemote = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_CLUSTER_LOADER_REMOTE_"
-
-// IsEnabledAddOnClusterLoaderRemote returns true if "AddOnClusterLoaderRemote" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnClusterLoaderRemote() bool {
- if cfg.AddOnClusterLoaderRemote == nil {
- return false
- }
- if cfg.AddOnClusterLoaderRemote.Enable {
- return true
- }
- cfg.AddOnClusterLoaderRemote = nil
- return false
-}
-
-func getDefaultAddOnClusterLoaderRemote() *AddOnClusterLoaderRemote {
- cfg := &AddOnClusterLoaderRemote{
- Enable: false,
-
- ClusterLoaderPath: "/tmp/clusterloader2",
- ClusterLoaderDownloadURL: "https://github.com/aws/aws-k8s-tester/releases/download/v1.5.2/clusterloader2-linux-amd64",
-
- Runs: 2,
- Timeout: 30 * time.Minute,
-
- Nodes: 10,
-
- NodesPerNamespace: 10,
- PodsPerNode: 10,
-
- BigGroupSize: 25,
- MediumGroupSize: 10,
- SmallGroupSize: 5,
-
- SmallStatefulSetsPerNamespace: 0,
- MediumStatefulSetsPerNamespace: 0,
-
- CL2UseHostNetworkPods: false,
-
- // ref. https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/testing/load/kubemark/throughput_override.yaml
- CL2LoadTestThroughput: 20,
- CL2EnablePVS: false,
- CL2SchedulerThroughputThreshold: 100,
- PrometheusScrapeKubeProxy: false,
- EnableSystemPodMetrics: false,
- }
- if runtime.GOOS == "darwin" {
- cfg.ClusterLoaderDownloadURL = strings.Replace(cfg.ClusterLoaderDownloadURL, "linux", "darwin", -1)
- }
- return cfg
-}
-
-func (cfg *Config) GetAddOnClusterLoaderRemoteRepositoryRegion() string {
- if !cfg.IsEnabledAddOnClusterLoaderRemote() {
- return cfg.Region
- }
- return cfg.AddOnClusterLoaderRemote.RepositoryRegion
-}
-
-func (cfg *Config) validateAddOnClusterLoaderRemote() error {
- if !cfg.IsEnabledAddOnClusterLoaderRemote() {
- return nil
- }
-
- if cfg.AddOnClusterLoaderRemote.S3Dir == "" {
- cfg.AddOnClusterLoaderRemote.S3Dir = path.Join(cfg.Name, "add-on-cluster-loader-remote")
- }
-
- if cfg.AddOnClusterLoaderRemote.Namespace == "" {
- cfg.AddOnClusterLoaderRemote.Namespace = cfg.Name + "-cluster-loader-remote"
- }
-
- if cfg.AddOnClusterLoaderRemote.RepositoryAccountID == "" {
- return errors.New("AddOnClusterLoaderRemote.RepositoryAccountID empty")
- }
- if cfg.AddOnClusterLoaderRemote.RepositoryRegion == "" {
- cfg.AddOnClusterLoaderRemote.RepositoryRegion = cfg.Region
- }
- if cfg.AddOnClusterLoaderRemote.RepositoryName == "" {
- return errors.New("AddOnClusterLoaderRemote.RepositoryName empty")
- }
- if cfg.AddOnClusterLoaderRemote.RepositoryImageTag == "" {
- return errors.New("AddOnClusterLoaderRemote.RepositoryImageTag empty")
- }
-
- if cfg.AddOnClusterLoaderRemote.S3Dir == "" {
- cfg.AddOnClusterLoaderRemote.S3Dir = path.Join(cfg.Name, "add-on-cluster-loader-remote")
- }
-
- if cfg.AddOnClusterLoaderRemote.ReportTarGzPath == "" {
- cfg.AddOnClusterLoaderRemote.ReportTarGzPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-cluster-loader-remote.tar.gz"
- }
- if cfg.AddOnClusterLoaderRemote.ReportTarGzS3Key == "" {
- cfg.AddOnClusterLoaderRemote.ReportTarGzS3Key = path.Join(
- cfg.AddOnClusterLoaderRemote.S3Dir,
- filepath.Base(cfg.AddOnClusterLoaderRemote.ReportTarGzPath),
- )
- }
- if cfg.AddOnClusterLoaderRemote.LogPath == "" {
- cfg.AddOnClusterLoaderRemote.LogPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-cluster-loader-remote.log"
- }
- if cfg.AddOnClusterLoaderRemote.LogS3Key == "" {
- cfg.AddOnClusterLoaderRemote.LogS3Key = path.Join(
- cfg.AddOnClusterLoaderRemote.S3Dir,
- filepath.Base(cfg.AddOnClusterLoaderRemote.LogPath),
- )
- }
- if cfg.AddOnClusterLoaderRemote.PodStartupLatencyPath == "" {
- cfg.AddOnClusterLoaderRemote.PodStartupLatencyPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-cluster-loader-remote.log"
- }
- if cfg.AddOnClusterLoaderRemote.PodStartupLatencyS3Key == "" {
- cfg.AddOnClusterLoaderRemote.PodStartupLatencyS3Key = path.Join(
- cfg.AddOnClusterLoaderRemote.S3Dir,
- filepath.Base(cfg.AddOnClusterLoaderRemote.PodStartupLatencyPath),
- )
- }
-
- if cfg.AddOnClusterLoaderRemote.ClusterLoaderPath == "" && cfg.AddOnClusterLoaderRemote.ClusterLoaderDownloadURL == "" {
- return errors.New("empty AddOnClusterLoaderRemote.ClusterLoaderPath and ClusterLoaderDownloadURL")
- }
-
- if cfg.AddOnClusterLoaderRemote.Runs == 0 {
- return errors.New("unexpected zero AddOnClusterLoaderRemote.Runs")
- }
- if cfg.AddOnClusterLoaderRemote.Timeout == 0 {
- return errors.New("unexpected zero AddOnClusterLoaderRemote.Timeout")
- }
-
- if cfg.AddOnClusterLoaderRemote.Nodes == 0 {
- return errors.New("unexpected zero AddOnClusterLoaderRemote.Nodes")
- }
-
- if cfg.AddOnClusterLoaderRemote.CL2LoadTestThroughput == 0 {
- // ref. https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/testing/load/kubemark/throughput_override.yaml
- cfg.AddOnClusterLoaderRemote.CL2LoadTestThroughput = 20
- }
- if cfg.AddOnClusterLoaderRemote.PrometheusScrapeKubeProxy {
- return fmt.Errorf("unexpected AddOnClusterLoaderRemote.PrometheusScrapeKubeProxy %v; not supported yet", cfg.AddOnClusterLoaderRemote.PrometheusScrapeKubeProxy)
- }
- if cfg.AddOnClusterLoaderRemote.EnableSystemPodMetrics {
- return fmt.Errorf("unexpected AddOnClusterLoaderRemote.EnableSystemPodMetrics %v; not supported yet", cfg.AddOnClusterLoaderRemote.EnableSystemPodMetrics)
- }
- if cfg.AddOnClusterLoaderRemote.CL2SchedulerThroughputThreshold <= 0 {
- cfg.AddOnClusterLoaderRemote.CL2SchedulerThroughputThreshold = 100
- }
-
- return nil
-}
diff --git a/eksconfig/add-on-cluster-version-upgrade.go b/eksconfig/add-on-cluster-version-upgrade.go
deleted file mode 100644
index f81bb20ea..000000000
--- a/eksconfig/add-on-cluster-version-upgrade.go
+++ /dev/null
@@ -1,77 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
- "strconv"
- "time"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnClusterVersionUpgrade defines parameters
-// for EKS cluster version upgrade add-on.
-type AddOnClusterVersionUpgrade struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
-
- // WaitBeforeUpgrade is the wait duration before it starts cluster version upgrade.
- WaitBeforeUpgrade time.Duration `json:"wait-before-upgrade"`
- WaitBeforeUpgradeString string `json:"wait-before-upgrade-string" read-only:"true"`
-
- // Version is the target version of EKS Kubernetes "cluster".
- // If empty, set default version.
- Version string `json:"version"`
- VersionValue float64 `json:"version-value" read-only:"true"`
-}
-
-// EnvironmentVariablePrefixAddOnClusterVersionUpgrade is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnClusterVersionUpgrade = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_CLUSTER_VERSION_UPGRADE_"
-
-// IsEnabledAddOnClusterVersionUpgrade returns true if "AddOnClusterVersionUpgrade" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnClusterVersionUpgrade() bool {
- if cfg.AddOnClusterVersionUpgrade == nil {
- return false
- }
- if cfg.AddOnClusterVersionUpgrade.Enable {
- return true
- }
- cfg.AddOnClusterVersionUpgrade = nil
- return false
-}
-
-func getDefaultAddOnClusterVersionUpgrade() *AddOnClusterVersionUpgrade {
- return &AddOnClusterVersionUpgrade{
- Enable: false,
- Version: "1.20",
- WaitBeforeUpgrade: 3 * time.Minute,
- }
-}
-
-func (cfg *Config) validateAddOnClusterVersionUpgrade() error {
- if !cfg.IsEnabledAddOnClusterVersionUpgrade() {
- return nil
- }
-
- cfg.AddOnClusterVersionUpgrade.WaitBeforeUpgradeString = cfg.AddOnClusterVersionUpgrade.WaitBeforeUpgrade.String()
-
- if cfg.AddOnClusterVersionUpgrade.Version == "" {
- return errors.New("empty AddOnClusterVersionUpgrade.Version")
- }
- var err error
- cfg.AddOnClusterVersionUpgrade.VersionValue, err = strconv.ParseFloat(cfg.AddOnClusterVersionUpgrade.Version, 64)
- if err != nil {
- return fmt.Errorf("cannot parse AddOnClusterVersionUpgrade.Version %q (%v)", cfg.Version, err)
- }
- delta := cfg.AddOnClusterVersionUpgrade.VersionValue - cfg.VersionValue
- if fmt.Sprintf("%.2f", delta) != "0.01" {
- return fmt.Errorf("AddOnClusterVersionUpgrade only supports one minor version upgrade but got %.2f [invalid: %q -> %q]", delta, cfg.Version, cfg.AddOnClusterVersionUpgrade.Version)
- }
-
- return nil
-}
diff --git a/eksconfig/add-on-cni-vpc.go b/eksconfig/add-on-cni-vpc.go
deleted file mode 100644
index ed7a6a2f0..000000000
--- a/eksconfig/add-on-cni-vpc.go
+++ /dev/null
@@ -1,140 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnCNIVPC defines parameters for https://github.com/aws/amazon-vpc-cni-k8s.
-// If not enabled, uses the default one.
-// The version must be at least "v1.7".
-// ref. https://github.com/aws/amazon-vpc-cni-k8s/blob/release-1.7/config/v1.7/aws-k8s-cni.yaml
-type AddOnCNIVPC struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // Version defines the release version for https://github.com/aws/amazon-vpc-cni-k8s.
- // Must be at least "v1.7".
- // "v1.6" is not supported.
- // ref. https://github.com/aws/amazon-vpc-cni-k8s/releases
- Version string `json:"version"`
-
- // RepositoryInitAccountID is the account ID for tester ECR image.
- // e.g. "602401143452" for "602401143452.dkr.ecr.[REGION].amazonaws.com/amazon-k8s-cni-init"
- RepositoryInitAccountID string `json:"repository-init-account-id,omitempty"`
- // RepositoryInitRegion is the ECR repository region to pull from.
- RepositoryInitRegion string `json:"repository-init-region,omitempty"`
- // RepositoryInitName is the repositoryName for tester ECR image.
- // e.g. "amazon-k8s-cni-init" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/amazon-k8s-cni-init"
- RepositoryInitName string `json:"repository-init-name,omitempty"`
- // RepositoryInitImageTag is the image tag for tester ECR image.
- // e.g. "v1.7.0-rc1" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/amazon-k8s-cni-init:v1.7.0-rc1"
- RepositoryInitImageTag string `json:"repository-init-image-tag,omitempty"`
-
- // RepositoryAccountID is the account ID for tester ECR image.
- // e.g. "602401143452" for "602401143452.dkr.ecr.[REGION].amazonaws.com/amazon-k8s-cni"
- RepositoryAccountID string `json:"repository-account-id,omitempty"`
- // RepositoryRegion is the ECR repository region to pull from.
- RepositoryRegion string `json:"repository-region,omitempty"`
- // RepositoryName is the repositoryName for tester ECR image.
- // e.g. "amazon-k8s-cni" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/amazon-k8s-cni"
- RepositoryName string `json:"repository-name,omitempty"`
- // RepositoryImageTag is the image tag for tester ECR image.
- // e.g. "v1.7.0-rc1" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/amazon-k8s-cni:v1.7.0-rc1"
- RepositoryImageTag string `json:"repository-image-tag,omitempty"`
-
- // MinimumIPTarget configures "MINIMUM_IP_TARGET" for VPC CNI plugin daemon set.
- // ref. https://github.com/aws/amazon-vpc-cni-k8s#cni-configuration-variables
- MinimumIPTarget int `json:"minimum-ip-target"`
- // WarmIPTarget configures "WARM_IP_TARGET" for VPC CNI plugin daemon set.
- // ref. https://github.com/aws/amazon-vpc-cni-k8s#cni-configuration-variables
- WarmIPTarget int `json:"warm-ip-target"`
-
- // NodeSelector is configured to overwrite existing node selector
- // for amazon-vpc-cni-k8s DaemonSet.
- NodeSelector map[string]string `json:"node-selector"`
-}
-
-// AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_PREFIX is the environment variable prefix used for "eksconfig".
-const AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_PREFIX = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_CNI_VPC_"
-
-// IsEnabledAddOnCNIVPC returns true if "AddOnCNIVPC" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnCNIVPC() bool {
- if cfg.AddOnCNIVPC == nil {
- return false
- }
- if cfg.AddOnCNIVPC.Enable {
- return true
- }
- cfg.AddOnCNIVPC = nil
- return false
-}
-
-func getDefaultAddOnCNIVPC() *AddOnCNIVPC {
- return &AddOnCNIVPC{
- Enable: false,
-
- // https://github.com/aws/amazon-vpc-cni-k8s/releases
- Version: "v1.7",
-
- NodeSelector: map[string]string{
- // do not deploy in fake nodes, obviously
- "NodeType": "regular",
- },
- }
-}
-
-func (cfg *Config) GetAddOnCNIVPCRepositoryRegion() string {
- if !cfg.IsEnabledAddOnCNIVPC() {
- return cfg.Region
- }
- return cfg.AddOnCNIVPC.RepositoryRegion
-}
-
-func (cfg *Config) validateAddOnCNIVPC() error {
- if !cfg.IsEnabledAddOnCNIVPC() {
- return nil
- }
-
- switch cfg.AddOnCNIVPC.Version {
- case "v1.7":
- default:
- return fmt.Errorf("unknown AddOnCNIVPC.Version %q", cfg.AddOnCNIVPC.Version)
- }
-
- if cfg.AddOnCNIVPC.RepositoryInitAccountID == "" {
- return errors.New("AddOnCNIVPC.RepositoryInitAccountID empty")
- }
- if cfg.AddOnCNIVPC.RepositoryInitRegion == "" {
- cfg.AddOnCNIVPC.RepositoryInitRegion = cfg.Region
- }
- if cfg.AddOnCNIVPC.RepositoryInitName == "" {
- return errors.New("AddOnCNIVPC.RepositoryInitName empty")
- }
- if cfg.AddOnCNIVPC.RepositoryInitImageTag == "" {
- return errors.New("AddOnCNIVPC.RepositoryInitImageTag empty")
- }
-
- if cfg.AddOnCNIVPC.RepositoryAccountID == "" {
- return errors.New("AddOnCNIVPC.RepositoryAccountID empty")
- }
- if cfg.AddOnCNIVPC.RepositoryRegion == "" {
- cfg.AddOnCNIVPC.RepositoryRegion = cfg.Region
- }
- if cfg.AddOnCNIVPC.RepositoryName == "" {
- return errors.New("AddOnCNIVPC.RepositoryName empty")
- }
- if cfg.AddOnCNIVPC.RepositoryImageTag == "" {
- return errors.New("AddOnCNIVPC.RepositoryImageTag empty")
- }
-
- return nil
-}
diff --git a/eksconfig/add-on-configmaps-local.go b/eksconfig/add-on-configmaps-local.go
deleted file mode 100644
index afea5eeb2..000000000
--- a/eksconfig/add-on-configmaps-local.go
+++ /dev/null
@@ -1,229 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
- "path"
- "path/filepath"
- "strings"
-
- "github.com/aws/aws-k8s-tester/pkg/metrics"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnConfigmapsLocal defines parameters for EKS cluster
-// add-on "ConfigMap" local.
-// It generates loads from the local host machine.
-// Every object is written serially with no concurrency.
-// Use remote tester to write with concurrency.
-// The main use case is to write large objects to fill up etcd database.
-type AddOnConfigmapsLocal struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // S3Dir is the S3 directory to store all test results.
- // It is under the bucket "eksconfig.Config.S3BucketName".
- S3Dir string `json:"s3-dir"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // Objects is the number of "ConfigMap" objects to create.
- Objects int `json:"objects"`
- // ObjectSize is the "ConfigMap" value size in bytes.
- ObjectSize int `json:"object-size"`
-
- // CreatedNames is the list of created "ConfigMap" object names.
- CreatedNames []string `json:"created-names" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- RequestsRawWritesJSONPath string `json:"requests-raw-writes-json-path" read-only:"true"`
- RequestsRawWritesJSONS3Key string `json:"requests-raw-writes-json-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsRawWritesCompareS3Dir is the s3 directory to store raw data points.
- // Used to comparison results.
- // ref. https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
- RequestsRawWritesCompareS3Dir string `json:"requests-raw-writes-compare-s3-dir"`
- RequestsRawWritesCompareAllJSONPath string `json:"requests-raw-writes-compare-all-json-path" read-only:"true"`
- RequestsRawWritesCompareAllJSONS3Key string `json:"requests-raw-writes-compare-all-json-s3-key" read-only:"true"`
- RequestsRawWritesCompareAllCSVPath string `json:"requests-raw-writes-compare-all-csv-path" read-only:"true"`
- RequestsRawWritesCompareAllCSVS3Key string `json:"requests-raw-writes-compare-all-csv-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWrites is the writes results.
- RequestsSummaryWrites metrics.RequestsSummary `json:"requests-summary-writes,omitempty" read-only:"true"`
- RequestsSummaryWritesJSONPath string `json:"requests-summary-writes-json-path" read-only:"true"`
- RequestsSummaryWritesJSONS3Key string `json:"requests-summary-writes-json-s3-key" read-only:"true"`
- RequestsSummaryWritesTablePath string `json:"requests-summary-writes-table-path" read-only:"true"`
- RequestsSummaryWritesTableS3Key string `json:"requests-summary-writes-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWritesCompareS3Dir is the S3 directory of previous/latest "RequestsSummary".
- // Specify the S3 key in the same bucket of "eksconfig.Config.S3BucketName".
- // Use for regression tests. Specify the value not bound to the cluster directory.
- // Different runs from different clusters reads and writes in this directory.
- RequestsSummaryWritesCompareS3Dir string `json:"requests-summary-writes-compare-s3-dir"`
- RequestsSummaryWritesCompare metrics.RequestsCompare `json:"requests-summary-writes-compare" read-only:"true"`
- RequestsSummaryWritesCompareJSONPath string `json:"requests-summary-writes-compare-json-path" read-only:"true"`
- RequestsSummaryWritesCompareJSONS3Key string `json:"requests-summary-writes-compare-json-s3-key" read-only:"true"`
- RequestsSummaryWritesCompareTablePath string `json:"requests-summary-writes-compare-table-path" read-only:"true"`
- RequestsSummaryWritesCompareTableS3Key string `json:"requests-summary-writes-compare-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-}
-
-// EnvironmentVariablePrefixAddOnConfigmapsLocal is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnConfigmapsLocal = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_CONFIGMAPS_LOCAL_"
-
-// IsEnabledAddOnConfigmapsLocal returns true if "AddOnConfigmapsLocal" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnConfigmapsLocal() bool {
- if cfg.AddOnConfigmapsLocal == nil {
- return false
- }
- if cfg.AddOnConfigmapsLocal.Enable {
- return true
- }
- cfg.AddOnConfigmapsLocal = nil
- return false
-}
-
-func getDefaultAddOnConfigmapsLocal() *AddOnConfigmapsLocal {
- return &AddOnConfigmapsLocal{
- Enable: false,
- Objects: 10,
- ObjectSize: 10 * 1024, // 10 KB
-
- // writes total 300 MB data to etcd
- // Objects: 1000,
- // ObjectSize: 300000, // 0.3 MB
- }
-}
-
-func (cfg *Config) validateAddOnConfigmapsLocal() error {
- if !cfg.IsEnabledAddOnConfigmapsLocal() {
- return nil
- }
-
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnConfigmapsLocal.Enable true but no node group is enabled")
- }
-
- if cfg.AddOnConfigmapsLocal.S3Dir == "" {
- cfg.AddOnConfigmapsLocal.S3Dir = path.Join(cfg.Name, "add-on-configmaps-local")
- }
-
- if cfg.AddOnConfigmapsLocal.Namespace == "" {
- cfg.AddOnConfigmapsLocal.Namespace = cfg.Name + "-configmaps-local"
- }
-
- if cfg.AddOnConfigmapsLocal.Objects == 0 {
- cfg.AddOnConfigmapsLocal.Objects = 10
- }
- if cfg.AddOnConfigmapsLocal.ObjectSize == 0 {
- cfg.AddOnConfigmapsLocal.ObjectSize = 10 * 1024
- }
- if cfg.AddOnConfigmapsLocal.ObjectSize > 900000 {
- return fmt.Errorf("AddOnConfigmapsLocal.ObjectSize limit is 0.9 MB, got %d", cfg.AddOnConfigmapsLocal.ObjectSize)
- }
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnConfigmapsLocal.RequestsRawWritesJSONPath == "" {
- cfg.AddOnConfigmapsLocal.RequestsRawWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-configmaps-local-requests-writes-raw.json"
- }
- if cfg.AddOnConfigmapsLocal.RequestsRawWritesJSONS3Key == "" {
- cfg.AddOnConfigmapsLocal.RequestsRawWritesJSONS3Key = path.Join(
- cfg.AddOnConfigmapsLocal.S3Dir,
- "requests-raw-writes",
- filepath.Base(cfg.AddOnConfigmapsLocal.RequestsRawWritesJSONPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnConfigmapsLocal.RequestsRawWritesCompareS3Dir == "" {
- cfg.AddOnConfigmapsLocal.RequestsRawWritesCompareS3Dir = path.Join("add-on-configmaps-local", "requests-raw-writes-compare", cfg.Version)
- }
- if cfg.AddOnConfigmapsLocal.RequestsRawWritesCompareAllJSONPath == "" {
- cfg.AddOnConfigmapsLocal.RequestsRawWritesCompareAllJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-configmaps-local-requests-raw-writes-compare-all.json"
- }
- if cfg.AddOnConfigmapsLocal.RequestsRawWritesCompareAllJSONS3Key == "" {
- cfg.AddOnConfigmapsLocal.RequestsRawWritesCompareAllJSONS3Key = path.Join(
- cfg.AddOnConfigmapsLocal.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnConfigmapsLocal.RequestsRawWritesCompareAllJSONPath),
- )
- }
- if cfg.AddOnConfigmapsLocal.RequestsRawWritesCompareAllCSVPath == "" {
- cfg.AddOnConfigmapsLocal.RequestsRawWritesCompareAllCSVPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-configmaps-local-requests-raw-writes-compare-all.csv"
- }
- if cfg.AddOnConfigmapsLocal.RequestsRawWritesCompareAllCSVS3Key == "" {
- cfg.AddOnConfigmapsLocal.RequestsRawWritesCompareAllCSVS3Key = path.Join(
- cfg.AddOnConfigmapsLocal.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnConfigmapsLocal.RequestsRawWritesCompareAllCSVPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnConfigmapsLocal.RequestsSummaryWritesJSONPath == "" {
- cfg.AddOnConfigmapsLocal.RequestsSummaryWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-configmaps-local-requests-summary-writes.json"
- }
- if cfg.AddOnConfigmapsLocal.RequestsSummaryWritesJSONS3Key == "" {
- cfg.AddOnConfigmapsLocal.RequestsSummaryWritesJSONS3Key = path.Join(
- cfg.AddOnConfigmapsLocal.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnConfigmapsLocal.RequestsSummaryWritesJSONPath),
- )
- }
- if cfg.AddOnConfigmapsLocal.RequestsSummaryWritesTablePath == "" {
- cfg.AddOnConfigmapsLocal.RequestsSummaryWritesTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-configmaps-local-requests-summary-writes.txt"
- }
- if cfg.AddOnConfigmapsLocal.RequestsSummaryWritesTableS3Key == "" {
- cfg.AddOnConfigmapsLocal.RequestsSummaryWritesTableS3Key = path.Join(
- cfg.AddOnConfigmapsLocal.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnConfigmapsLocal.RequestsSummaryWritesTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnConfigmapsLocal.RequestsSummaryWritesCompareS3Dir == "" {
- cfg.AddOnConfigmapsLocal.RequestsSummaryWritesCompareS3Dir = path.Join("add-on-configmaps-local", "requests-summary-writes-compare", cfg.Version)
- }
- if cfg.AddOnConfigmapsLocal.RequestsSummaryWritesCompareJSONPath == "" {
- cfg.AddOnConfigmapsLocal.RequestsSummaryWritesCompareJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-configmaps-local-requests-summary-writes-compare.json"
- }
- if cfg.AddOnConfigmapsLocal.RequestsSummaryWritesCompareJSONS3Key == "" {
- cfg.AddOnConfigmapsLocal.RequestsSummaryWritesCompareJSONS3Key = path.Join(
- cfg.AddOnConfigmapsLocal.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnConfigmapsLocal.RequestsSummaryWritesCompareJSONPath),
- )
- }
- if cfg.AddOnConfigmapsLocal.RequestsSummaryWritesCompareTablePath == "" {
- cfg.AddOnConfigmapsLocal.RequestsSummaryWritesCompareTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-configmaps-local-requests-summary-writes-compare.txt"
- }
- if cfg.AddOnConfigmapsLocal.RequestsSummaryWritesCompareTableS3Key == "" {
- cfg.AddOnConfigmapsLocal.RequestsSummaryWritesCompareTableS3Key = path.Join(
- cfg.AddOnConfigmapsLocal.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnConfigmapsLocal.RequestsSummaryWritesCompareTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- return nil
-}
diff --git a/eksconfig/add-on-configmaps-remote.go b/eksconfig/add-on-configmaps-remote.go
deleted file mode 100644
index eb776f654..000000000
--- a/eksconfig/add-on-configmaps-remote.go
+++ /dev/null
@@ -1,285 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
- "path"
- "path/filepath"
- "strings"
-
- "github.com/aws/aws-k8s-tester/pkg/metrics"
- "github.com/aws/aws-k8s-tester/pkg/randutil"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnConfigmapsRemote defines parameters for EKS cluster
-// add-on "ConfigMap" remote.
-// It generates loads from the remote workers (Pod) in the cluster.
-// Each worker writes serially with no concurrency.
-// Configure "DeploymentReplicas" accordingly to increase the concurrency.
-// The main use case is to write large objects to fill up etcd database.
-type AddOnConfigmapsRemote struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // S3Dir is the S3 directory to store all test results.
- // It is under the bucket "eksconfig.Config.S3BucketName".
- S3Dir string `json:"s3-dir"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // RepositoryAccountID is the account ID for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryAccountID string `json:"repository-account-id,omitempty"`
- // RepositoryRegion is the ECR repository region to pull from.
- RepositoryRegion string `json:"repository-region,omitempty"`
- // RepositoryName is the repositoryName for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryName string `json:"repository-name,omitempty"`
- // RepositoryImageTag is the image tag for tester ECR image.
- // e.g. "latest" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester:latest"
- RepositoryImageTag string `json:"repository-image-tag,omitempty"`
-
- // Completes is the desired number of successfully finished pods.
- // Write QPS will be client QPS * replicas.
- // Read QPS will be client QPS * replicas.
- // The total number of objects to be created is "Completes" * "Objects".
- Completes int `json:"completes"`
- // Parallels is the the maximum desired number of pods the
- // job should run at any given time.
- // Write QPS will be client QPS * replicas.
- // Read QPS will be client QPS * replicas.
- // The total number of objects to be created is "Completes" * "Objects".
- Parallels int `json:"parallels"`
-
- // Objects is the number of "ConfigMap" objects to create.
- Objects int `json:"objects"`
- // ObjectSize is the "ConfigMap" value size in bytes.
- ObjectSize int `json:"object-size"`
-
- // CreatedNames is the list of created "ConfigMap" object names.
- CreatedNames []string `json:"created-names" read-only:"true"`
-
- // RequestsSummaryWritesOutputNamePrefix is the output path name in "/var/log" directory, used in remote worker.
- RequestsSummaryWritesOutputNamePrefix string `json:"requests-summary-writes-output-name-prefix"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- RequestsRawWritesJSONPath string `json:"requests-raw-writes-json-path" read-only:"true"`
- RequestsRawWritesJSONS3Key string `json:"requests-raw-writes-json-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsRawWritesCompareS3Dir is the s3 directory to store raw data points.
- // Used to comparison results.
- // ref. https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
- RequestsRawWritesCompareS3Dir string `json:"requests-raw-writes-compare-s3-dir"`
- RequestsRawWritesCompareAllJSONPath string `json:"requests-raw-writes-compare-all-json-path" read-only:"true"`
- RequestsRawWritesCompareAllJSONS3Key string `json:"requests-raw-writes-compare-all-json-s3-key" read-only:"true"`
- RequestsRawWritesCompareAllCSVPath string `json:"requests-raw-writes-compare-all-csv-path" read-only:"true"`
- RequestsRawWritesCompareAllCSVS3Key string `json:"requests-raw-writes-compare-all-csv-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWrites is the writes results.
- RequestsSummaryWrites metrics.RequestsSummary `json:"requests-summary-writes,omitempty" read-only:"true"`
- RequestsSummaryWritesJSONPath string `json:"requests-summary-writes-json-path" read-only:"true"`
- RequestsSummaryWritesJSONS3Key string `json:"requests-summary-writes-json-s3-key" read-only:"true"`
- RequestsSummaryWritesTablePath string `json:"requests-summary-writes-table-path" read-only:"true"`
- RequestsSummaryWritesTableS3Key string `json:"requests-summary-writes-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWritesCompareS3Dir is the S3 directory of previous/latest "RequestsSummary".
- // Specify the S3 key in the same bucket of "eksconfig.Config.S3BucketName".
- // Use for regression tests. Specify the value not bound to the cluster directory.
- // Different runs from different clusters reads and writes in this directory.
- RequestsSummaryWritesCompareS3Dir string `json:"requests-summary-writes-compare-s3-dir"`
- RequestsSummaryWritesCompare metrics.RequestsCompare `json:"requests-summary-writes-compare" read-only:"true"`
- RequestsSummaryWritesCompareJSONPath string `json:"requests-summary-writes-compare-json-path" read-only:"true"`
- RequestsSummaryWritesCompareJSONS3Key string `json:"requests-summary-writes-compare-json-s3-key" read-only:"true"`
- RequestsSummaryWritesCompareTablePath string `json:"requests-summary-writes-compare-table-path" read-only:"true"`
- RequestsSummaryWritesCompareTableS3Key string `json:"requests-summary-writes-compare-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-}
-
-// EnvironmentVariablePrefixAddOnConfigmapsRemote is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnConfigmapsRemote = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_CONFIGMAPS_REMOTE_"
-
-// IsEnabledAddOnConfigmapsRemote returns true if "AddOnConfigmapsRemote" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnConfigmapsRemote() bool {
- if cfg.AddOnConfigmapsRemote == nil {
- return false
- }
- if cfg.AddOnConfigmapsRemote.Enable {
- return true
- }
- cfg.AddOnConfigmapsRemote = nil
- return false
-}
-
-func getDefaultAddOnConfigmapsRemote() *AddOnConfigmapsRemote {
- return &AddOnConfigmapsRemote{
- Enable: false,
- Completes: 5,
- Parallels: 5,
- Objects: 10,
- ObjectSize: 10 * 1024, // 10 KB
-
- // writes total 300 MB data to etcd
- // Objects: 1000,
- // ObjectSize: 300000, // 0.3 MB
-
- RequestsSummaryWritesOutputNamePrefix: "configmaps-writes-" + randutil.String(10),
- }
-}
-
-func (cfg *Config) GetAddOnConfigmapsRemoteRepositoryRegion() string {
- if !cfg.IsEnabledAddOnConfigmapsRemote() {
- return cfg.Region
- }
- return cfg.AddOnConfigmapsRemote.RepositoryRegion
-}
-
-func (cfg *Config) validateAddOnConfigmapsRemote() error {
- if !cfg.IsEnabledAddOnConfigmapsRemote() {
- return nil
- }
-
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnConfigmapsRemote.Enable true but no node group is enabled")
- }
-
- if cfg.AddOnConfigmapsRemote.S3Dir == "" {
- cfg.AddOnConfigmapsRemote.S3Dir = path.Join(cfg.Name, "add-on-configmaps-remote")
- }
-
- if cfg.AddOnConfigmapsRemote.Namespace == "" {
- cfg.AddOnConfigmapsRemote.Namespace = cfg.Name + "-configmaps-remote"
- }
-
- if cfg.AddOnConfigmapsRemote.RepositoryAccountID == "" {
- return errors.New("AddOnConfigmapsRemote.RepositoryAccountID empty")
- }
- if cfg.AddOnConfigmapsRemote.RepositoryRegion == "" {
- cfg.AddOnConfigmapsRemote.RepositoryRegion = cfg.Region
- }
- if cfg.AddOnConfigmapsRemote.RepositoryName == "" {
- return errors.New("AddOnConfigmapsRemote.RepositoryName empty")
- }
- if cfg.AddOnConfigmapsRemote.RepositoryImageTag == "" {
- return errors.New("AddOnConfigmapsRemote.RepositoryImageTag empty")
- }
-
- if cfg.AddOnConfigmapsRemote.Objects == 0 {
- cfg.AddOnConfigmapsRemote.Objects = 10
- }
- if cfg.AddOnConfigmapsRemote.ObjectSize == 0 {
- cfg.AddOnConfigmapsRemote.ObjectSize = 10 * 1024
- }
- if cfg.AddOnConfigmapsRemote.ObjectSize > 900000 {
- return fmt.Errorf("AddOnConfigmapsRemote.ObjectSize limit is 0.9 MB, got %d", cfg.AddOnConfigmapsRemote.ObjectSize)
- }
-
- if cfg.AddOnConfigmapsRemote.RequestsSummaryWritesOutputNamePrefix == "" {
- cfg.AddOnConfigmapsRemote.RequestsSummaryWritesOutputNamePrefix = "configmaps-writes-" + randutil.String(10)
- }
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnConfigmapsRemote.RequestsRawWritesJSONPath == "" {
- cfg.AddOnConfigmapsRemote.RequestsRawWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-configmaps-remote-requests-writes-raw.json"
- }
- if cfg.AddOnConfigmapsRemote.RequestsRawWritesJSONS3Key == "" {
- cfg.AddOnConfigmapsRemote.RequestsRawWritesJSONS3Key = path.Join(
- cfg.AddOnConfigmapsRemote.S3Dir,
- "requests-raw-writes",
- filepath.Base(cfg.AddOnConfigmapsRemote.RequestsRawWritesJSONPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnConfigmapsRemote.RequestsRawWritesCompareS3Dir == "" {
- cfg.AddOnConfigmapsRemote.RequestsRawWritesCompareS3Dir = path.Join("add-on-configmaps-remote", "requests-raw-writes-compare", cfg.Version)
- }
- if cfg.AddOnConfigmapsRemote.RequestsRawWritesCompareAllJSONPath == "" {
- cfg.AddOnConfigmapsRemote.RequestsRawWritesCompareAllJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-configmaps-remote-requests-raw-writes-compare-all.json"
- }
- if cfg.AddOnConfigmapsRemote.RequestsRawWritesCompareAllJSONS3Key == "" {
- cfg.AddOnConfigmapsRemote.RequestsRawWritesCompareAllJSONS3Key = path.Join(
- cfg.AddOnConfigmapsRemote.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnConfigmapsRemote.RequestsRawWritesCompareAllJSONPath),
- )
- }
- if cfg.AddOnConfigmapsRemote.RequestsRawWritesCompareAllCSVPath == "" {
- cfg.AddOnConfigmapsRemote.RequestsRawWritesCompareAllCSVPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-configmaps-remote-requests-raw-writes-compare-all.csv"
- }
- if cfg.AddOnConfigmapsRemote.RequestsRawWritesCompareAllCSVS3Key == "" {
- cfg.AddOnConfigmapsRemote.RequestsRawWritesCompareAllCSVS3Key = path.Join(
- cfg.AddOnConfigmapsRemote.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnConfigmapsRemote.RequestsRawWritesCompareAllCSVPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnConfigmapsRemote.RequestsSummaryWritesJSONPath == "" {
- cfg.AddOnConfigmapsRemote.RequestsSummaryWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-configmaps-remote-requests-summary-writes.json"
- }
- if cfg.AddOnConfigmapsRemote.RequestsSummaryWritesJSONS3Key == "" {
- cfg.AddOnConfigmapsRemote.RequestsSummaryWritesJSONS3Key = path.Join(
- cfg.AddOnConfigmapsRemote.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnConfigmapsRemote.RequestsSummaryWritesJSONPath),
- )
- }
- if cfg.AddOnConfigmapsRemote.RequestsSummaryWritesTablePath == "" {
- cfg.AddOnConfigmapsRemote.RequestsSummaryWritesTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-configmaps-remote-requests-summary-writes.txt"
- }
- if cfg.AddOnConfigmapsRemote.RequestsSummaryWritesTableS3Key == "" {
- cfg.AddOnConfigmapsRemote.RequestsSummaryWritesTableS3Key = path.Join(
- cfg.AddOnConfigmapsRemote.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnConfigmapsRemote.RequestsSummaryWritesTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnConfigmapsRemote.RequestsSummaryWritesCompareS3Dir == "" {
- cfg.AddOnConfigmapsRemote.RequestsSummaryWritesCompareS3Dir = path.Join("add-on-configmaps-remote", "requests-summary-writes-compare", cfg.Version)
- }
- if cfg.AddOnConfigmapsRemote.RequestsSummaryWritesCompareJSONPath == "" {
- cfg.AddOnConfigmapsRemote.RequestsSummaryWritesCompareJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-configmaps-remote-requests-summary-writes-compare.json"
- }
- if cfg.AddOnConfigmapsRemote.RequestsSummaryWritesCompareJSONS3Key == "" {
- cfg.AddOnConfigmapsRemote.RequestsSummaryWritesCompareJSONS3Key = path.Join(
- cfg.AddOnConfigmapsRemote.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnConfigmapsRemote.RequestsSummaryWritesCompareJSONPath),
- )
- }
- if cfg.AddOnConfigmapsRemote.RequestsSummaryWritesCompareTablePath == "" {
- cfg.AddOnConfigmapsRemote.RequestsSummaryWritesCompareTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-configmaps-remote-requests-summary-writes-compare.txt"
- }
- if cfg.AddOnConfigmapsRemote.RequestsSummaryWritesCompareTableS3Key == "" {
- cfg.AddOnConfigmapsRemote.RequestsSummaryWritesCompareTableS3Key = path.Join(
- cfg.AddOnConfigmapsRemote.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnConfigmapsRemote.RequestsSummaryWritesCompareTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- return nil
-}
diff --git a/eksconfig/add-on-conformance.go b/eksconfig/add-on-conformance.go
deleted file mode 100644
index ea31e6fcf..000000000
--- a/eksconfig/add-on-conformance.go
+++ /dev/null
@@ -1,237 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
- "os"
- "path"
- "path/filepath"
- "runtime"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnConformance defines parameters for EKS cluster
-// add-on Conformance.
-// ref. https://github.com/cncf/k8s-conformance/blob/master/instructions.md
-// ref. https://github.com/vmware-tanzu/sonobuoy
-type AddOnConformance struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // S3Dir is the S3 directory to store all test results.
- // It is under the bucket "eksconfig.Config.S3BucketName".
- S3Dir string `json:"s3-dir"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // SonobuoyPath is the path to download the "sonobuoy".
- SonobuoyPath string `json:"sonobuoy-path,omitempty"`
- // SonobuoyDownloadURL is the download URL to download "sonobuoy" binary from.
- // ref. https://github.com/vmware-tanzu/sonobuoy/releases
- SonobuoyDownloadURL string `json:"sonobuoy-download-url,omitempty"`
- // SonobuoyE2eRepoConfig File path to e2e registry config
- // ref. https://sonobuoy.io/docs/master/airgap/
- SonobuoyE2eRepoConfig string `json:"sonobuoy-e2e-repo-config"`
- // SonobuoyImage Container override for the sonobuoy worker image
- SonobuoyImage string `json:"sonobuoy-image"`
- // SystemdLogsImage Container override for systemd-logs plugin image
- SystemdLogsImage string `json:"systemd-logs-image"`
-
- SonobuoyDeleteTimeout time.Duration `json:"sonobuoy-delete-timeout"`
- SonobuoyDeleteTimeoutString string `json:"sonobuoy-delete-timeout-string" read-only:"true"`
- SonobuoyRunTimeout time.Duration `json:"sonobuoy-run-timeout"`
- SonobuoyRunTimeoutString string `json:"sonobuoy-run-timeout-string" read-only:"true"`
-
- // SonobuoyRunMode is the mode to run sonobuoy in.
- // Valid modes are 'non-disruptive-conformance', 'quick', 'certified-conformance'.
- // The default is 'certified-conformance'.
- // ref. https://github.com/vmware-tanzu/sonobuoy
- SonobuoyRunMode string `json:"sonobuoy-run-mode"`
- SonobuoyRunKubeConformanceImage string `json:"sonobuoy-run-kube-conformance-image"`
-
- SonobuoyRunE2eFocus string `json:"sonobuoy-run-e2e-focus"`
- SonobuoyRunE2eSkip string `json:"sonobuoy-run-e2e-skip"`
-
- SonobuoyResultTarGzPath string `json:"sonobuoy-result-tar-gz-path" read-only:"true"`
- SonobuoyResultTarGzS3Key string `json:"sonobuoy-result-tar-gz-s3-key" read-only:"true"`
- SonobuoyResultDir string `json:"sonobuoy-result-dir" read-only:"true"`
- SonobuoyResultE2eLogPath string `json:"sonobuoy-result-e2e-log-path" read-only:"true"`
- SonobuoyResultE2eLogS3Key string `json:"sonobuoy-result-e2e-log-s3-key" read-only:"true"`
- SonobuoyResultJunitXMLPath string `json:"sonobuoy-result-junit-xml-path" read-only:"true"`
- SonobuoyResultJunitXMLS3Key string `json:"sonobuoy-result-junit-xml-s3-key" read-only:"true"`
-}
-
-// EnvironmentVariablePrefixAddOnConformance is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnConformance = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_CONFORMANCE_"
-
-// IsEnabledAddOnConformance returns true if "AddOnConformance" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnConformance() bool {
- if cfg.AddOnConformance == nil {
- return false
- }
- if cfg.AddOnConformance.Enable {
- return true
- }
- cfg.AddOnConformance = nil
- return false
-}
-
-func getDefaultAddOnConformance() *AddOnConformance {
- addOn := &AddOnConformance{
- Enable: false,
- SonobuoyPath: "/tmp/sonobuoy",
- SonobuoyDownloadURL: "https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.56.16/sonobuoy_0.56.16_linux_amd64.tar.gz",
- SonobuoyImage: "",
- SystemdLogsImage: "",
- SonobuoyE2eRepoConfig: "",
- }
- if runtime.GOOS == "darwin" {
- addOn.SonobuoyDownloadURL = strings.Replace(addOn.SonobuoyDownloadURL, "linux", "darwin", -1)
- }
- return addOn
-}
-
-/*
-TODO: fix this... conformance with managed node group does not work
-
-Plugin: e2e
-Status: failed
-Total: 4897
-Passed: 267
-Failed: 9
-Skipped: 4621
-
-Failed tests:
-[sig-network] Services should be able to change the type from ClusterIP to ExternalName [Conformance]
-[sig-network] Services should be able to create a functioning NodePort service [Conformance]
-[sig-api-machinery] Aggregator Should be able to support the 1.10 Sample API Server using the current Aggregator [Conformance]
-[sig-network] Networking Granular Checks: Pods should function for intra-pod communication: udp [LinuxOnly] [NodeConformance] [Conformance]
-[sig-cli] Kubectl client Kubectl run --rm job should create a job from an image, then delete the job [Conformance]
-[sig-network] Services should be able to change the type from ExternalName to NodePort [Conformance]
-[sig-network] DNS should provide DNS for ExternalName services [Conformance]
-[sig-network] Networking Granular Checks: Pods should function for node-pod communication: udp [LinuxOnly] [NodeConformance] [Conformance]
-[sig-network] DNS should provide DNS for pods for Hostname [LinuxOnly] [Conformance]
-
-Plugin: systemd-logs
-Status: passed
-Total: 14
-Passed: 14
-Failed: 0
-Skipped: 0
-*/
-
-func (cfg *Config) validateAddOnConformance() error {
- if !cfg.IsEnabledAddOnConformance() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnConformance.Enable true but no node group is enabled")
- }
-
- // TODO: fix this...
- if cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnConformance.Enable true with AddOnManagedNodeGroups.Enable true")
- }
-
- if cfg.AddOnConformance.S3Dir == "" {
- cfg.AddOnConformance.S3Dir = path.Join(cfg.Name, "add-on-conformance")
- }
-
- if cfg.AddOnConformance.Namespace == "" {
- cfg.AddOnConformance.Namespace = cfg.Name + "-conformance"
- }
-
- if cfg.AddOnConformance.SonobuoyDeleteTimeout == time.Duration(0) {
- cfg.AddOnConformance.SonobuoyDeleteTimeout = 5 * time.Minute
- }
- cfg.AddOnConformance.SonobuoyDeleteTimeoutString = cfg.AddOnConformance.SonobuoyDeleteTimeout.String()
-
- // "certified-conformance" takes >=3-hour
- if cfg.AddOnConformance.SonobuoyRunTimeout == time.Duration(0) {
- cfg.AddOnConformance.SonobuoyRunTimeout = 5 * time.Hour
- }
- cfg.AddOnConformance.SonobuoyRunTimeoutString = cfg.AddOnConformance.SonobuoyRunTimeout.String()
-
- if cfg.AddOnConformance.SonobuoyRunMode == "" {
- cfg.AddOnConformance.SonobuoyRunMode = "certified-conformance"
- }
- switch cfg.AddOnConformance.SonobuoyRunMode {
- case "non-disruptive-conformance":
- case "quick":
- case "certified-conformance":
- default:
- return fmt.Errorf("unknown AddOnConformance.SonobuoyRunMode %q", cfg.AddOnConformance.SonobuoyRunMode)
- }
-
- if cfg.AddOnConformance.SonobuoyRunKubeConformanceImage == "" {
- cfg.AddOnConformance.SonobuoyRunKubeConformanceImage = fmt.Sprintf("k8s.gcr.io/conformance:v%s.0", cfg.Version)
- }
-
- cfg.AddOnConformance.SonobuoyResultDir = filepath.Join(
- filepath.Dir(cfg.ConfigPath),
- fmt.Sprintf("%s-sonobuoy-results", cfg.Name),
- )
-
- if cfg.AddOnConformance.SonobuoyResultE2eLogPath == "" {
- cfg.AddOnConformance.SonobuoyResultE2eLogPath = filepath.Join(
- filepath.Dir(cfg.ConfigPath),
- fmt.Sprintf("%s-sonobuoy-result.e2e.log", cfg.Name),
- )
- os.RemoveAll(cfg.AddOnConformance.SonobuoyResultE2eLogPath)
- }
- if !strings.HasSuffix(cfg.AddOnConformance.SonobuoyResultE2eLogPath, ".log") {
- return fmt.Errorf("AddOnConformance.SonobuoyResultE2eLogPath[%q] must have '.log' extension", cfg.AddOnConformance.SonobuoyResultTarGzPath)
- }
- if cfg.AddOnConformance.SonobuoyResultE2eLogS3Key == "" {
- cfg.AddOnConformance.SonobuoyResultE2eLogS3Key = path.Join(
- cfg.AddOnConformance.S3Dir,
- filepath.Base(cfg.AddOnConformance.SonobuoyResultE2eLogPath),
- )
- }
-
- if cfg.AddOnConformance.SonobuoyResultTarGzPath == "" {
- cfg.AddOnConformance.SonobuoyResultTarGzPath = filepath.Join(
- filepath.Dir(cfg.ConfigPath),
- fmt.Sprintf("%s-sonobuoy-result.tar.gz", cfg.Name),
- )
- os.RemoveAll(cfg.AddOnConformance.SonobuoyResultTarGzPath)
- }
- if !strings.HasSuffix(cfg.AddOnConformance.SonobuoyResultTarGzPath, ".tar.gz") {
- return fmt.Errorf("AddOnConformance.SonobuoyResultTarGzPath[%q] must have '.tar.gz' extension", cfg.AddOnConformance.SonobuoyResultTarGzPath)
- }
- if cfg.AddOnConformance.SonobuoyResultTarGzS3Key == "" {
- cfg.AddOnConformance.SonobuoyResultTarGzS3Key = path.Join(
- cfg.AddOnConformance.S3Dir,
- filepath.Base(cfg.AddOnConformance.SonobuoyResultTarGzPath),
- )
- }
-
- if cfg.AddOnConformance.SonobuoyResultJunitXMLPath == "" {
- cfg.AddOnConformance.SonobuoyResultJunitXMLPath = filepath.Join(
- filepath.Dir(cfg.ConfigPath),
- fmt.Sprintf("%s-sonobuoy-result.junit.xml", cfg.Name),
- )
- os.RemoveAll(cfg.AddOnConformance.SonobuoyResultJunitXMLPath)
- }
- if !strings.HasSuffix(cfg.AddOnConformance.SonobuoyResultJunitXMLPath, ".xml") {
- return fmt.Errorf("AddOnConformance.SonobuoyResultJunitXMLPath[%q] must have '.xml' extension", cfg.AddOnConformance.SonobuoyResultTarGzPath)
- }
- if cfg.AddOnConformance.SonobuoyResultJunitXMLS3Key == "" {
- cfg.AddOnConformance.SonobuoyResultJunitXMLS3Key = path.Join(
- cfg.AddOnConformance.S3Dir,
- filepath.Base(cfg.AddOnConformance.SonobuoyResultJunitXMLPath),
- )
- }
-
- return nil
-}
diff --git a/eksconfig/add-on-cron-jobs.go b/eksconfig/add-on-cron-jobs.go
deleted file mode 100644
index 01465fedc..000000000
--- a/eksconfig/add-on-cron-jobs.go
+++ /dev/null
@@ -1,106 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnCronJobs defines parameters for EKS cluster
-// add-on with CronJob.
-type AddOnCronJobs struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // RepositoryBusyboxAccountID is the account ID for tester ECR image.
- // e.g. "busybox" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/busybox"
- RepositoryBusyboxAccountID string `json:"repository-busybox-account-id,omitempty"`
- // RepositoryBusyboxRegion is the ECR repository region to pull from.
- RepositoryBusyboxRegion string `json:"repository-busybox-region,omitempty"`
- // RepositoryBusyboxName is the repositoryName for tester ECR image.
- // e.g. "busybox" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/busybox"
- RepositoryBusyboxName string `json:"repository-busybox-name,omitempty"`
- // RepositoryBusyboxImageTag is the image tag for tester ECR image.
- // e.g. "latest" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/busybox:latest"
- RepositoryBusyboxImageTag string `json:"repository-busybox-image-tag,omitempty"`
-
- // Schedule is the cron schedule (e.g. "*/1 * * * *").
- Schedule string `json:"schedule"`
- // Completes is the desired number of successfully finished pods.
- Completes int `json:"completes"`
- // Parallels is the the maximum desired number of pods the
- // job should run at any given time.
- Parallels int `json:"parallels"`
- // SuccessfulJobsHistoryLimit is the number of successful finished
- // jobs to retain. Defaults to 3.
- SuccessfulJobsHistoryLimit int32 `json:"successful-jobs-history-limit"`
- // FailedJobsHistoryLimit is the number of failed finished jobs
- // to retain. Defaults to 1.
- FailedJobsHistoryLimit int32 `json:"failed-jobs-history-limit"`
-
- // EchoSize is the job object size in bytes.
- // "Request entity too large: limit is 3145728" (3.1 MB).
- // "The Job "echo" is invalid: metadata.annotations:
- // Too long: must have at most 262144 characters". (0.26 MB)
- EchoSize int `json:"echo-size"`
-}
-
-// EnvironmentVariablePrefixAddOnCronJobs is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnCronJobs = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_CRON_JOBS_"
-
-// IsEnabledAddOnCronJobs returns true if "AddOnCronJobs" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnCronJobs() bool {
- if cfg.AddOnCronJobs == nil {
- return false
- }
- if cfg.AddOnCronJobs.Enable {
- return true
- }
- cfg.AddOnCronJobs = nil
- return false
-}
-
-func getDefaultAddOnCronJobs() *AddOnCronJobs {
- return &AddOnCronJobs{
- Enable: false,
- Schedule: "*/10 * * * *", // every 10-min
- Completes: 10,
- Parallels: 10,
- SuccessfulJobsHistoryLimit: 3,
- FailedJobsHistoryLimit: 1,
- EchoSize: 100 * 1024, // 100 KB
- }
-}
-
-func (cfg *Config) GetAddOnCronJobsRepositoryBusyboxRegion() string {
- if !cfg.IsEnabledAddOnCronJobs() {
- return cfg.Region
- }
- return cfg.AddOnCronJobs.RepositoryBusyboxRegion
-}
-
-func (cfg *Config) validateAddOnCronJobs() error {
- if !cfg.IsEnabledAddOnCronJobs() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnCronJobs.Enable true but no node group is enabled")
- }
- if cfg.AddOnCronJobs.Namespace == "" {
- cfg.AddOnCronJobs.Namespace = cfg.Name + "-cronjob"
- }
- if cfg.AddOnCronJobs.EchoSize > 250000 {
- return fmt.Errorf("echo size limit is 0.25 MB, got %d", cfg.AddOnCronJobs.EchoSize)
- }
- return nil
-}
diff --git a/eksconfig/add-on-csi-ebs.go b/eksconfig/add-on-csi-ebs.go
deleted file mode 100644
index e6a15aa0b..000000000
--- a/eksconfig/add-on-csi-ebs.go
+++ /dev/null
@@ -1,58 +0,0 @@
-package eksconfig
-
-import (
- "errors"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnCSIEBS defines parameters for EKS cluster
-// add-on AWS EBS CSI Driver.
-// ref. https://github.com/kubernetes-sigs/aws-ebs-csi-driver#deploy-driver
-type AddOnCSIEBS struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // ChartRepoURL is the chart repo URL.
- // e.g. https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/v0.5.0/helm-chart.tgz
- ChartRepoURL string `json:"chart-repo-url"`
-}
-
-// EnvironmentVariablePrefixAddOnCSIEBS is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnCSIEBS = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_CSI_EBS_"
-
-// IsEnabledAddOnCSIEBS returns true if "AddOnCSIEBS" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnCSIEBS() bool {
- if cfg.AddOnCSIEBS == nil {
- return false
- }
- if cfg.AddOnCSIEBS.Enable {
- return true
- }
- cfg.AddOnCSIEBS = nil
- return false
-}
-
-func getDefaultAddOnCSIEBS() *AddOnCSIEBS {
- return &AddOnCSIEBS{
- Enable: false,
- // https://github.com/kubernetes-sigs/aws-ebs-csi-driver#deploy-driver
- ChartRepoURL: "https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/v0.5.0/helm-chart.tgz",
- }
-}
-
-func (cfg *Config) validateAddOnCSIEBS() error {
- if !cfg.IsEnabledAddOnCSIEBS() {
- return nil
- }
- if cfg.AddOnCSIEBS.ChartRepoURL == "" {
- return errors.New("unexpected empty AddOnCSIEBS.ChartRepoURL")
- }
- return nil
-}
diff --git a/eksconfig/add-on-csrs-local.go b/eksconfig/add-on-csrs-local.go
deleted file mode 100644
index 5449f1f1a..000000000
--- a/eksconfig/add-on-csrs-local.go
+++ /dev/null
@@ -1,231 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
- "path"
- "path/filepath"
- "strings"
-
- "github.com/aws/aws-k8s-tester/pkg/metrics"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnCSRsLocal defines parameters for EKS cluster
-// add-on "CertificateSigningRequest" local.
-// It generates loads from the local host machine.
-// Every object is written serially with no concurrency.
-// Use remote tester to write with concurrency.
-// The main use case is to write a large number of objects to fill up etcd database.
-type AddOnCSRsLocal struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // S3Dir is the S3 directory to store all test results.
- // It is under the bucket "eksconfig.Config.S3BucketName".
- S3Dir string `json:"s3-dir"`
-
- // Objects is the number of "CertificateSigningRequest" objects to create.
- Objects int `json:"objects"`
-
- // InitialRequestConditionType is the initial CSR condition type
- // to simulate CSR condition.
- //
- // Valid values are:
- // "k8s.io/api/certificates/v1beta1.CertificateApproved" == "Approved"
- // "k8s.io/api/certificates/v1beta1.CertificateDenied" == "Denied"
- // "Random"
- // "Pending"
- // ""
- //
- InitialRequestConditionType string `json:"initial-request-condition-type"`
-
- // CreatedNames is the list of created "CertificateSigningRequest" object names.
- CreatedNames []string `json:"created-names" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- RequestsRawWritesJSONPath string `json:"requests-raw-writes-json-path" read-only:"true"`
- RequestsRawWritesJSONS3Key string `json:"requests-raw-writes-json-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsRawWritesCompareS3Dir is the s3 directory to store raw data points.
- // Used to comparison results.
- // ref. https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
- RequestsRawWritesCompareS3Dir string `json:"requests-raw-writes-compare-s3-dir"`
- RequestsRawWritesCompareAllJSONPath string `json:"requests-raw-writes-compare-all-json-path" read-only:"true"`
- RequestsRawWritesCompareAllJSONS3Key string `json:"requests-raw-writes-compare-all-json-s3-key" read-only:"true"`
- RequestsRawWritesCompareAllCSVPath string `json:"requests-raw-writes-compare-all-csv-path" read-only:"true"`
- RequestsRawWritesCompareAllCSVS3Key string `json:"requests-raw-writes-compare-all-csv-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWrites is the writes results.
- RequestsSummaryWrites metrics.RequestsSummary `json:"requests-summary-writes,omitempty" read-only:"true"`
- RequestsSummaryWritesJSONPath string `json:"requests-summary-writes-json-path" read-only:"true"`
- RequestsSummaryWritesJSONS3Key string `json:"requests-summary-writes-json-s3-key" read-only:"true"`
- RequestsSummaryWritesTablePath string `json:"requests-summary-writes-table-path" read-only:"true"`
- RequestsSummaryWritesTableS3Key string `json:"requests-summary-writes-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWritesCompareS3Dir is the S3 directory of previous/latest "RequestsSummary".
- // Specify the S3 key in the same bucket of "eksconfig.Config.S3BucketName".
- // Use for regression tests. Specify the value not bound to the cluster directory.
- // Different runs from different clusters reads and writes in this directory.
- RequestsSummaryWritesCompareS3Dir string `json:"requests-summary-writes-compare-s3-dir"`
- RequestsSummaryWritesCompare metrics.RequestsCompare `json:"requests-summary-writes-compare" read-only:"true"`
- RequestsSummaryWritesCompareJSONPath string `json:"requests-summary-writes-compare-json-path" read-only:"true"`
- RequestsSummaryWritesCompareJSONS3Key string `json:"requests-summary-writes-compare-json-s3-key" read-only:"true"`
- RequestsSummaryWritesCompareTablePath string `json:"requests-summary-writes-compare-table-path" read-only:"true"`
- RequestsSummaryWritesCompareTableS3Key string `json:"requests-summary-writes-compare-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-}
-
-// EnvironmentVariablePrefixAddOnCSRsLocal is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnCSRsLocal = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_CSRS_LOCAL_"
-
-// IsEnabledAddOnCSRsLocal returns true if "AddOnCSRsLocal" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnCSRsLocal() bool {
- if cfg.AddOnCSRsLocal == nil {
- return false
- }
- if cfg.AddOnCSRsLocal.Enable {
- return true
- }
- cfg.AddOnCSRsLocal = nil
- return false
-}
-
-func getDefaultAddOnCSRsLocal() *AddOnCSRsLocal {
- return &AddOnCSRsLocal{
- Enable: false,
- Objects: 10, // 1000 objects generates 5 MB data to etcd
- InitialRequestConditionType: "",
- }
-}
-
-func (cfg *Config) validateAddOnCSRsLocal() error {
- if !cfg.IsEnabledAddOnCSRsLocal() {
- return nil
- }
-
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnCSRsLocal.Enable true but no node group is enabled")
- }
-
- if cfg.AddOnCSRsLocal.S3Dir == "" {
- cfg.AddOnCSRsLocal.S3Dir = path.Join(cfg.Name, "add-on-csrs-local")
- }
-
- if cfg.AddOnCSRsLocal.Objects == 0 {
- cfg.AddOnCSRsLocal.Objects = 10
- }
-
- switch cfg.AddOnCSRsLocal.InitialRequestConditionType {
- case "Approved":
- case "Denied":
- case "Pending", "":
- case "Random":
- default:
- return fmt.Errorf("unknown AddOnCSRsLocal.InitialRequestConditionType %q", cfg.AddOnCSRsLocal.InitialRequestConditionType)
- }
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnCSRsLocal.RequestsRawWritesJSONPath == "" {
- cfg.AddOnCSRsLocal.RequestsRawWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-csrs-local-requests-writes-raw.json"
- }
- if cfg.AddOnCSRsLocal.RequestsRawWritesJSONS3Key == "" {
- cfg.AddOnCSRsLocal.RequestsRawWritesJSONS3Key = path.Join(
- cfg.AddOnCSRsLocal.S3Dir,
- "requests-raw-writes",
- filepath.Base(cfg.AddOnCSRsLocal.RequestsRawWritesJSONPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnCSRsLocal.RequestsRawWritesCompareS3Dir == "" {
- cfg.AddOnCSRsLocal.RequestsRawWritesCompareS3Dir = path.Join("add-on-csrs-local", "requests-raw-writes-compare", cfg.Version)
- }
- if cfg.AddOnCSRsLocal.RequestsRawWritesCompareAllJSONPath == "" {
- cfg.AddOnCSRsLocal.RequestsRawWritesCompareAllJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-csrs-local-requests-raw-writes-compare-all.json"
- }
- if cfg.AddOnCSRsLocal.RequestsRawWritesCompareAllJSONS3Key == "" {
- cfg.AddOnCSRsLocal.RequestsRawWritesCompareAllJSONS3Key = path.Join(
- cfg.AddOnCSRsLocal.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnCSRsLocal.RequestsRawWritesCompareAllJSONPath),
- )
- }
- if cfg.AddOnCSRsLocal.RequestsRawWritesCompareAllCSVPath == "" {
- cfg.AddOnCSRsLocal.RequestsRawWritesCompareAllCSVPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-csrs-local-requests-raw-writes-compare-all.csv"
- }
- if cfg.AddOnCSRsLocal.RequestsRawWritesCompareAllCSVS3Key == "" {
- cfg.AddOnCSRsLocal.RequestsRawWritesCompareAllCSVS3Key = path.Join(
- cfg.AddOnCSRsLocal.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnCSRsLocal.RequestsRawWritesCompareAllCSVPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnCSRsLocal.RequestsSummaryWritesJSONPath == "" {
- cfg.AddOnCSRsLocal.RequestsSummaryWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-csrs-local-requests-summary-writes.json"
- }
- if cfg.AddOnCSRsLocal.RequestsSummaryWritesJSONS3Key == "" {
- cfg.AddOnCSRsLocal.RequestsSummaryWritesJSONS3Key = path.Join(
- cfg.AddOnCSRsLocal.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnCSRsLocal.RequestsSummaryWritesJSONPath),
- )
- }
- if cfg.AddOnCSRsLocal.RequestsSummaryWritesTablePath == "" {
- cfg.AddOnCSRsLocal.RequestsSummaryWritesTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-csrs-local-requests-summary-writes.txt"
- }
- if cfg.AddOnCSRsLocal.RequestsSummaryWritesTableS3Key == "" {
- cfg.AddOnCSRsLocal.RequestsSummaryWritesTableS3Key = path.Join(
- cfg.AddOnCSRsLocal.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnCSRsLocal.RequestsSummaryWritesTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnCSRsLocal.RequestsSummaryWritesCompareS3Dir == "" {
- cfg.AddOnCSRsLocal.RequestsSummaryWritesCompareS3Dir = path.Join("add-on-csrs-local", "requests-summary-writes-compare", cfg.Version)
- }
- if cfg.AddOnCSRsLocal.RequestsSummaryWritesCompareJSONPath == "" {
- cfg.AddOnCSRsLocal.RequestsSummaryWritesCompareJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-csrs-local-requests-summary-writes-compare.json"
- }
- if cfg.AddOnCSRsLocal.RequestsSummaryWritesCompareJSONS3Key == "" {
- cfg.AddOnCSRsLocal.RequestsSummaryWritesCompareJSONS3Key = path.Join(
- cfg.AddOnCSRsLocal.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnCSRsLocal.RequestsSummaryWritesCompareJSONPath),
- )
- }
- if cfg.AddOnCSRsLocal.RequestsSummaryWritesCompareTablePath == "" {
- cfg.AddOnCSRsLocal.RequestsSummaryWritesCompareTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-csrs-local-requests-summary-writes-compare.txt"
- }
- if cfg.AddOnCSRsLocal.RequestsSummaryWritesCompareTableS3Key == "" {
- cfg.AddOnCSRsLocal.RequestsSummaryWritesCompareTableS3Key = path.Join(
- cfg.AddOnCSRsLocal.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnCSRsLocal.RequestsSummaryWritesCompareTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- return nil
-}
diff --git a/eksconfig/add-on-csrs-remote.go b/eksconfig/add-on-csrs-remote.go
deleted file mode 100644
index 3f12cf3ec..000000000
--- a/eksconfig/add-on-csrs-remote.go
+++ /dev/null
@@ -1,288 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
- "path"
- "path/filepath"
- "strings"
-
- "github.com/aws/aws-k8s-tester/pkg/metrics"
- "github.com/aws/aws-k8s-tester/pkg/randutil"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnCSRsRemote defines parameters for EKS cluster
-// add-on "CertificateSigningRequest" remote.
-// It generates loads from the remote workers (Pod) in the cluster.
-// Each worker writes serially with no concurrency.
-// Configure "DeploymentReplicas" accordingly to increase the concurrency.
-// The main use case is to write a large number of objects to fill up etcd database.
-type AddOnCSRsRemote struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // S3Dir is the S3 directory to store all test results.
- // It is under the bucket "eksconfig.Config.S3BucketName".
- S3Dir string `json:"s3-dir"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // RepositoryAccountID is the account ID for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryAccountID string `json:"repository-account-id,omitempty"`
- // RepositoryRegion is the ECR repository region to pull from.
- RepositoryRegion string `json:"repository-region,omitempty"`
- // RepositoryName is the repositoryName for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryName string `json:"repository-name,omitempty"`
- // RepositoryImageTag is the image tag for tester ECR image.
- // e.g. "latest" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester:latest"
- RepositoryImageTag string `json:"repository-image-tag,omitempty"`
-
- // Completes is the desired number of successfully finished pods.
- // Write QPS will be client QPS * replicas.
- // Read QPS will be client QPS * replicas.
- Completes int `json:"completes"`
- // Parallels is the the maximum desired number of pods the
- // job should run at any given time.
- // Write QPS will be client QPS * replicas.
- // Read QPS will be client QPS * replicas.
- Parallels int `json:"parallels"`
-
- // Objects is the number of "CertificateSigningRequest" objects to create.
- Objects int `json:"objects"`
-
- // InitialRequestConditionType is the initial CSR condition type
- // to simulate CSR condition.
- //
- // Valid values are:
- // "k8s.io/api/certificates/v1beta1.CertificateApproved" == "Approved"
- // "k8s.io/api/certificates/v1beta1.CertificateDenied" == "Denied"
- // "Random"
- // "Pending"
- // ""
- //
- InitialRequestConditionType string `json:"initial-request-condition-type"`
-
- // RequestsSummaryWritesOutputNamePrefix is the output path name in "/var/log" directory, used in remote worker.
- RequestsSummaryWritesOutputNamePrefix string `json:"requests-summary-writes-output-name-prefix"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- RequestsRawWritesJSONPath string `json:"requests-raw-writes-json-path" read-only:"true"`
- RequestsRawWritesJSONS3Key string `json:"requests-raw-writes-json-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsRawWritesCompareS3Dir is the s3 directory to store raw data points.
- // Used to comparison results.
- // ref. https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
- RequestsRawWritesCompareS3Dir string `json:"requests-raw-writes-compare-s3-dir"`
- RequestsRawWritesCompareAllJSONPath string `json:"requests-raw-writes-compare-all-json-path" read-only:"true"`
- RequestsRawWritesCompareAllJSONS3Key string `json:"requests-raw-writes-compare-all-json-s3-key" read-only:"true"`
- RequestsRawWritesCompareAllCSVPath string `json:"requests-raw-writes-compare-all-csv-path" read-only:"true"`
- RequestsRawWritesCompareAllCSVS3Key string `json:"requests-raw-writes-compare-all-csv-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWrites is the writes results.
- RequestsSummaryWrites metrics.RequestsSummary `json:"requests-summary-writes,omitempty" read-only:"true"`
- RequestsSummaryWritesJSONPath string `json:"requests-summary-writes-json-path" read-only:"true"`
- RequestsSummaryWritesJSONS3Key string `json:"requests-summary-writes-json-s3-key" read-only:"true"`
- RequestsSummaryWritesTablePath string `json:"requests-summary-writes-table-path" read-only:"true"`
- RequestsSummaryWritesTableS3Key string `json:"requests-summary-writes-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWritesCompareS3Dir is the S3 directory of previous/latest "RequestsSummary".
- // Specify the S3 key in the same bucket of "eksconfig.Config.S3BucketName".
- // Use for regression tests. Specify the value not bound to the cluster directory.
- // Different runs from different clusters reads and writes in this directory.
- RequestsSummaryWritesCompareS3Dir string `json:"requests-summary-writes-compare-s3-dir"`
- RequestsSummaryWritesCompare metrics.RequestsCompare `json:"requests-summary-writes-compare" read-only:"true"`
- RequestsSummaryWritesCompareJSONPath string `json:"requests-summary-writes-compare-json-path" read-only:"true"`
- RequestsSummaryWritesCompareJSONS3Key string `json:"requests-summary-writes-compare-json-s3-key" read-only:"true"`
- RequestsSummaryWritesCompareTablePath string `json:"requests-summary-writes-compare-table-path" read-only:"true"`
- RequestsSummaryWritesCompareTableS3Key string `json:"requests-summary-writes-compare-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-}
-
-// EnvironmentVariablePrefixAddOnCSRsRemote is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnCSRsRemote = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_CSRS_REMOTE_"
-
-// IsEnabledAddOnCSRsRemote returns true if "AddOnCSRsRemote" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnCSRsRemote() bool {
- if cfg.AddOnCSRsRemote == nil {
- return false
- }
- if cfg.AddOnCSRsRemote.Enable {
- return true
- }
- cfg.AddOnCSRsRemote = nil
- return false
-}
-
-func getDefaultAddOnCSRsRemote() *AddOnCSRsRemote {
- return &AddOnCSRsRemote{
- Enable: false,
- Completes: 5,
- Parallels: 5,
- Objects: 10, // 1000 objects generates 5 MB data to etcd
- InitialRequestConditionType: "",
- RequestsSummaryWritesOutputNamePrefix: "csrs-writes-" + randutil.String(10),
- }
-}
-
-func (cfg *Config) GetAddOnCSRsRemoteRepositoryRegion() string {
- if !cfg.IsEnabledAddOnCSRsRemote() {
- return cfg.Region
- }
- return cfg.AddOnCSRsRemote.RepositoryRegion
-}
-
-func (cfg *Config) validateAddOnCSRsRemote() error {
- if !cfg.IsEnabledAddOnCSRsRemote() {
- return nil
- }
-
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnCSRsRemote.Enable true but no node group is enabled")
- }
-
- if cfg.AddOnCSRsRemote.S3Dir == "" {
- cfg.AddOnCSRsRemote.S3Dir = path.Join(cfg.Name, "add-on-csrs-remote")
- }
-
- if cfg.AddOnCSRsRemote.Namespace == "" {
- cfg.AddOnCSRsRemote.Namespace = cfg.Name + "-csrs-remote"
- }
-
- if cfg.AddOnCSRsRemote.RepositoryAccountID == "" {
- return errors.New("AddOnCSRsRemote.RepositoryAccountID empty")
- }
- if cfg.AddOnCSRsRemote.RepositoryRegion == "" {
- cfg.AddOnCSRsRemote.RepositoryRegion = cfg.Region
- }
- if cfg.AddOnCSRsRemote.RepositoryName == "" {
- return errors.New("AddOnCSRsRemote.RepositoryName empty")
- }
- if cfg.AddOnCSRsRemote.RepositoryImageTag == "" {
- return errors.New("AddOnCSRsRemote.RepositoryImageTag empty")
- }
-
- if cfg.AddOnCSRsRemote.Objects == 0 {
- cfg.AddOnCSRsRemote.Objects = 10
- }
-
- switch cfg.AddOnCSRsRemote.InitialRequestConditionType {
- case "Approved":
- case "Denied":
- case "Pending", "":
- case "Random":
- default:
- return fmt.Errorf("unknown AddOnCSRsRemote.InitialRequestConditionType %q", cfg.AddOnCSRsRemote.InitialRequestConditionType)
- }
-
- if cfg.AddOnCSRsRemote.RequestsSummaryWritesOutputNamePrefix == "" {
- cfg.AddOnCSRsRemote.RequestsSummaryWritesOutputNamePrefix = "csrs-writes-" + randutil.String(10)
- }
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnCSRsRemote.RequestsRawWritesJSONPath == "" {
- cfg.AddOnCSRsRemote.RequestsRawWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-csrs-remote-requests-writes-raw.json"
- }
- if cfg.AddOnCSRsRemote.RequestsRawWritesJSONS3Key == "" {
- cfg.AddOnCSRsRemote.RequestsRawWritesJSONS3Key = path.Join(
- cfg.AddOnCSRsRemote.S3Dir,
- "requests-raw-writes",
- filepath.Base(cfg.AddOnCSRsRemote.RequestsRawWritesJSONPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnCSRsRemote.RequestsRawWritesCompareS3Dir == "" {
- cfg.AddOnCSRsRemote.RequestsRawWritesCompareS3Dir = path.Join("add-on-csrs-remote", "requests-raw-writes-compare", cfg.Version)
- }
- if cfg.AddOnCSRsRemote.RequestsRawWritesCompareAllJSONPath == "" {
- cfg.AddOnCSRsRemote.RequestsRawWritesCompareAllJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-csrs-remote-requests-raw-writes-compare-all.json"
- }
- if cfg.AddOnCSRsRemote.RequestsRawWritesCompareAllJSONS3Key == "" {
- cfg.AddOnCSRsRemote.RequestsRawWritesCompareAllJSONS3Key = path.Join(
- cfg.AddOnCSRsRemote.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnCSRsRemote.RequestsRawWritesCompareAllJSONPath),
- )
- }
- if cfg.AddOnCSRsRemote.RequestsRawWritesCompareAllCSVPath == "" {
- cfg.AddOnCSRsRemote.RequestsRawWritesCompareAllCSVPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-csrs-remote-requests-raw-writes-compare-all.csv"
- }
- if cfg.AddOnCSRsRemote.RequestsRawWritesCompareAllCSVS3Key == "" {
- cfg.AddOnCSRsRemote.RequestsRawWritesCompareAllCSVS3Key = path.Join(
- cfg.AddOnCSRsRemote.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnCSRsRemote.RequestsRawWritesCompareAllCSVPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnCSRsRemote.RequestsSummaryWritesJSONPath == "" {
- cfg.AddOnCSRsRemote.RequestsSummaryWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-csrs-remote-requests-summary-writes.json"
- }
- if cfg.AddOnCSRsRemote.RequestsSummaryWritesJSONS3Key == "" {
- cfg.AddOnCSRsRemote.RequestsSummaryWritesJSONS3Key = path.Join(
- cfg.AddOnCSRsRemote.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnCSRsRemote.RequestsSummaryWritesJSONPath),
- )
- }
- if cfg.AddOnCSRsRemote.RequestsSummaryWritesTablePath == "" {
- cfg.AddOnCSRsRemote.RequestsSummaryWritesTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-csrs-remote-requests-summary-writes.txt"
- }
- if cfg.AddOnCSRsRemote.RequestsSummaryWritesTableS3Key == "" {
- cfg.AddOnCSRsRemote.RequestsSummaryWritesTableS3Key = path.Join(
- cfg.AddOnCSRsRemote.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnCSRsRemote.RequestsSummaryWritesTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnCSRsRemote.RequestsSummaryWritesCompareS3Dir == "" {
- cfg.AddOnCSRsRemote.RequestsSummaryWritesCompareS3Dir = path.Join("add-on-csrs-remote", "requests-summary-writes-compare", cfg.Version)
- }
- if cfg.AddOnCSRsRemote.RequestsSummaryWritesCompareJSONPath == "" {
- cfg.AddOnCSRsRemote.RequestsSummaryWritesCompareJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-csrs-remote-requests-summary-writes-compare.json"
- }
- if cfg.AddOnCSRsRemote.RequestsSummaryWritesCompareJSONS3Key == "" {
- cfg.AddOnCSRsRemote.RequestsSummaryWritesCompareJSONS3Key = path.Join(
- cfg.AddOnCSRsRemote.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnCSRsRemote.RequestsSummaryWritesCompareJSONPath),
- )
- }
- if cfg.AddOnCSRsRemote.RequestsSummaryWritesCompareTablePath == "" {
- cfg.AddOnCSRsRemote.RequestsSummaryWritesCompareTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-csrs-remote-requests-summary-writes-compare.txt"
- }
- if cfg.AddOnCSRsRemote.RequestsSummaryWritesCompareTableS3Key == "" {
- cfg.AddOnCSRsRemote.RequestsSummaryWritesCompareTableS3Key = path.Join(
- cfg.AddOnCSRsRemote.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnCSRsRemote.RequestsSummaryWritesCompareTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- return nil
-}
diff --git a/eksconfig/add-on-cuda-vector-add.go b/eksconfig/add-on-cuda-vector-add.go
deleted file mode 100644
index f1b475cad..000000000
--- a/eksconfig/add-on-cuda-vector-add.go
+++ /dev/null
@@ -1,83 +0,0 @@
-package eksconfig
-
-import (
- "errors"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/service/eks"
-)
-
-// AddOnCUDAVectorAdd defines parameters for EKS cluster
-// add-on cuda-vector-add.
-type AddOnCUDAVectorAdd struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-}
-
-// EnvironmentVariablePrefixAddOnCUDAVectorAdd is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnCUDAVectorAdd = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_CUDA_VECTOR_ADD_"
-
-// IsEnabledAddOnCUDAVectorAdd returns true if "AddOnCUDAVectorAdd" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnCUDAVectorAdd() bool {
- if cfg.AddOnCUDAVectorAdd == nil {
- return false
- }
- if cfg.AddOnCUDAVectorAdd.Enable {
- return true
- }
- cfg.AddOnCUDAVectorAdd = nil
- return false
-}
-
-// TBD !!!!!!!!!!!
-func getDefaultAddOnCUDAVectorAdd() *AddOnCUDAVectorAdd {
- return &AddOnCUDAVectorAdd{
- Enable: false,
- }
-}
-
-func (cfg *Config) validateAddOnCUDAVectorAdd() error {
- if !cfg.IsEnabledAddOnCUDAVectorAdd() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnCUDAVectorAdd.Enable true but no node group is enabled")
- }
-
- gpuFound := false
- if cfg.IsEnabledAddOnNodeGroups() {
- for _, cur := range cfg.AddOnNodeGroups.ASGs {
- if cur.AMIType == ec2config.AMITypeAL2X8664GPU {
- gpuFound = true
- break
- }
- }
- }
- if !gpuFound && cfg.IsEnabledAddOnManagedNodeGroups() {
- for _, cur := range cfg.AddOnManagedNodeGroups.MNGs {
- if cur.AMIType == eks.AMITypesAl2X8664Gpu {
- gpuFound = true
- break
- }
- }
- }
- if !gpuFound {
- return errors.New("AddOnCUDAVectorAdd requires GPU AMI")
- }
-
- if cfg.AddOnCUDAVectorAdd.Namespace == "" {
- cfg.AddOnCUDAVectorAdd.Namespace = cfg.Name + "-cuda-vector-add"
- }
-
- return nil
-}
diff --git a/eksconfig/add-on-cw-agent.go b/eksconfig/add-on-cw-agent.go
deleted file mode 100644
index 0490b2edc..000000000
--- a/eksconfig/add-on-cw-agent.go
+++ /dev/null
@@ -1,59 +0,0 @@
-package eksconfig
-
-import (
- "errors"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnCWAgent defines parameters for EKS cluster
-// add-on CloudWatch agent.
-// Publishes worker nodes logs to:
-// - /aws/containerinsights/[CLUSTER-NAME]/performance
-type AddOnCWAgent struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-}
-
-// AWS_K8S_TESTER_EKS_ADD_ON_CW_AGENT_PREFIX is the environment variable prefix used for "eksconfig".
-const AWS_K8S_TESTER_EKS_ADD_ON_CW_AGENT_PREFIX = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_CW_AGENT_"
-
-// IsEnabledAddOnCWAgent returns true if "AddOnCWAgent" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnCWAgent() bool {
- if cfg.AddOnCWAgent == nil {
- return false
- }
- if cfg.AddOnCWAgent.Enable {
- return true
- }
- cfg.AddOnCWAgent = nil
- return false
-}
-
-func getDefaultAddOnCWAgent() *AddOnCWAgent {
- return &AddOnCWAgent{
- Enable: false,
- }
-}
-
-func (cfg *Config) validateAddOnCWAgent() error {
- if !cfg.IsEnabledAddOnCWAgent() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnCWAgent.Enable true but no node group is enabled")
- }
- if cfg.AddOnCWAgent.Namespace == "" {
- cfg.AddOnCWAgent.Namespace = cfg.Name + "-cw-agent"
- }
- return nil
-}
diff --git a/eksconfig/add-on-fargate.go b/eksconfig/add-on-fargate.go
deleted file mode 100644
index dcc6aa4e3..000000000
--- a/eksconfig/add-on-fargate.go
+++ /dev/null
@@ -1,178 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
- "path"
- "path/filepath"
- "regexp"
- "strings"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnFargate defines parameters for EKS cluster
-// add-on "EKS on Fargate".
-type AddOnFargate struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // S3Dir is the S3 directory to store all test results.
- // It is under the bucket "eksconfig.Config.S3BucketName".
- S3Dir string `json:"s3-dir"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // RepositoryAccountID is the account ID for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryAccountID string `json:"repository-account-id,omitempty"`
- // RepositoryRegion is the ECR repository region to pull from.
- RepositoryRegion string `json:"repository-region,omitempty"`
- // RepositoryName is the repositoryName for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryName string `json:"repository-name,omitempty"`
- // RepositoryImageTag is the image tag for tester ECR image.
- // e.g. "latest" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester:latest"
- RepositoryImageTag string `json:"repository-image-tag,omitempty"`
-
- // RoleName is the role name for Fargate.
- RoleName string `json:"role-name"`
- // RoleCreate is true to auto-create and delete role.
- RoleCreate bool `json:"role-create"`
- // RoleARN is the role ARN for Fargate.
- RoleARN string `json:"role-arn"`
- // RoleServicePrincipals is the Fargate role Service Principals
- RoleServicePrincipals []string `json:"role-service-principals"`
- // RoleManagedPolicyARNs is Fargate role managed policy ARNs.
- RoleManagedPolicyARNs []string `json:"role-managed-policy-arns"`
- RoleCFNStackID string `json:"role-cfn-stack-id" read-only:"true"`
- RoleCFNStackYAMLPath string `json:"role-cfn-stack-yaml-path" read-only:"true"`
- RoleCFNStackYAMLS3Key string `json:"role-cfn-stack-yaml-s3-key" read-only:"true"`
-
- // ProfileName is the profile name for Fargate.
- ProfileName string `json:"profile-name"`
-
- // SecretName is the secret name for Fargate.
- SecretName string `json:"secret-name"`
-}
-
-// EnvironmentVariablePrefixAddOnFargate is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnFargate = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_FARGATE_"
-
-// IsEnabledAddOnFargate returns true if "AddOnFargate" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnFargate() bool {
- if cfg.AddOnFargate == nil {
- return false
- }
- if cfg.AddOnFargate.Enable {
- return true
- }
- cfg.AddOnFargate = nil
- return false
-}
-
-func getDefaultAddOnFargate() *AddOnFargate {
- return &AddOnFargate{
- Enable: false,
- RoleCreate: true,
- }
-}
-
-func (cfg *Config) GetAddOnFargateRepositoryRegion() string {
- if !cfg.IsEnabledAddOnFargate() {
- return cfg.Region
- }
- return cfg.AddOnFargate.RepositoryRegion
-}
-
-// only letters and numbers for Secret key names
-var fargateSecretRegex = regexp.MustCompile("[^a-zA-Z0-9]+")
-
-func (cfg *Config) validateAddOnFargate() error {
- if !cfg.IsEnabledAddOnFargate() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnFargate.Enable true but no node group is enabled")
- }
- if cfg.VersionValue < 1.14 {
- return fmt.Errorf("Version %q not supported for AddOnFargate", cfg.Version)
- }
-
- if cfg.AddOnFargate.S3Dir == "" {
- cfg.AddOnFargate.S3Dir = path.Join(cfg.Name, "add-on-fargate")
- }
-
- if cfg.AddOnFargate.Namespace == "" {
- cfg.AddOnFargate.Namespace = cfg.Name + "-fargate"
- }
-
- if cfg.AddOnFargate.RepositoryName != "" {
- if cfg.AddOnFargate.RepositoryAccountID == "" {
- return errors.New("AddOnFargate.RepositoryAccountID empty")
- }
- if cfg.AddOnFargate.RepositoryRegion == "" {
- cfg.AddOnFargate.RepositoryRegion = cfg.Region
- }
- if cfg.AddOnFargate.RepositoryImageTag == "" {
- return errors.New("AddOnFargate.RepositoryImageTag empty")
- }
- }
-
- // do not prefix with "eks-"
- // e.g. "The fargate profile name starts with the reserved prefix: 'eks-'."
- if cfg.AddOnFargate.ProfileName == "" {
- cfg.AddOnFargate.ProfileName = cfg.Name + "-fargate-profile"
- }
- if strings.HasPrefix(cfg.AddOnFargate.ProfileName, "eks-") {
- cfg.AddOnFargate.ProfileName = strings.Replace(cfg.AddOnFargate.ProfileName, "eks-", "", 1)
- }
-
- if cfg.AddOnFargate.SecretName == "" {
- cfg.AddOnFargate.SecretName = cfg.Name + "addonfargatesecret"
- }
- cfg.AddOnFargate.SecretName = strings.ToLower(fargateSecretRegex.ReplaceAllString(cfg.AddOnFargate.SecretName, ""))
-
- if cfg.AddOnFargate.RoleCFNStackYAMLPath == "" {
- cfg.AddOnFargate.RoleCFNStackYAMLPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + ".add-on-fargate.role.cfn.yaml"
- }
- if cfg.AddOnFargate.RoleCFNStackYAMLS3Key == "" {
- cfg.AddOnFargate.RoleCFNStackYAMLS3Key = path.Join(
- cfg.AddOnFargate.S3Dir,
- filepath.Base(cfg.AddOnFargate.RoleCFNStackYAMLPath),
- )
- }
- switch cfg.AddOnFargate.RoleCreate {
- case true: // need create one, or already created
- if cfg.AddOnFargate.RoleName == "" {
- cfg.AddOnFargate.RoleName = cfg.Name + "-add-on-fargate-role"
- }
- if cfg.AddOnFargate.RoleARN != "" {
- // just ignore...
- // could be populated from previous run
- // do not error, so long as RoleCreate false, role won't be deleted
- }
- case false: // use existing one
- if cfg.AddOnFargate.RoleARN == "" {
- return fmt.Errorf("AddOnFargate.RoleCreate false; expect non-empty RoleARN but got %q", cfg.AddOnFargate.RoleARN)
- }
- if cfg.AddOnFargate.RoleName == "" {
- cfg.AddOnFargate.RoleName = getNameFromARN(cfg.AddOnFargate.RoleARN)
- }
- if len(cfg.AddOnFargate.RoleManagedPolicyARNs) > 0 {
- return fmt.Errorf("AddOnFargate.RoleCreate false; expect empty RoleManagedPolicyARNs but got %q", cfg.AddOnFargate.RoleManagedPolicyARNs)
- }
- if len(cfg.AddOnFargate.RoleServicePrincipals) > 0 {
- return fmt.Errorf("AddOnFargate.RoleCreate false; expect empty RoleServicePrincipals but got %q", cfg.AddOnFargate.RoleServicePrincipals)
- }
- }
-
- return nil
-}
diff --git a/eksconfig/add-on-fluentd.go b/eksconfig/add-on-fluentd.go
deleted file mode 100644
index 82622f80a..000000000
--- a/eksconfig/add-on-fluentd.go
+++ /dev/null
@@ -1,125 +0,0 @@
-package eksconfig
-
-import (
- "errors"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnFluentd defines parameters for EKS cluster
-// add-on Fluentd.
-// Publishes worker nodes logs to:
-// - /aws/containerinsights/[CLUSTER-NAME]/application
-// - /aws/containerinsights/[CLUSTER-NAME]/dataplane
-// - /aws/containerinsights/[CLUSTER-NAME]/host
-type AddOnFluentd struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // RepositoryBusyboxAccountID is the account ID for tester ECR image.
- // e.g. "busybox" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/busybox"
- RepositoryBusyboxAccountID string `json:"repository-busybox-account-id,omitempty"`
- // RepositoryBusyboxRegion is the ECR repository region to pull from.
- RepositoryBusyboxRegion string `json:"repository-busybox-region,omitempty"`
- // RepositoryBusyboxName is the repositoryName for tester ECR image.
- // e.g. "busybox" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/busybox"
- RepositoryBusyboxName string `json:"repository-busybox-name,omitempty"`
- // RepositoryBusyboxImageTag is the image tag for tester ECR image.
- // e.g. "latest" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/busybox:latest"
- RepositoryBusyboxImageTag string `json:"repository-busybox-image-tag,omitempty"`
-
- // Threads is the number of threads for fluentd.
- // ref. https://docs.fluentd.org/v/0.12/output#num_threads
- Threads int `json:"threads"`
- // MetadataLogLevel is the log level for "@type kubernetes_metadata".
- // ref. https://docs.fluentd.org/deployment/system-config
- // ref. https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
- MetadataLogLevel string `json:"metadata-log-level"`
- // MetadataCacheSize is the size of the cache of Kubernetes metadata
- // to reduce the requests to the kube-apiserver.
- // ref. "@type kubernetes_metadata"
- // ref. https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
- MetadataCacheSize int `json:"metadata-cache-size"`
- // MetadataWatch is true to enable watch on pods on the kube-apiserver
- // for updates to the metadata.
- // ref. "@type kubernetes_metadata"
- // ref. https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
- MetadataWatch bool `json:"metadata-watch"`
- // MetadataSkipLabels is true to skip all label fields from the metadata.
- // ref. "@type kubernetes_metadata"
- // ref. https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
- MetadataSkipLabels bool `json:"metadata-skip-labels"`
- // MetadataSkipMasterURL is true to skip "master_url" field from the metadata.
- // ref. "@type kubernetes_metadata"
- // ref. https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
- MetadataSkipMasterURL bool `json:"metadata-skip-master-url"`
- // MetadataSkipContainerMetadata is true to skip some container data of the metadata.
- // For example, if true, it skips container image and image ID fields.
- // ref. "@type kubernetes_metadata"
- // ref. https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
- MetadataSkipContainerMetadata bool `json:"metadata-skip-container-metadata"`
- // MetadataSkipNamespaceMetadata is true to skip "namespace_id" field from the metadata.
- // If true, the plugin will be faster with less CPU consumption.
- // ref. "@type kubernetes_metadata"
- // ref. https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
- MetadataSkipNamespaceMetadata bool `json:"metadata-skip-namespace-metadata"`
-}
-
-// AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_PREFIX is the environment variable prefix used for "eksconfig".
-const AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_PREFIX = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_FLUENTD_"
-
-// IsEnabledAddOnFluentd returns true if "AddOnFluentd" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnFluentd() bool {
- if cfg.AddOnFluentd == nil {
- return false
- }
- if cfg.AddOnFluentd.Enable {
- return true
- }
- cfg.AddOnFluentd = nil
- return false
-}
-
-func getDefaultAddOnFluentd() *AddOnFluentd {
- return &AddOnFluentd{
- Enable: false,
-
- Threads: 8,
- MetadataLogLevel: "warn",
- MetadataCacheSize: 20000,
- MetadataWatch: false,
- MetadataSkipLabels: true,
- MetadataSkipMasterURL: true,
- MetadataSkipContainerMetadata: true,
- MetadataSkipNamespaceMetadata: true,
- }
-}
-
-func (cfg *Config) GetAddOnFluentdRepositoryBusyboxRegion() string {
- if !cfg.IsEnabledAddOnFluentd() {
- return cfg.Region
- }
- return cfg.AddOnFluentd.RepositoryBusyboxRegion
-}
-
-func (cfg *Config) validateAddOnFluentd() error {
- if !cfg.IsEnabledAddOnFluentd() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnFluentd.Enable true but no node group is enabled")
- }
- if cfg.AddOnFluentd.Namespace == "" {
- cfg.AddOnFluentd.Namespace = cfg.Name + "-fluentd"
- }
- return nil
-}
diff --git a/eksconfig/add-on-irsa-fargate.go b/eksconfig/add-on-irsa-fargate.go
deleted file mode 100644
index 0ad9ac4bc..000000000
--- a/eksconfig/add-on-irsa-fargate.go
+++ /dev/null
@@ -1,152 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
- "path"
- "path/filepath"
- "strings"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnIRSAFargate defines parameters for EKS cluster
-// add-on "IAM Roles for Service Accounts (IRSA)" with Fargate.
-type AddOnIRSAFargate struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // S3Dir is the S3 directory to store all test results.
- // It is under the bucket "eksconfig.Config.S3BucketName".
- S3Dir string `json:"s3-dir"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // RepositoryAccountID is the account ID for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryAccountID string `json:"repository-account-id,omitempty"`
- // RepositoryRegion is the ECR repository region to pull from.
- RepositoryRegion string `json:"repository-region,omitempty"`
- // RepositoryName is the repositoryName for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryName string `json:"repository-name,omitempty"`
- // RepositoryImageTag is the image tag for tester ECR image.
- // e.g. "latest" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester:latest"
- RepositoryImageTag string `json:"repository-image-tag,omitempty"`
-
- // RoleName is the role name for IRSA.
- RoleName string `json:"role-name"`
- // RoleARN is the role ARN for IRSA.
- RoleARN string `json:"role-arn"`
- // RoleServicePrincipals is the Fargate role Service Principals
- RoleServicePrincipals []string `json:"role-service-principals"`
- // RoleManagedPolicyARNs is IRSA role managed policy ARNs.
- // ref. https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/
- RoleManagedPolicyARNs []string `json:"role-managed-policy-arns"`
- RoleCFNStackID string `json:"role-cfn-stack-id" read-only:"true"`
- RoleCFNStackYAMLPath string `json:"role-cfn-stack-yaml-path" read-only:"true"`
- RoleCFNStackYAMLS3Key string `json:"role-cfn-stack-yaml-s3-key" read-only:"true"`
-
- // S3Key is the S3 key to write for IRSA tests.
- S3Key string `json:"s3-key"`
-
- // ProfileName is the profile name for Fargate.
- ProfileName string `json:"profile-name"`
-}
-
-// EnvironmentVariablePrefixAddOnIRSAFargate is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnIRSAFargate = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_IRSA_FARGATE_"
-
-// IsEnabledAddOnIRSAFargate returns true if "AddOnIRSAFargate" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnIRSAFargate() bool {
- if cfg.AddOnIRSAFargate == nil {
- return false
- }
- if cfg.AddOnIRSAFargate.Enable {
- return true
- }
- cfg.AddOnIRSAFargate = nil
- return false
-}
-
-func getDefaultAddOnIRSAFargate() *AddOnIRSAFargate {
- return &AddOnIRSAFargate{
- Enable: false,
- }
-}
-
-func (cfg *Config) GetAddOnIRSAFargateRepositoryRegion() string {
- if !cfg.IsEnabledAddOnIRSAFargate() {
- return cfg.Region
- }
- return cfg.AddOnIRSAFargate.RepositoryRegion
-}
-
-func (cfg *Config) validateAddOnIRSAFargate() error {
- if !cfg.IsEnabledAddOnIRSAFargate() {
- return nil
- }
-
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnIRSAFargate.Enable true but no node group is enabled")
- }
-
- if cfg.VersionValue < 1.14 {
- return fmt.Errorf("Version %q not supported for AddOnIRSAFargate", cfg.Version)
- }
-
- if cfg.AddOnIRSAFargate.S3Dir == "" {
- cfg.AddOnIRSAFargate.S3Dir = path.Join(cfg.Name, "add-on-irsa-fargate")
- }
-
- if cfg.AddOnIRSAFargate.Namespace == "" {
- cfg.AddOnIRSAFargate.Namespace = cfg.Name + "-irsa-fargate"
- }
-
- if cfg.AddOnIRSAFargate.RepositoryAccountID == "" {
- return errors.New("AddOnIRSAFargate.RepositoryAccountID empty")
- }
- if cfg.AddOnIRSAFargate.RepositoryRegion == "" {
- cfg.AddOnIRSAFargate.RepositoryRegion = cfg.Region
- }
- if cfg.AddOnIRSAFargate.RepositoryName == "" {
- return errors.New("AddOnIRSAFargate.RepositoryName empty")
- }
- if cfg.AddOnIRSAFargate.RepositoryImageTag == "" {
- return errors.New("AddOnIRSAFargate.RepositoryImageTag empty")
- }
-
- if cfg.AddOnIRSAFargate.RoleName == "" {
- cfg.AddOnIRSAFargate.RoleName = cfg.Name + "add-on-irsa-fargate-role"
- }
- if cfg.AddOnIRSAFargate.RoleCFNStackYAMLPath == "" {
- cfg.AddOnIRSAFargate.RoleCFNStackYAMLPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + ".add-on-irsa-fargate.role.cfn.yaml"
- }
- if cfg.AddOnIRSAFargate.RoleCFNStackYAMLS3Key == "" {
- cfg.AddOnIRSAFargate.RoleCFNStackYAMLS3Key = path.Join(
- cfg.AddOnIRSAFargate.S3Dir,
- filepath.Base(cfg.AddOnIRSAFargate.RoleCFNStackYAMLPath),
- )
- }
-
- if cfg.AddOnIRSAFargate.S3Key == "" {
- cfg.AddOnIRSAFargate.S3Key = path.Join(cfg.AddOnIRSAFargate.S3Dir, "irsa-fargate-s3-key")
- }
- // do not prefix with "eks-"
- // e.g. "The fargate profile name starts with the reserved prefix: 'eks-'."
- if cfg.AddOnIRSAFargate.ProfileName == "" {
- cfg.AddOnIRSAFargate.ProfileName = cfg.Name + "-add-on-irsa-fargate-profile"
- }
- if strings.HasPrefix(cfg.AddOnIRSAFargate.ProfileName, "eks-") {
- cfg.AddOnIRSAFargate.ProfileName = strings.Replace(cfg.AddOnIRSAFargate.ProfileName, "eks-", "", 1)
- }
-
- return nil
-}
diff --git a/eksconfig/add-on-irsa.go b/eksconfig/add-on-irsa.go
deleted file mode 100644
index 99d3ff59c..000000000
--- a/eksconfig/add-on-irsa.go
+++ /dev/null
@@ -1,150 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
- "path"
- "path/filepath"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnIRSA defines parameters for EKS cluster
-// add-on "IAM Roles for Service Accounts (IRSA)".
-type AddOnIRSA struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // S3Dir is the S3 directory to store all test results.
- // It is under the bucket "eksconfig.Config.S3BucketName".
- S3Dir string `json:"s3-dir"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // RepositoryAccountID is the account ID for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryAccountID string `json:"repository-account-id,omitempty"`
- // RepositoryRegion is the ECR repository region to pull from.
- RepositoryRegion string `json:"repository-region,omitempty"`
- // RepositoryName is the repositoryName for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryName string `json:"repository-name,omitempty"`
- // RepositoryImageTag is the image tag for tester ECR image.
- // e.g. "latest" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester:latest"
- RepositoryImageTag string `json:"repository-image-tag,omitempty"`
-
- // RoleName is the role name for IRSA.
- RoleName string `json:"role-name"`
- // RoleARN is the role ARN for IRSA.
- RoleARN string `json:"role-arn"`
- RoleCFNStackID string `json:"role-cfn-stack-id" read-only:"true"`
- RoleCFNStackYAMLPath string `json:"role-cfn-stack-yaml-path" read-only:"true"`
- RoleCFNStackYAMLS3Key string `json:"role-cfn-stack-yaml-s3-key" read-only:"true"`
-
- // S3Key is the S3 key to write for IRSA tests.
- S3Key string `json:"s3-key"`
-
- // DeploymentReplicas is the number of Deployment replicas.
- DeploymentReplicas int32 `json:"deployment-replicas"`
- // DeploymentResultPath is the output of "Deployment" run.
- DeploymentResultPath string `json:"deployment-result-path"`
- // DeploymentTook is the duration that took for Deployment resource.
- DeploymentTook time.Duration `json:"deployment-took,omitempty" read-only:"true"`
- // DeploymentTookString is the duration that took for Deployment resource.
- DeploymentTookString string `json:"deployment-took-string,omitempty" read-only:"true"`
-}
-
-// EnvironmentVariablePrefixAddOnIRSA is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnIRSA = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_IRSA_"
-
-// IsEnabledAddOnIRSA returns true if "AddOnIRSA" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnIRSA() bool {
- if cfg.AddOnIRSA == nil {
- return false
- }
- if cfg.AddOnIRSA.Enable {
- return true
- }
- cfg.AddOnIRSA = nil
- return false
-}
-
-func getDefaultAddOnIRSA() *AddOnIRSA {
- return &AddOnIRSA{
- Enable: false,
- DeploymentReplicas: 1,
- }
-}
-
-func (cfg *Config) GetAddOnIRSARepositoryRegion() string {
- if !cfg.IsEnabledAddOnIRSA() {
- return cfg.Region
- }
- return cfg.AddOnIRSA.RepositoryRegion
-}
-
-func (cfg *Config) validateAddOnIRSA() error {
- if !cfg.IsEnabledAddOnIRSA() {
- return nil
- }
-
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnIRSA.Enable true but no node group is enabled")
- }
-
- if cfg.VersionValue < 1.14 {
- return fmt.Errorf("Version %q not supported for AddOnIRSA", cfg.Version)
- }
-
- if cfg.AddOnIRSA.S3Dir == "" {
- cfg.AddOnIRSA.S3Dir = path.Join(cfg.Name, "add-on-irsa")
- }
-
- if cfg.AddOnIRSA.Namespace == "" {
- cfg.AddOnIRSA.Namespace = cfg.Name + "-irsa"
- }
-
- if cfg.AddOnIRSA.RepositoryAccountID == "" {
- return errors.New("AddOnIRSA.RepositoryAccountID empty")
- }
- if cfg.AddOnIRSA.RepositoryRegion == "" {
- cfg.AddOnIRSA.RepositoryRegion = cfg.Region
- }
- if cfg.AddOnIRSA.RepositoryName == "" {
- return errors.New("AddOnIRSA.RepositoryName empty")
- }
- if cfg.AddOnIRSA.RepositoryImageTag == "" {
- return errors.New("AddOnIRSA.RepositoryImageTag empty")
- }
-
- if cfg.AddOnIRSA.RoleName == "" {
- cfg.AddOnIRSA.RoleName = cfg.Name + "-add-on-irsa-role"
- }
- if cfg.AddOnIRSA.RoleCFNStackYAMLPath == "" {
- cfg.AddOnIRSA.RoleCFNStackYAMLPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + ".add-on-irsa.role.cfn.yaml"
- }
- if cfg.AddOnIRSA.RoleCFNStackYAMLS3Key == "" {
- cfg.AddOnIRSA.RoleCFNStackYAMLS3Key = path.Join(
- cfg.AddOnIRSA.S3Dir,
- filepath.Base(cfg.AddOnIRSA.RoleCFNStackYAMLPath),
- )
- }
-
- if cfg.AddOnIRSA.S3Key == "" {
- cfg.AddOnIRSA.S3Key = path.Join(cfg.AddOnIRSA.S3Dir, "irsa-s3-key")
- }
-
- if cfg.AddOnIRSA.DeploymentResultPath == "" {
- cfg.AddOnIRSA.DeploymentResultPath = filepath.Join(filepath.Dir(cfg.ConfigPath), cfg.Name+"-deployment-irsa-result.log")
- }
- return nil
-}
diff --git a/eksconfig/add-on-jobs-echo.go b/eksconfig/add-on-jobs-echo.go
deleted file mode 100644
index e6294ec5e..000000000
--- a/eksconfig/add-on-jobs-echo.go
+++ /dev/null
@@ -1,99 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnJobsEcho defines parameters for EKS cluster
-// add-on Job with echo.
-type AddOnJobsEcho struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // RepositoryBusyboxAccountID is the account ID for tester ECR image.
- // e.g. "busybox" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/busybox"
- RepositoryBusyboxAccountID string `json:"repository-busybox-account-id,omitempty"`
- // RepositoryBusyboxRegion is the ECR repository region to pull from.
- RepositoryBusyboxRegion string `json:"repository-busybox-region,omitempty"`
- // RepositoryBusyboxName is the repositoryName for tester ECR image.
- // e.g. "busybox" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/busybox"
- RepositoryBusyboxName string `json:"repository-busybox-name,omitempty"`
- // RepositoryBusyboxImageTag is the image tag for tester ECR image.
- // e.g. "latest" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/busybox:latest"
- RepositoryBusyboxImageTag string `json:"repository-busybox-image-tag,omitempty"`
-
- // Completes is the desired number of successfully finished pods.
- Completes int `json:"completes"`
- // Parallels is the the maximum desired number of pods the
- // job should run at any given time.
- Parallels int `json:"parallels"`
- // EchoSize is the job object size in bytes.
- // "Request entity too large: limit is 3145728" (3.1 MB).
- // "The Job "echo" is invalid: metadata.annotations:
- // Too long: must have at most 262144 characters". (0.26 MB)
- EchoSize int `json:"echo-size"`
-}
-
-// EnvironmentVariablePrefixAddOnJobsEcho is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnJobsEcho = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_JOBS_ECHO_"
-
-// IsEnabledAddOnJobsEcho returns true if "AddOnJobsEcho" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnJobsEcho() bool {
- if cfg.AddOnJobsEcho == nil {
- return false
- }
- if cfg.AddOnJobsEcho.Enable {
- return true
- }
- cfg.AddOnJobsEcho = nil
- return false
-}
-
-func getDefaultAddOnJobsEcho() *AddOnJobsEcho {
- return &AddOnJobsEcho{
- Enable: false,
- Completes: 10,
- Parallels: 10,
- EchoSize: 100 * 1024, // 100 KB
-
- // writes total 100 MB data to etcd
- // Completes: 1000,
- // Parallels: 100,
- // EchoSize: 100 * 1024, // 100 KB
- }
-}
-
-func (cfg *Config) GetAddOnJobsEchoRepositoryBusyboxRegion() string {
- if !cfg.IsEnabledAddOnJobsEcho() {
- return cfg.Region
- }
- return cfg.AddOnJobsEcho.RepositoryBusyboxRegion
-}
-
-func (cfg *Config) validateAddOnJobsEcho() error {
- if !cfg.IsEnabledAddOnJobsEcho() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnJobsEcho.Enable true but no node group is enabled")
- }
- if cfg.AddOnJobsEcho.Namespace == "" {
- cfg.AddOnJobsEcho.Namespace = cfg.Name + "-jobs-echo"
- }
- if cfg.AddOnJobsEcho.EchoSize > 250000 {
- return fmt.Errorf("echo size limit is 0.25 MB, got %d", cfg.AddOnJobsEcho.EchoSize)
- }
- return nil
-}
diff --git a/eksconfig/add-on-jobs-pi.go b/eksconfig/add-on-jobs-pi.go
deleted file mode 100644
index 5242a096b..000000000
--- a/eksconfig/add-on-jobs-pi.go
+++ /dev/null
@@ -1,65 +0,0 @@
-package eksconfig
-
-import (
- "errors"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnJobsPi defines parameters for EKS cluster
-// add-on Job with Perl.
-type AddOnJobsPi struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // Completes is the desired number of successfully finished pods.
- Completes int `json:"completes"`
- // Parallels is the the maximum desired number of pods the
- // job should run at any given time.
- Parallels int `json:"parallels"`
-}
-
-// EnvironmentVariablePrefixAddOnJobsPi is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnJobsPi = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_JOBS_PI_"
-
-// IsEnabledAddOnJobsPi returns true if "AddOnJobsPi" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnJobsPi() bool {
- if cfg.AddOnJobsPi == nil {
- return false
- }
- if cfg.AddOnJobsPi.Enable {
- return true
- }
- cfg.AddOnJobsPi = nil
- return false
-}
-
-func getDefaultAddOnJobsPi() *AddOnJobsPi {
- return &AddOnJobsPi{
- Enable: false,
- Completes: 10,
- Parallels: 10,
- }
-}
-
-func (cfg *Config) validateAddOnJobsPi() error {
- if !cfg.IsEnabledAddOnJobsPi() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnJobsPi.Enable true but no node group is enabled")
- }
- if cfg.AddOnJobsPi.Namespace == "" {
- cfg.AddOnJobsPi.Namespace = cfg.Name + "-jobs-pi"
- }
- return nil
-}
diff --git a/eksconfig/add-on-jupyter-hub.go b/eksconfig/add-on-jupyter-hub.go
deleted file mode 100644
index dc369c64f..000000000
--- a/eksconfig/add-on-jupyter-hub.go
+++ /dev/null
@@ -1,105 +0,0 @@
-package eksconfig
-
-import (
- "encoding/hex"
- "errors"
- "fmt"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/pkg/randutil"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/service/eks"
-)
-
-// AddOnJupyterHub defines parameters for EKS cluster
-// add-on Jupyter Hub.
-// ref. https://zero-to-jupyterhub.readthedocs.io/en/latest/index.html
-type AddOnJupyterHub struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // ProxySecretToken is 32-byte hexadecimal encoded secret token string.
- // e.g. "openssl rand -hex 32"
- ProxySecretToken string `json:"proxy-secret-token"`
-
- // NLBARN is the ARN of the NLB created from the service.
- NLBARN string `json:"nlb-arn" read-only:"true"`
- // NLBName is the name of the NLB created from the service.
- NLBName string `json:"nlb-name" read-only:"true"`
- // URL is the host name for Jupyter Hub service.
- URL string `json:"url" read-only:"true"`
-}
-
-// EnvironmentVariablePrefixAddOnJupyterHub is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnJupyterHub = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_JUPYTER_HUB_"
-
-// IsEnabledAddOnJupyterHub returns true if "AddOnJupyterHub" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnJupyterHub() bool {
- if cfg.AddOnJupyterHub == nil {
- return false
- }
- if cfg.AddOnJupyterHub.Enable {
- return true
- }
- cfg.AddOnJupyterHub = nil
- return false
-}
-
-func getDefaultAddOnJupyterHub() *AddOnJupyterHub {
- return &AddOnJupyterHub{
- Enable: false,
- }
-}
-
-func (cfg *Config) validateAddOnJupyterHub() error {
- if !cfg.IsEnabledAddOnJupyterHub() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnJupyterHub.Enable true but no node group is enabled")
- }
-
- gpuFound := false
- if cfg.IsEnabledAddOnNodeGroups() {
- for _, cur := range cfg.AddOnNodeGroups.ASGs {
- if cur.AMIType == ec2config.AMITypeAL2X8664GPU {
- gpuFound = true
- break
- }
- }
- }
- if !gpuFound && cfg.IsEnabledAddOnManagedNodeGroups() {
- for _, cur := range cfg.AddOnManagedNodeGroups.MNGs {
- if cur.AMIType == eks.AMITypesAl2X8664Gpu {
- gpuFound = true
- break
- }
- }
- }
- if !gpuFound {
- return errors.New("AddOnJupyterHub requires GPU AMI")
- }
-
- if cfg.AddOnJupyterHub.Namespace == "" {
- cfg.AddOnJupyterHub.Namespace = cfg.Name + "-jupyter-hub"
- }
-
- if cfg.AddOnJupyterHub.ProxySecretToken == "" {
- cfg.AddOnJupyterHub.ProxySecretToken = randutil.Hex(32)
- }
- _, err := hex.DecodeString(cfg.AddOnJupyterHub.ProxySecretToken)
- if err != nil {
- return fmt.Errorf("cannot hex decode AddOnJupyterHub.ProxySecretToken %q", err)
- }
-
- return nil
-}
diff --git a/eksconfig/add-on-kubeflow.go b/eksconfig/add-on-kubeflow.go
deleted file mode 100644
index 900b37bfb..000000000
--- a/eksconfig/add-on-kubeflow.go
+++ /dev/null
@@ -1,80 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "path/filepath"
- "runtime"
- "strings"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnKubeflow defines parameters for EKS cluster
-// add-on Kubeflow.
-// ref. https://www.kubeflow.org/docs/aws/deploy/install-kubeflow/
-type AddOnKubeflow struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // KfctlPath is the path to download the "kfctl".
- KfctlPath string `json:"kfctl-path,omitempty"`
- // KfctlDownloadURL is the download URL to download "kfctl" binary from.
- // ref. https://github.com/kubeflow/kfctl/releases
- KfctlDownloadURL string `json:"kfctl-download-url,omitempty"`
-
- // BaseDir is the base directory where you want to store one or more
- // Kubeflow deployments.
- BaseDir string `json:"base-dir"`
- KfDir string `json:"kf-dir" read-only:"true"`
- // KfctlConfigPath is the path to write "kfctl" configuration.
- // The existing configuration file is overwritten.
- KfctlConfigPath string `json:"kfctl-config-path" read-only:"true"`
-}
-
-// EnvironmentVariablePrefixAddOnKubeflow is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnKubeflow = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_KUBEFLOW_"
-
-// IsEnabledAddOnKubeflow returns true if "AddOnKubeflow" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnKubeflow() bool {
- if cfg.AddOnKubeflow == nil {
- return false
- }
- if cfg.AddOnKubeflow.Enable {
- return true
- }
- cfg.AddOnKubeflow = nil
- return false
-}
-
-func getDefaultAddOnKubeflow() *AddOnKubeflow {
- addOn := &AddOnKubeflow{
- Enable: false,
- KfctlPath: "/tmp/kfctl-test-v1.0.2",
- KfctlDownloadURL: "https://github.com/kubeflow/kfctl/releases/download/v1.0.2/kfctl_v1.0.2-0-ga476281_linux.tar.gz",
- }
- if runtime.GOOS == "darwin" {
- addOn.KfctlDownloadURL = strings.Replace(addOn.KfctlDownloadURL, "linux", "darwin", -1)
- }
- return addOn
-}
-
-func (cfg *Config) validateAddOnKubeflow() error {
- if !cfg.IsEnabledAddOnKubeflow() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnKubeflow.Enable true but no node group is enabled")
- }
- if cfg.AddOnKubeflow.BaseDir == "" {
- cfg.AddOnKubeflow.BaseDir = filepath.Join(filepath.Dir(cfg.ConfigPath), cfg.Name+"-kubeflow")
- }
- cfg.AddOnKubeflow.KfDir = filepath.Join(cfg.AddOnKubeflow.BaseDir, cfg.Name)
- cfg.AddOnKubeflow.KfctlConfigPath = filepath.Join(cfg.AddOnKubeflow.KfDir, "kfctl_aws.yaml")
- return nil
-}
diff --git a/eksconfig/add-on-kubernetes-dashboard.go b/eksconfig/add-on-kubernetes-dashboard.go
deleted file mode 100644
index 41199159d..000000000
--- a/eksconfig/add-on-kubernetes-dashboard.go
+++ /dev/null
@@ -1,84 +0,0 @@
-package eksconfig
-
-import (
- "errors"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnKubernetesDashboard defines parameters for EKS cluster
-// add-on Kubernetes Dashboard.
-// ref. https://docs.aws.amazon.com/eks/latest/userguide/dashboard-tutorial.html
-type AddOnKubernetesDashboard struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // AuthenticationToken is the authentication token for eks-admin service account.
- AuthenticationToken string `json:"authentication-token,omitempty" read-only:"true"`
- // URL is the host name for Kubernetes Dashboard service.
- URL string `json:"url" read-only:"true"`
-
- // KubectlProxyPID is the PID for kubectl proxy.
- KubectlProxyPID int `json:"kubectl-proxy-pid" read-only:"true"`
-}
-
-// EnvironmentVariablePrefixAddOnKubernetesDashboard is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnKubernetesDashboard = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_KUBERNETES_DASHBOARD_"
-
-// IsEnabledAddOnKubernetesDashboard returns true if "AddOnKubernetesDashboard" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnKubernetesDashboard() bool {
- if cfg.AddOnKubernetesDashboard == nil {
- return false
- }
- if cfg.AddOnKubernetesDashboard.Enable {
- return true
- }
- cfg.AddOnKubernetesDashboard = nil
- return false
-}
-
-func (cfg *Config) getAddOnKubernetesDashboardAuthenticationToken() string {
- if cfg.AddOnKubernetesDashboard == nil {
- return ""
- }
- return cfg.AddOnKubernetesDashboard.AuthenticationToken
-}
-
-func (cfg *Config) getAddOnKubernetesDashboardURL() string {
- if cfg.AddOnKubernetesDashboard == nil {
- return ""
- }
- return cfg.AddOnKubernetesDashboard.URL
-}
-
-func getDefaultAddOnKubernetesDashboard() *AddOnKubernetesDashboard {
- return &AddOnKubernetesDashboard{
- Enable: false,
- URL: defaultKubernetesDashboardURL,
- }
-}
-
-// ref. https://docs.aws.amazon.com/eks/latest/userguide/dashboard-tutorial.html
-const defaultKubernetesDashboardURL = "http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/#/login"
-
-func (cfg *Config) validateAddOnKubernetesDashboard() error {
- if !cfg.IsEnabledAddOnKubernetesDashboard() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnKubernetesDashboard.Enable true but no node group is enabled")
- }
- if !cfg.IsEnabledAddOnMetricsServer() {
- return errors.New("AddOnKubernetesDashboard.Enable true but AddOnMetricsServer.Enable false")
- }
- if cfg.AddOnKubernetesDashboard.URL == "" {
- cfg.AddOnKubernetesDashboard.URL = defaultKubernetesDashboardURL
- }
- return nil
-}
diff --git a/eksconfig/add-on-managed-node-groups.go b/eksconfig/add-on-managed-node-groups.go
deleted file mode 100644
index f089e74cd..000000000
--- a/eksconfig/add-on-managed-node-groups.go
+++ /dev/null
@@ -1,489 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
- "path/filepath"
- "strconv"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/service/eks"
-)
-
-// AddOnManagedNodeGroups defines parameters for EKS "Managed Node Group" creation.
-// ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
-// ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html
-type AddOnManagedNodeGroups struct {
- // Enable is true to auto-create a managed node group.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // FetchLogs is true to fetch logs from remote nodes using SSH.
- FetchLogs bool `json:"fetch-logs"`
-
- Role *Role `json:"role"`
-
- // RequestHeaderKey defines EKS managed node group create cluster request header key.
- RequestHeaderKey string `json:"request-header-key,omitempty"`
- // RequestHeaderValue defines EKS managed node group create cluster request header value.
- RequestHeaderValue string `json:"request-header-value,omitempty"`
- // ResolverURL defines an AWS resolver endpoint for EKS API.
- // Must be left empty to use production EKS managed node group service.
- ResolverURL string `json:"resolver-url"`
- // SigningName is the EKS managed node group create request signing name.
- SigningName string `json:"signing-name"`
-
- // LogsDir is set to specify the target directory to store all remote log files.
- // If empty, it stores in the same directory as "ConfigPath".
- LogsDir string `json:"logs-dir,omitempty"`
- // LogsTarGzPath is the .tar.gz archived file for "LogsDir".
- LogsTarGzPath string `json:"logs-tar-gz-path"`
- // MNGs maps from EKS Managed Node Group name to "MNG".
- // "GetRef.Name" is the reserved key and MNG name from eksconfig.Config.Name.
- MNGs map[string]MNG `json:"mngs,omitempty"`
-}
-
-// MNG represents parameters for one EKS "Managed Node Group".
-type MNG struct {
- // Name is the name of the managed node group.
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
- // ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html
- Name string `json:"name,omitempty"`
- // ASGName is the ASG name from a created managed node group.
- ASGName string `json:"asg-name,omitempty" read-only:"true"`
-
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // RemoteAccessUserName is the user name for managed node group SSH access.
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
- // ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html
- RemoteAccessUserName string `json:"remote-access-user-name,omitempty"`
- // Tags defines EKS managed node group create tags.
- Tags map[string]string `json:"tags,omitempty"`
- // ReleaseVersion is the AMI version of the Amazon EKS-optimized AMI for the node group.
- // The version may differ from EKS "cluster" version.
- // e.g. "1.16.8-20200609"
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
- // ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/eks-linux-ami-versions.html
- ReleaseVersion string `json:"release-version,omitempty"`
- ReleaseVersionValue float64 `json:"release-version-value" read-only:"true"`
-
- // AMIType is the AMI type for the node group.
- // Allowed values are AL2_x86_64, AL2_x86_64_GPU and AL2_ARM_64.
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
- // ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-amitype
- AMIType string `json:"ami-type,omitempty"`
-
- // InstanceTypes is the EC2 instance types for the node instances.
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
- // ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html
- InstanceTypes []string `json:"instance-types,omitempty"`
- // VolumeSize is the node volume size.
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
- // ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html
- VolumeSize int `json:"volume-size,omitempty"`
-
- // ASGMinSize is the minimum size of Node Group Auto Scaling Group.
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
- // ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html
- ASGMinSize int `json:"asg-min-size,omitempty"`
- // ASGMaxSize is the maximum size of Node Group Auto Scaling Group.
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
- // ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html
- ASGMaxSize int `json:"asg-max-size,omitempty"`
- // ASGDesiredCapacity is the desired capacity of Node Group ASG.
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
- // ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html
- ASGDesiredCapacity int `json:"asg-desired-capacity,omitempty"`
-
- // CreateRequested is true if "CreateNodegroupRequest" has been sent.
- CreateRequested bool `json:"create-requested" read-only:"true"`
-
- // PhysicalID is the Physical ID for the created "AWS::EKS::Nodegroup".
- PhysicalID string `json:"physical-id" read-only:"true"`
-
- // RemoteAccessSecurityGroupID is the security group ID for the MNG.
- // Returned from EKS MNG API.
- RemoteAccessSecurityGroupID string `json:"remote-access-security-group-id" read-only:"true"`
-
- // Status is the current status of EKS "Managed Node Group".
- Status string `json:"status" read-only:"true"`
- // Instances maps an instance ID to an EC2 instance object for the node group.
- Instances map[string]ec2config.Instance `json:"instances" read-only:"true"`
- // Logs maps each instance ID to a list of log file paths fetched via SSH access.
- Logs map[string][]string `json:"logs" read-only:"true"`
-
- // ScaleUpdates configures MNG scale update.
- ScaleUpdates []MNGScaleUpdate `json:"scale-updates,omitempty"`
-
- // VersionUpgrade configures MNG version upgarde.
- VersionUpgrade *MNGVersionUpgrade `json:"version-upgrade,omitempty"`
-}
-
-// MNGScaleUpdate contains the minimum, maximum, and desired node counts for a nodegroup.
-// ref, https://docs.aws.amazon.com/cli/latest/reference/eks/update-nodegroup-config.html
-type MNGScaleUpdate struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- // InitialWait is the wait time before triggering version upgrades.
- // All managed node group upgrades are triggered after all existing
- // add-on installation is complete.
- InitialWait time.Duration `json:"initial-wait" read-only:"true"`
- InitialWaitString string `json:"initial-wait-string"`
-
- ID string `json:"id"`
- ASGMinSize int64 `json:"asg-min-size,omitempty"`
- ASGMaxSize int64 `json:"asg-max-size,omitempty"`
- ASGDesiredCapacity int64 `json:"asg-desired-capacity,omitempty"`
-}
-
-// MNGVersionUpgrade defines parameters
-// for EKS managed node group version upgrade add-on.
-// ref. https://docs.aws.amazon.com/cli/latest/reference/eks/update-nodegroup-version.html
-type MNGVersionUpgrade struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // InitialWait is the wait time before triggering version upgrades.
- // All managed node group upgrades are triggered after all existing
- // add-on installation is complete.
- InitialWait time.Duration `json:"initial-wait" read-only:"true"`
- InitialWaitString string `json:"initial-wait-string"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
-
- // Version is the target version of EKS managed node group.
- // This cannot be empty. Must be provided by the user.
- // The value is passed via "aws eks update-nodegroup-version --kubernetes-version".
- // e.g. Upgrade to "Version" == "1.17" when Parameters.Version is "1.16"
- // that has created "1.16" MNG by default.
- Version string `json:"version"`
- VersionValue float64 `json:"version-value" read-only:"true"`
-}
-
-const (
- // AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_PREFIX is the environment variable prefix used for "eksconfig".
- AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_PREFIX = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_MANAGED_NODE_GROUPS_"
- AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_PREFIX = AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_PREFIX + "ROLE_"
-)
-
-// IsEnabledAddOnManagedNodeGroups returns true if "AddOnManagedNodeGroups" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnManagedNodeGroups() bool {
- if cfg.AddOnManagedNodeGroups == nil {
- return false
- }
- if cfg.AddOnManagedNodeGroups.Enable {
- return len(cfg.AddOnManagedNodeGroups.MNGs) > 0
- }
- cfg.AddOnManagedNodeGroups = nil
- return false
-}
-
-func getDefaultAddOnManagedNodeGroups(name string) *AddOnManagedNodeGroups {
- return &AddOnManagedNodeGroups{
- Enable: false,
- FetchLogs: false,
- SigningName: "eks",
- Role: getDefaultRole(),
- LogsDir: "", // to be auto-generated
- MNGs: map[string]MNG{
- name + "-mng-cpu": {
- Name: name + "-mng-cpu",
- RemoteAccessUserName: "ec2-user", // assume Amazon Linux 2
- ReleaseVersion: "", // to be auto-filled by EKS API
- AMIType: eks.AMITypesAl2X8664,
- InstanceTypes: []string{DefaultNodeInstanceTypeCPU},
- VolumeSize: DefaultNodeVolumeSize,
- ASGMinSize: 1,
- ASGMaxSize: 1,
- ASGDesiredCapacity: 1,
- VersionUpgrade: &MNGVersionUpgrade{Enable: false},
- },
- },
- }
-}
-
-func (cfg *Config) validateAddOnManagedNodeGroups() error {
- if !cfg.IsEnabledAddOnManagedNodeGroups() {
- return nil
- }
-
- n := len(cfg.AddOnManagedNodeGroups.MNGs)
- if n == 0 {
- return errors.New("empty MNGs")
- }
- if n > MNGsMaxLimit {
- return fmt.Errorf("MNGs %d exceeds maximum number of MNGs which is %d", n, MNGsMaxLimit)
- }
-
- if cfg.VersionValue < 1.14 {
- return fmt.Errorf("Version %q not supported for AddOnManagedNodeGroups", cfg.Version)
- }
-
- if cfg.AddOnManagedNodeGroups.LogsDir == "" {
- cfg.AddOnManagedNodeGroups.LogsDir = filepath.Join(filepath.Dir(cfg.ConfigPath), cfg.Name+"-logs-mngs")
- }
- if cfg.AddOnManagedNodeGroups.LogsTarGzPath == "" {
- cfg.AddOnManagedNodeGroups.LogsTarGzPath = filepath.Join(filepath.Dir(cfg.ConfigPath), cfg.Name+"-logs-mngs.tar.gz")
- }
- if !strings.HasSuffix(cfg.AddOnManagedNodeGroups.LogsTarGzPath, ".tar.gz") {
- return fmt.Errorf("AddOnManagedNodeGroups.LogsTarGzPath %q must end with .tar.gz", cfg.AddOnManagedNodeGroups.LogsTarGzPath)
- }
-
- switch cfg.AddOnManagedNodeGroups.Role.Create {
- case true: // need create one, or already created
- if cfg.AddOnManagedNodeGroups.Role.Name == "" {
- cfg.AddOnManagedNodeGroups.Role.Name = cfg.Name + "-mng-role"
- }
- if len(cfg.AddOnManagedNodeGroups.Role.ServicePrincipals) > 0 {
- /*
- (InvalidParameterException: Following required service principals [ec2.amazonaws.com] were not found in the trust relationships of nodeRole arn:aws:iam::...:role/test-mng-role
- {
- ClusterName: "test",
- Message_: "Following required service principals [ec2.amazonaws.com] were not found in the trust relationships of nodeRole arn:aws:iam::...:role/test-mng-role",
- NodegroupName: "test-mng-cpu"
- })
- */
- found := false
- for _, pv := range cfg.AddOnManagedNodeGroups.Role.ServicePrincipals {
- if pv == "ec2.amazonaws.com" || pv == "ec2.amazonaws.com.cn" {
- found = true
- break
- }
- }
- if !found {
- return fmt.Errorf("AddOnManagedNodeGroups.Role.ServicePrincipals %q must include 'ec2.amazonaws.com' or 'ec2.amazonaws.com.cn'", cfg.AddOnManagedNodeGroups.Role.ServicePrincipals)
- }
- }
-
- case false: // use existing one
- if cfg.AddOnManagedNodeGroups.Role.ARN == "" {
- return fmt.Errorf("AddOnManagedNodeGroups.Role.Create false; expect non-empty RoleARN but got %q", cfg.AddOnManagedNodeGroups.Role.ARN)
- }
- if cfg.AddOnManagedNodeGroups.Role.Name == "" {
- cfg.AddOnManagedNodeGroups.Role.Name = getNameFromARN(cfg.AddOnManagedNodeGroups.Role.ARN)
- }
- if cfg.IsEnabledAddOnStresserRemote() {
- return errors.New("'AddOnStresserRemote.Enable == true' requires 'AddOnManagedNodeGroups.Role.Create == true' but got 'false'")
- }
- }
- if cfg.AddOnManagedNodeGroups.Role.PolicyName == "" {
- cfg.AddOnManagedNodeGroups.Role.PolicyName = cfg.Name + "-managed-node-group-policy"
- }
-
- names, processed := make(map[string]struct{}), make(map[string]MNG)
- for k, cur := range cfg.AddOnManagedNodeGroups.MNGs {
- k = strings.ReplaceAll(k, "GetRef.Name", cfg.Name)
- cur.Name = strings.ReplaceAll(cur.Name, "GetRef.Name", cfg.Name)
-
- if cur.Name == "" {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q].Name is empty", k)
- }
- if k != cur.Name {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q].Name has different Name field %q", k, cur.Name)
- }
- _, ok := names[cur.Name]
- if !ok {
- names[cur.Name] = struct{}{}
- } else {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q].Name %q is redundant", k, cur.Name)
- }
- if cfg.IsEnabledAddOnNodeGroups() {
- _, ok = cfg.AddOnNodeGroups.ASGs[cur.Name]
- if ok {
- return fmt.Errorf("MNGs[%q] name already exists (conflict) in AddOnNodeGroups.ASGs", cur.Name)
- }
- }
-
- if cur.ReleaseVersion != "" {
- // e.g. "1.16.8-20200609"
- ss := strings.Split(cur.ReleaseVersion, ".")
- if len(ss) > 2 {
- sv := strings.Join(ss[:2], ".")
- var err error
- cur.ReleaseVersionValue, err = strconv.ParseFloat(sv, 64)
- if err != nil {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q] invalid ReleaseVersion %q (%q, %v)", cur.Name, cur.ReleaseVersion, sv, err)
- }
- }
- }
-
- if len(cur.ScaleUpdates) > 0 {
- for idx := range cur.ScaleUpdates {
- if !cur.ScaleUpdates[idx].Enable {
- continue
- }
- var err error
- if cur.ScaleUpdates[idx].InitialWaitString != "" {
- cur.ScaleUpdates[idx].InitialWait, err = time.ParseDuration(cur.ScaleUpdates[idx].InitialWaitString)
- if err != nil {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q] invalid cur.ScaleUpdates[%d].InitialWaitString %q (%v)", cur.Name, idx, cur.VersionUpgrade.InitialWaitString, err)
- }
- }
- if cur.ScaleUpdates[idx].ASGDesiredCapacity == 0 {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q] invalid cur.ScaleUpdates[%d].ASGDesiredCapacity == 0", cur.Name, idx)
- }
- if cur.ScaleUpdates[idx].ASGDesiredCapacity < cur.ScaleUpdates[idx].ASGMinSize {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q] invalid cur.ScaleUpdates[%d].ASGDesiredCapacity %d < ASGMinSize %d", cur.Name, idx, cur.ScaleUpdates[idx].ASGDesiredCapacity, cur.ScaleUpdates[idx].ASGMinSize)
- }
- }
- }
-
- // check optional mng version upgrade add-on
- if cur.VersionUpgrade != nil && cur.VersionUpgrade.Enable {
- var err error
- if cur.VersionUpgrade.InitialWaitString != "" {
- cur.VersionUpgrade.InitialWait, err = time.ParseDuration(cur.VersionUpgrade.InitialWaitString)
- if err != nil {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q] invalid cur.VersionUpgrade.InitialWaitString %q (%v)", cur.Name, cur.VersionUpgrade.InitialWaitString, err)
- }
- }
-
- // do not set any defaults
- // hard to keep everything in sync and find right values:
- // - original cluster version
- // - cluster upgrade version
- // - default mng version
- // - custom mng version
- if cur.VersionUpgrade.Version == "" {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q] VersionUpgrade.Enable but empty VersionUpgrade.Version", cur.Name)
- }
- cur.VersionUpgrade.VersionValue, err = strconv.ParseFloat(cur.VersionUpgrade.Version, 64)
- if err != nil {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q] invalid VersionUpgrade.Version %q (%v)", cur.Name, cur.VersionUpgrade.Version, err)
- }
- origVer := cfg.VersionValue
- if cur.ReleaseVersionValue > 0.0 {
- // e.g. "1.16" in "1.16.8-20200609"
- origVer = cur.ReleaseVersionValue
- }
-
- delta := cur.VersionUpgrade.VersionValue - origVer
- if fmt.Sprintf("%.2f", delta) != "0.01" {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q] VersionUpgrade only supports one minor version upgrade but got %.2f [cluster version %q, mng release version %q, mng upgrade version %q]", cur.Name, delta, cfg.Version, cur.ReleaseVersion, cur.VersionUpgrade.Version)
- }
- // target version must match with the Kubernetes control plane version
- // can't upgrade to 1.17 MNG when EKS is 1.16
- // e.g. "Nodegroup Kubernetes version should be equal to Cluster kubernetes version 1.16 or NodeGroup kubernetes version 1.16"
- if cur.ReleaseVersionValue == 0.0 && !cfg.IsEnabledAddOnClusterVersionUpgrade() {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q] VersionUpgrade %q would diverge from Parameters.Version %q (IsEnabledAddOnClusterVersionUpgrade %v)", cur.Name, cur.VersionUpgrade.Version, cfg.Version, cfg.IsEnabledAddOnClusterVersionUpgrade())
- }
- }
-
- if len(cur.InstanceTypes) > 4 {
- return fmt.Errorf("too many InstaceTypes[%q]", cur.InstanceTypes)
- }
- if cur.VolumeSize == 0 {
- cur.VolumeSize = DefaultNodeVolumeSize
- }
- if cur.RemoteAccessUserName == "" {
- cur.RemoteAccessUserName = "ec2-user"
- }
-
- switch cur.AMIType {
- case eks.AMITypesAl2X8664:
- if cur.RemoteAccessUserName != "ec2-user" {
- return fmt.Errorf("AMIType %q but unexpected RemoteAccessUserName %q", cur.AMIType, cur.RemoteAccessUserName)
- }
- case eks.AMITypesAl2X8664Gpu:
- if cur.RemoteAccessUserName != "ec2-user" {
- return fmt.Errorf("AMIType %q but unexpected RemoteAccessUserName %q", cur.AMIType, cur.RemoteAccessUserName)
- }
- case eks.AMITypesAl2Arm64:
- if cur.RemoteAccessUserName != "ec2-user" {
- return fmt.Errorf("AMIType %q but unexpected RemoteAccessUserName %q", cur.AMIType, cur.RemoteAccessUserName)
- }
- default:
- return fmt.Errorf("unknown ASGs[%q].AMIType %q", k, cur.AMIType)
- }
-
- switch cur.AMIType {
- case eks.AMITypesAl2X8664:
- if len(cur.InstanceTypes) == 0 {
- cur.InstanceTypes = []string{DefaultNodeInstanceTypeCPU}
- }
- case eks.AMITypesAl2X8664Gpu:
- if len(cur.InstanceTypes) == 0 {
- cur.InstanceTypes = []string{DefaultNodeInstanceTypeGPU}
- }
- case eks.AMITypesAl2Arm64:
- if len(cur.InstanceTypes) == 0 {
- cur.InstanceTypes = []string{DefaultNodeInstanceTypeARMCPU}
- }
- default:
- return fmt.Errorf("unknown AddOnManagedNodeGroups.MNGs[%q].AMIType %q", k, cur.AMIType)
- }
-
- if cfg.IsEnabledAddOnNLBHelloWorld() || cfg.IsEnabledAddOnALB2048() {
- for _, itp := range cur.InstanceTypes {
- // "m3.xlarge" or "c4.xlarge" will fail with "InvalidTarget: Targets {...} are not supported"
- // ref. https://github.com/aws/amazon-vpc-cni-k8s/pull/821
- // ref. https://github.com/kubernetes/kubernetes/issues/66044#issuecomment-408188524
- switch {
- case strings.HasPrefix(itp, "m3."),
- strings.HasPrefix(itp, "c4."):
- return fmt.Errorf("AddOnNLBHelloWorld.Enable[%v] || AddOnALB2048.Enable[%v], but older instance type InstanceTypes %q for %q",
- cfg.IsEnabledAddOnNLBHelloWorld(),
- cfg.IsEnabledAddOnALB2048(),
- itp, k)
- default:
- }
- }
- }
-
- if cur.ASGMinSize == 0 {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q].ASGMinSize must be >0", k)
- }
- if cur.ASGDesiredCapacity == 0 {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q].ASGDesiredCapacity must be >0", k)
- }
- if cur.ASGMaxSize == 0 {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q].ASGMaxSize must be >0", k)
- }
- if cur.ASGMinSize > cur.ASGMaxSize {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q].ASGMinSize %d > ASGMaxSize %d", k, cur.ASGMinSize, cur.ASGMaxSize)
- }
- if cur.ASGDesiredCapacity > cur.ASGMaxSize {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q].ASGDesiredCapacity %d > ASGMaxSize %d", k, cur.ASGDesiredCapacity, cur.ASGMaxSize)
- }
- if cur.ASGMaxSize > MNGMaxLimit {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q].ASGMaxSize %d > MNGMaxLimit %d", k, cur.ASGMaxSize, MNGMaxLimit)
- }
- if cur.ASGDesiredCapacity > MNGMaxLimit {
- return fmt.Errorf("AddOnManagedNodeGroups.MNGs[%q].ASGDesiredCapacity %d > MNGMaxLimit %d", k, cur.ASGDesiredCapacity, MNGMaxLimit)
- }
-
- if cfg.IsEnabledAddOnNLBHelloWorld() && cfg.AddOnNLBHelloWorld.DeploymentReplicas < int32(cur.ASGDesiredCapacity) {
- cfg.AddOnNLBHelloWorld.DeploymentReplicas = int32(cur.ASGDesiredCapacity)
- }
- if cfg.IsEnabledAddOnNLBGuestbook() && cfg.AddOnNLBGuestbook.DeploymentReplicas < int32(cur.ASGDesiredCapacity) {
- cfg.AddOnNLBGuestbook.DeploymentReplicas = int32(cur.ASGDesiredCapacity)
- }
- if cfg.IsEnabledAddOnALB2048() && cfg.AddOnALB2048.DeploymentReplicasALB < int32(cur.ASGDesiredCapacity) {
- cfg.AddOnALB2048.DeploymentReplicasALB = int32(cur.ASGDesiredCapacity)
- }
- if cfg.IsEnabledAddOnALB2048() && cfg.AddOnALB2048.DeploymentReplicas2048 < int32(cur.ASGDesiredCapacity) {
- cfg.AddOnALB2048.DeploymentReplicas2048 = int32(cur.ASGDesiredCapacity)
- }
-
- processed[k] = cur
- }
-
- cfg.AddOnManagedNodeGroups.MNGs = processed
- return nil
-}
diff --git a/eksconfig/add-on-metrics-server.go b/eksconfig/add-on-metrics-server.go
deleted file mode 100644
index 2409565a6..000000000
--- a/eksconfig/add-on-metrics-server.go
+++ /dev/null
@@ -1,52 +0,0 @@
-package eksconfig
-
-import (
- "errors"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnMetricsServer defines parameters for EKS cluster
-// add-on metrics server.
-// ref. https://github.com/kubernetes-sigs/metrics-server/releases
-type AddOnMetricsServer struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-}
-
-// EnvironmentVariablePrefixAddOnMetricsServer is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnMetricsServer = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_METRICS_SERVER_"
-
-// IsEnabledAddOnMetricsServer returns true if "AddOnMetricsServer" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnMetricsServer() bool {
- if cfg.AddOnMetricsServer == nil {
- return false
- }
- if cfg.AddOnMetricsServer.Enable {
- return true
- }
- cfg.AddOnMetricsServer = nil
- return false
-}
-
-func getDefaultAddOnMetricsServer() *AddOnMetricsServer {
- return &AddOnMetricsServer{
- Enable: false,
- }
-}
-
-func (cfg *Config) validateAddOnMetricsServer() error {
- if !cfg.IsEnabledAddOnMetricsServer() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnMetricsServer.Enable true but no node group is enabled")
- }
- return nil
-}
diff --git a/eksconfig/add-on-nlb-guestbook.go b/eksconfig/add-on-nlb-guestbook.go
deleted file mode 100644
index 05999f435..000000000
--- a/eksconfig/add-on-nlb-guestbook.go
+++ /dev/null
@@ -1,74 +0,0 @@
-package eksconfig
-
-import (
- "errors"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnNLBGuestbook defines parameters for EKS cluster
-// add-on NLB guestbook service.
-// ref. https://github.com/kubernetes/examples/tree/master/guestbook-go
-// ref. https://docs.aws.amazon.com/eks/latest/userguide/eks-guestbook.html
-type AddOnNLBGuestbook struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // DeploymentReplicas is the number of replicas to deploy using "Deployment" object.
- DeploymentReplicas int32 `json:"deployment-replicas"`
- // DeploymentNodeSelector is configured to overwrite existing node selector
- // for NLB hello world deployment. If left empty, tester sets default selector.
- DeploymentNodeSelector map[string]string `json:"deployment-node-selector"`
-
- // NLBARN is the ARN of the NLB created from the service.
- NLBARN string `json:"nlb-arn" read-only:"true"`
- // NLBName is the name of the NLB created from the service.
- NLBName string `json:"nlb-name" read-only:"true"`
- // URL is the host name for guestbook service.
- URL string `json:"url" read-only:"true"`
-}
-
-// EnvironmentVariablePrefixAddOnNLBGuestbook is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnNLBGuestbook = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_NLB_GUESTBOOK_"
-
-// IsEnabledAddOnNLBGuestbook returns true if "AddOnNLBGuestbook" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnNLBGuestbook() bool {
- if cfg.AddOnNLBGuestbook == nil {
- return false
- }
- if cfg.AddOnNLBGuestbook.Enable {
- return true
- }
- cfg.AddOnNLBGuestbook = nil
- return false
-}
-
-func getDefaultAddOnNLBGuestbook() *AddOnNLBGuestbook {
- return &AddOnNLBGuestbook{
- Enable: false,
- DeploymentReplicas: 3,
- DeploymentNodeSelector: make(map[string]string),
- }
-}
-
-func (cfg *Config) validateAddOnNLBGuestbook() error {
- if !cfg.IsEnabledAddOnNLBGuestbook() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnNLBGuestbook.Enable true but no node group is enabled")
- }
- if cfg.AddOnNLBGuestbook.Namespace == "" {
- cfg.AddOnNLBGuestbook.Namespace = cfg.Name + "-nlb-guestbook"
- }
- return nil
-}
diff --git a/eksconfig/add-on-nlb-hello-world.go b/eksconfig/add-on-nlb-hello-world.go
deleted file mode 100644
index 773d82908..000000000
--- a/eksconfig/add-on-nlb-hello-world.go
+++ /dev/null
@@ -1,72 +0,0 @@
-package eksconfig
-
-import (
- "errors"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnNLBHelloWorld defines parameters for EKS cluster
-// add-on NLB hello-world service.
-type AddOnNLBHelloWorld struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // DeploymentReplicas is the number of replicas to deploy using "Deployment" object.
- DeploymentReplicas int32 `json:"deployment-replicas"`
- // DeploymentNodeSelector is configured to overwrite existing node selector
- // for NLB hello world deployment. If left empty, tester sets default selector.
- DeploymentNodeSelector map[string]string `json:"deployment-node-selector"`
-
- // NLBARN is the ARN of the NLB created from the service.
- NLBARN string `json:"nlb-arn" read-only:"true"`
- // NLBName is the name of the NLB created from the service.
- NLBName string `json:"nlb-name" read-only:"true"`
- // URL is the host name for hello-world service.
- URL string `json:"url" read-only:"true"`
-}
-
-// EnvironmentVariablePrefixAddOnNLBHelloWorld is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnNLBHelloWorld = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_NLB_HELLO_WORLD_"
-
-// IsEnabledAddOnNLBHelloWorld returns true if "AddOnNLBHelloWorld" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnNLBHelloWorld() bool {
- if cfg.AddOnNLBHelloWorld == nil {
- return false
- }
- if cfg.AddOnNLBHelloWorld.Enable {
- return true
- }
- cfg.AddOnNLBHelloWorld = nil
- return false
-}
-
-func getDefaultAddOnNLBHelloWorld() *AddOnNLBHelloWorld {
- return &AddOnNLBHelloWorld{
- Enable: false,
- DeploymentReplicas: 3,
- DeploymentNodeSelector: make(map[string]string),
- }
-}
-
-func (cfg *Config) validateAddOnNLBHelloWorld() error {
- if !cfg.IsEnabledAddOnNLBHelloWorld() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnNLBHelloWorld.Enable true but no node group is enabled")
- }
- if cfg.AddOnNLBHelloWorld.Namespace == "" {
- cfg.AddOnNLBHelloWorld.Namespace = cfg.Name + "-nlb-hello-world"
- }
- return nil
-}
diff --git a/eksconfig/add-on-node-groups.go b/eksconfig/add-on-node-groups.go
deleted file mode 100644
index f817fbc80..000000000
--- a/eksconfig/add-on-node-groups.go
+++ /dev/null
@@ -1,356 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
- "path/filepath"
- "strings"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- aws_eks_v2_types "github.com/aws/aws-sdk-go-v2/service/eks/types"
- "github.com/aws/aws-sdk-go/service/eks"
-)
-
-// AddOnNodeGroups defines parameters for EKS "Managed Node Group" creation.
-// ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
-// ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html
-type AddOnNodeGroups struct {
- // Enable is true to auto-create ad node group.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- Role *Role `json:"role"`
-
- // FetchLogs is true to fetch logs from remote nodes using SSH.
- FetchLogs bool `json:"fetch-logs"`
-
- // LogsDir is set to specify the target directory to store all remote log files.
- // If empty, it stores in the same directory as "ConfigPath".
- LogsDir string `json:"logs-dir,omitempty"`
- // LogsTarGzPath is the .tar.gz archived file for "LogsDir".
- LogsTarGzPath string `json:"logs-tar-gz-path"`
- // ASGs maps from EKS Node Group name to "ASG".
- // "GetRef.Name" is the reserved key and NG name from eksconfig.Config.Name.
- ASGs map[string]ASG `json:"asgs,omitempty"`
-}
-
-// NGClusterAutoscaler represents cluster auto-scaler.
-// ref. https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler
-type NGClusterAutoscaler struct {
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
-}
-
-// ASG represents an EKS Node Group ASG.
-type ASG struct {
- ec2config.ASG
-
- // KubeletExtraArgs represents "--kubelet-extra-args".
- // e.g. '--kubelet-extra-args --node-labels=nodesgroup=main,subnets=private'
- // e.g. '--kubelet-extra-args --hostname-override=string'
- // ref. https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh
- //
- // TODO: handle conflicting flag '--cloud-provider aws'
- // ref. https://github.com/kubernetes/kubernetes/issues/64659
- KubeletExtraArgs string `json:"kubelet-extra-args"`
-
- // BootstrapArgs additional bootstrap arguments.
- // e.g. '--pause-container-account 012345678901 --pause-container-version 3.3'
- BootstrapArgs string `json:"bootstrap-args"`
-
- // ClusterAutoscaler is enabled to run cluster auto-scaler per node group.
- // ref. https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler
- ClusterAutoscaler *NGClusterAutoscaler `json:"cluster-autoscaler,omitempty"`
-}
-
-const (
- // AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_PREFIX is the environment variable prefix used for "eksconfig".
- AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_PREFIX = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_NODE_GROUPS_"
- AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_PREFIX = AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_PREFIX + "ROLE_"
-)
-
-// IsEnabledAddOnNodeGroups returns true if "AddOnNodeGroups" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnNodeGroups() bool {
- if cfg.AddOnNodeGroups == nil {
- return false
- }
- if cfg.AddOnNodeGroups.Enable {
- return len(cfg.AddOnNodeGroups.ASGs) > 0
- }
- cfg.AddOnNodeGroups = nil
- return false
-}
-
-func getDefaultAddOnNodeGroups(name string) *AddOnNodeGroups {
- return &AddOnNodeGroups{
- Enable: false,
- Role: getDefaultRole(),
- FetchLogs: false,
- LogsDir: "", // to be auto-generated
- ASGs: map[string]ASG{
- name + "-ng-asg-cpu": {
- ASG: ec2config.ASG{
- Name: name + "-ng-asg-cpu",
- SSM: &ec2config.SSM{
- DocumentCreate: false,
- DocumentName: "",
- DocumentCommands: "",
- DocumentExecutionTimeoutSeconds: 3600,
- },
- RemoteAccessUserName: "ec2-user", // assume Amazon Linux 2
- AMIType: eks.AMITypesAl2X8664,
- ImageID: "",
- ImageIDSSMParameter: "/aws/service/eks/optimized-ami/1.20/amazon-linux-2/recommended/image_id",
- InstanceType: DefaultNodeInstanceTypeCPU,
- VolumeSize: DefaultNodeVolumeSize,
- VolumeType: DefaultNodeVolumeType,
- ASGMinSize: 1,
- ASGMaxSize: 1,
- ASGDesiredCapacity: 1,
- LaunchTemplateName: name + "-launch-template",
- },
- KubeletExtraArgs: "",
- BootstrapArgs: "",
- ClusterAutoscaler: &NGClusterAutoscaler{Enable: false},
- },
- },
- }
-}
-
-func (cfg *Config) validateAddOnNodeGroups() error {
- if !cfg.IsEnabledAddOnNodeGroups() {
- return nil
- }
-
- switch cfg.AddOnNodeGroups.Role.Create {
- case true: // need create one, or already created
- if cfg.AddOnNodeGroups.Role.Name == "" {
- cfg.AddOnNodeGroups.Role.Name = cfg.Name + "-node-group-role"
- }
- // just ignore...
- // could be populated from previous run
- // do not error, so long as RoleCreate false, role won't be deleted
-
- case false: // use existing one
- if cfg.AddOnNodeGroups.Role.ARN == "" {
- return fmt.Errorf("Role.Create false; expect non-empty RoleARN but got %q", cfg.AddOnNodeGroups.Role.ARN)
- }
- if cfg.AddOnNodeGroups.Role.Name == "" {
- cfg.AddOnNodeGroups.Role.Name = getNameFromARN(cfg.AddOnNodeGroups.Role.ARN)
- }
- }
- if cfg.AddOnNodeGroups.Role.PolicyName == "" {
- cfg.AddOnNodeGroups.Role.PolicyName = cfg.Name + "-node-group-policy"
- }
- if cfg.AddOnNodeGroups.Role.InstanceProfileName == "" {
- cfg.AddOnNodeGroups.Role.InstanceProfileName = cfg.Name + "-node-group-instance-profile"
- }
-
- n := len(cfg.AddOnNodeGroups.ASGs)
- if n == 0 {
- return errors.New("empty ASGs")
- }
- if n > NGsMaxLimit {
- return fmt.Errorf("NGs %d exceeds maximum number of NGs which is %d", n, NGsMaxLimit)
- }
-
- if cfg.VersionValue < 1.14 {
- return fmt.Errorf("version %q not supported for AddOnNodeGroups", cfg.Version)
- }
-
- if cfg.AddOnNodeGroups.LogsDir == "" {
- cfg.AddOnNodeGroups.LogsDir = filepath.Join(filepath.Dir(cfg.ConfigPath), cfg.Name+"-logs-ngs")
- }
- if cfg.AddOnNodeGroups.LogsTarGzPath == "" {
- cfg.AddOnNodeGroups.LogsTarGzPath = filepath.Join(filepath.Dir(cfg.ConfigPath), cfg.Name+"-logs-ngs.tar.gz")
- }
- if !strings.HasSuffix(cfg.AddOnNodeGroups.LogsTarGzPath, ".tar.gz") {
- return fmt.Errorf("AddOnNodeGroups.LogsTarGzPath %q must end with .tar.gz", cfg.AddOnNodeGroups.LogsTarGzPath)
- }
-
- names, processed := make(map[string]struct{}), make(map[string]ASG)
- for k, cur := range cfg.AddOnNodeGroups.ASGs {
- k = strings.ReplaceAll(k, "GetRef.Name", cfg.Name)
- cur.Name = strings.ReplaceAll(cur.Name, "GetRef.Name", cfg.Name)
- if cur.Name == "" {
- return fmt.Errorf("AddOnNodeGroups.ASGs[%q].Name is empty", k)
- }
- if k != cur.Name {
- return fmt.Errorf("AddOnNodeGroups.ASGs[%q].Name has different Name field %q", k, cur.Name)
- }
- _, ok := names[cur.Name]
- if !ok {
- names[cur.Name] = struct{}{}
- } else {
- return fmt.Errorf("AddOnNodeGroups.ASGs[%q].Name %q is redundant", k, cur.Name)
- }
-
- if cur.VolumeSize == 0 {
- cur.VolumeSize = DefaultNodeVolumeSize
- }
- if cur.VolumeType == "" {
- cur.VolumeType = DefaultNodeVolumeType
- }
- if cur.RemoteAccessUserName == "" {
- cur.RemoteAccessUserName = "ec2-user"
- }
-
- if cur.ImageID == "" && cur.ImageIDSSMParameter == "" {
- return fmt.Errorf("%q both ImageID and ImageIDSSMParameter are empty", cur.Name)
- }
- // prefer "ImageIDSSMParameter"
- if cur.ImageID != "" && cur.ImageIDSSMParameter != "" {
- cur.ImageID = ""
- }
- if cur.LaunchTemplateName == "" {
- cur.LaunchTemplateName = cur.Name + "-launch-template"
- }
-
- switch cur.AMIType {
- case ec2config.AMITypeBottleRocketCPU:
- if cur.RemoteAccessUserName != "ec2-user" {
- return fmt.Errorf("AMIType %q but unexpected RemoteAccessUserName %q", cur.AMIType, cur.RemoteAccessUserName)
- }
- if cur.SSM != nil {
- if cur.SSM.DocumentName != "" && cfg.S3.BucketName == "" {
- return fmt.Errorf("AMIType %q requires SSMDocumentName %q but no S3BucketName", cur.AMIType, cur.SSM.DocumentName)
- }
- }
- if cur.KubeletExtraArgs != "" {
- return fmt.Errorf("AMIType %q but unexpected KubeletExtraArgs %q", cur.AMIType, cur.KubeletExtraArgs)
- }
- case fmt.Sprint(aws_eks_v2_types.AMITypesAl2X8664):
- if cur.RemoteAccessUserName != "ec2-user" {
- return fmt.Errorf("AMIType %q but unexpected RemoteAccessUserName %q", cur.AMIType, cur.RemoteAccessUserName)
- }
- case fmt.Sprint(aws_eks_v2_types.AMITypesAl2Arm64):
- if cur.RemoteAccessUserName != "ec2-user" {
- return fmt.Errorf("AMIType %q but unexpected RemoteAccessUserName %q", cur.AMIType, cur.RemoteAccessUserName)
- }
- case fmt.Sprint(aws_eks_v2_types.AMITypesAl2X8664Gpu):
- if cur.RemoteAccessUserName != "ec2-user" {
- return fmt.Errorf("AMIType %q but unexpected RemoteAccessUserName %q", cur.AMIType, cur.RemoteAccessUserName)
- }
- case ec2config.AMITypeWindowsServerCore2019X8664:
- if cur.RemoteAccessUserName != "ec2-user" {
- return fmt.Errorf("AMIType %q but unexpected RemoteAccessUserName %q", cur.AMIType, cur.RemoteAccessUserName)
- }
- default:
- return fmt.Errorf("unknown ASGs[%q].AMIType %q", k, cur.AMIType)
- }
-
- switch cur.AMIType {
- case ec2config.AMITypeBottleRocketCPU:
- if cur.InstanceType == "" {
- cur.InstanceType = DefaultNodeInstanceTypeCPU
- }
- case fmt.Sprint(aws_eks_v2_types.AMITypesAl2X8664), ec2config.AMITypeWindowsServerCore2019X8664:
- if cur.InstanceType == "" {
- cur.InstanceType = DefaultNodeInstanceTypeCPU
- }
- case fmt.Sprint(aws_eks_v2_types.AMITypesAl2X8664Gpu):
- if cur.InstanceType == "" {
- cur.InstanceType = DefaultNodeInstanceTypeGPU
- }
- default:
- return fmt.Errorf("unknown AddOnNodeGroups.ASGs[%q].AMIType %q", k, cur.AMIType)
- }
-
- if cfg.IsEnabledAddOnNLBHelloWorld() || cfg.IsEnabledAddOnALB2048() {
- // "m3.xlarge" or "c4.xlarge" will fail with "InvalidTarget: Targets {...} are not supported"
- // ref. https://github.com/aws/amazon-vpc-cni-k8s/pull/821
- // ref. https://github.com/kubernetes/kubernetes/issues/66044#issuecomment-408188524
- switch {
- case strings.HasPrefix(cur.InstanceType, "m3."),
- strings.HasPrefix(cur.InstanceType, "c4."):
- return fmt.Errorf("AddOnNLBHelloWorld.Enable[%v] || AddOnALB2048.Enable[%v], but older instance type InstanceType %q for %q",
- cfg.IsEnabledAddOnNLBHelloWorld(),
- cfg.IsEnabledAddOnALB2048(),
- cur.InstanceType, k)
- }
- }
-
- if cur.ASGMinSize == 0 {
- return fmt.Errorf("AddOnNodeGroups.ASGs[%q].ASGMinSize must be >0", k)
- }
- if cur.ASGDesiredCapacity == 0 {
- return fmt.Errorf("AddOnNodeGroups.ASGs[%q].ASGDesiredCapacity must be >0", k)
- }
- if cur.ASGMaxSize == 0 {
- return fmt.Errorf("AddOnNodeGroups.ASGs[%q].ASGMaxSize must be >0", k)
- }
- if cur.ASGMinSize > cur.ASGMaxSize {
- return fmt.Errorf("AddOnNodeGroups.ASGs[%q].ASGMinSize %d > ASGMaxSize %d", k, cur.ASGMinSize, cur.ASGMaxSize)
- }
- if cur.ASGDesiredCapacity > cur.ASGMaxSize {
- return fmt.Errorf("AddOnNodeGroups.ASGs[%q].ASGDesiredCapacity %d > ASGMaxSize %d", k, cur.ASGDesiredCapacity, cur.ASGMaxSize)
- }
- if cur.ASGMaxSize > NGMaxLimit {
- return fmt.Errorf("AddOnNodeGroups.ASGs[%q].ASGMaxSize %d > NGMaxLimit %d", k, cur.ASGMaxSize, NGMaxLimit)
- }
- if cur.ASGDesiredCapacity > NGMaxLimit {
- return fmt.Errorf("AddOnNodeGroups.ASGs[%q].ASGDesiredCapacity %d > NGMaxLimit %d", k, cur.ASGDesiredCapacity, NGMaxLimit)
- }
-
- if cur.SSM != nil {
- switch cur.SSM.DocumentCreate {
- case true: // need create one, or already created
- if cur.SSM.DocumentName == "" {
- cur.SSM.DocumentName = cur.Name + "SSMDocument"
- }
- cur.SSM.DocumentName = strings.ReplaceAll(cur.SSM.DocumentName, "GetRef.Name", cfg.Name)
- cur.SSM.DocumentName = regex.ReplaceAllString(cur.SSM.DocumentName, "")
- if cur.SSM.DocumentExecutionTimeoutSeconds == 0 {
- cur.SSM.DocumentExecutionTimeoutSeconds = 3600
- }
- if cur.SSM.DocumentCommands == "" {
- return errors.New("empty SSM.DocumentCommands")
- }
-
- case false: // use existing one, or don't run any SSM
- }
- }
-
- if cfg.IsEnabledAddOnNLBHelloWorld() && cfg.AddOnNLBHelloWorld.DeploymentReplicas < int32(cur.ASGDesiredCapacity) {
- cfg.AddOnNLBHelloWorld.DeploymentReplicas = int32(cur.ASGDesiredCapacity)
- }
- if cfg.IsEnabledAddOnNLBGuestbook() && cfg.AddOnNLBGuestbook.DeploymentReplicas < int32(cur.ASGDesiredCapacity) {
- cfg.AddOnNLBGuestbook.DeploymentReplicas = int32(cur.ASGDesiredCapacity)
- }
- if cfg.IsEnabledAddOnALB2048() && cfg.AddOnALB2048.DeploymentReplicasALB < int32(cur.ASGDesiredCapacity) {
- cfg.AddOnALB2048.DeploymentReplicasALB = int32(cur.ASGDesiredCapacity)
- }
- if cfg.IsEnabledAddOnALB2048() && cfg.AddOnALB2048.DeploymentReplicas2048 < int32(cur.ASGDesiredCapacity) {
- cfg.AddOnALB2048.DeploymentReplicas2048 = int32(cur.ASGDesiredCapacity)
- }
-
- processed[k] = cur
- }
-
- cfg.AddOnNodeGroups.ASGs = processed
- return nil
-}
-
-func (addOn *AddOnNodeGroups) IsEnabledClusterAutoscaler() bool {
- if addOn == nil {
- return false
- }
- if len(addOn.ASGs) == 0 {
- return false
- }
- for _, cur := range addOn.ASGs {
- if cur.ClusterAutoscaler != nil && cur.ClusterAutoscaler.Enable {
- return true
- }
- }
- return false
-}
diff --git a/eksconfig/add-on-php-apache.go b/eksconfig/add-on-php-apache.go
deleted file mode 100644
index 429c98280..000000000
--- a/eksconfig/add-on-php-apache.go
+++ /dev/null
@@ -1,84 +0,0 @@
-package eksconfig
-
-import (
- "errors"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnPHPApache defines parameters for EKS cluster
-// add-on PHP Apache.
-type AddOnPHPApache struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // RepositoryAccountID is the account ID for tester ECR image.
- // e.g. "php-apache" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/php-apache"
- RepositoryAccountID string `json:"repository-account-id,omitempty"`
- // RepositoryRegion is the ECR repository region to pull from.
- RepositoryRegion string `json:"repository-region,omitempty"`
- // RepositoryName is the repositoryName for tester ECR image.
- // e.g. "php-apache" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/php-apache"
- RepositoryName string `json:"repository-name,omitempty"`
- // RepositoryImageTag is the image tag for tester ECR image.
- // e.g. "latest" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/php-apache:latest"
- RepositoryImageTag string `json:"repository-image-tag,omitempty"`
-
- // DeploymentReplicas is the number of replicas to deploy using "Deployment" object.
- DeploymentReplicas int32 `json:"deployment-replicas"`
- // DeploymentNodeSelector is configured to overwrite existing node selector
- // for PHP Apache deployment. If left empty, tester sets default selector.
- DeploymentNodeSelector map[string]string `json:"deployment-node-selector"`
-}
-
-// EnvironmentVariablePrefixAddOnPHPApache is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnPHPApache = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_PHP_APACHE_"
-
-// IsEnabledAddOnPHPApache returns true if "AddOnPHPApache" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnPHPApache() bool {
- if cfg.AddOnPHPApache == nil {
- return false
- }
- if cfg.AddOnPHPApache.Enable {
- return true
- }
- cfg.AddOnPHPApache = nil
- return false
-}
-
-func getDefaultAddOnPHPApache() *AddOnPHPApache {
- return &AddOnPHPApache{
- Enable: false,
- DeploymentReplicas: 3,
- DeploymentNodeSelector: make(map[string]string),
- }
-}
-
-func (cfg *Config) GetAddOnPHPApacheRepositoryRegion() string {
- if !cfg.IsEnabledAddOnPHPApache() {
- return cfg.Region
- }
- return cfg.AddOnPHPApache.RepositoryRegion
-}
-
-func (cfg *Config) validateAddOnPHPApache() error {
- if !cfg.IsEnabledAddOnPHPApache() {
- return nil
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnPHPApache.Enable true but no node group is enabled")
- }
- if cfg.AddOnPHPApache.Namespace == "" {
- cfg.AddOnPHPApache.Namespace = cfg.Name + "-php-apache"
- }
- return nil
-}
diff --git a/eksconfig/add-on-prometheus-grafana.go b/eksconfig/add-on-prometheus-grafana.go
deleted file mode 100644
index 4c1efe814..000000000
--- a/eksconfig/add-on-prometheus-grafana.go
+++ /dev/null
@@ -1,117 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/pkg/randutil"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/service/eks"
-)
-
-// AddOnPrometheusGrafana defines parameters for EKS cluster
-// add-on Prometheus/Grafana.
-// ref. https://docs.aws.amazon.com/eks/latest/userguide/prometheus.html
-// ref. https://eksworkshop.com/intermediate/240_monitoring/deploy-prometheus/
-type AddOnPrometheusGrafana struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // GrafanaAdminUserName is the admin user for the Grafana service.
- GrafanaAdminUserName string `json:"grafana-admin-user-name"`
- // GrafanaAdminPassword is the admin password for the Grafana service.
- GrafanaAdminPassword string `json:"grafana-admin-password"`
- // GrafanaNLBARN is the ARN of the NLB created from the Grafana service.
- GrafanaNLBARN string `json:"grafana-nlb-arn" read-only:"true"`
- // GrafanaNLBName is the name of the NLB created from the Grafana service.
- GrafanaNLBName string `json:"grafana-nlb-name" read-only:"true"`
- // GrafanaURL is the host name for Grafana service.
- GrafanaURL string `json:"grafana-url" read-only:"true"`
-}
-
-// EnvironmentVariablePrefixAddOnPrometheusGrafana is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnPrometheusGrafana = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_PROMETHEUS_GRAFANA_"
-
-// IsEnabledAddOnPrometheusGrafana returns true if "AddOnPrometheusGrafana" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnPrometheusGrafana() bool {
- if cfg.AddOnPrometheusGrafana == nil {
- return false
- }
- if cfg.AddOnPrometheusGrafana.Enable {
- return true
- }
- cfg.AddOnPrometheusGrafana = nil
- return false
-}
-
-func getDefaultAddOnPrometheusGrafana() *AddOnPrometheusGrafana {
- return &AddOnPrometheusGrafana{
- Enable: false,
- GrafanaAdminUserName: "admin",
- GrafanaAdminPassword: "",
- }
-}
-
-func (cfg *Config) validateAddOnPrometheusGrafana() error {
- if !cfg.IsEnabledAddOnPrometheusGrafana() {
- return nil
- }
- if !cfg.IsEnabledAddOnCSIEBS() {
- return errors.New("AddOnPrometheusGrafana.Enable true but IsEnabledAddOnCSIEBS.Enable false")
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnPrometheusGrafana.Enable true but no node group is enabled")
- }
-
- // TODO: PVC not working on BottleRocket
- // do not assign mariadb to Bottlerocket
- // e.g. MountVolume.MountDevice failed for volume "pvc-8e035a13-4d33-472f-a4c0-f36c7d39d170" : executable file not found in $PATH
- // e.g. Unable to mount volumes for pod "wordpress-84c567b89b-2jgh5_eks-2020042114-exclusivea3i-wordpress(d02336a3-1799-4b08-9f15-b90871f6a2f0)": timeout expired waiting for volumes to attach or mount for pod "eks-2020042114-exclusivea3i-wordpress"/"wordpress-84c567b89b-2jgh5". list of unmounted volumes=[wordpress-data]. list of unattached volumes=[wordpress-data default-token-7bdc2]
- // TODO: fix CSI EBS https://github.com/bottlerocket-os/bottlerocket/issues/877
- if cfg.IsEnabledAddOnNodeGroups() {
- x86Found, rocketFound := false, false
- for _, asg := range cfg.AddOnNodeGroups.ASGs {
- switch asg.AMIType {
- case ec2config.AMITypeAL2X8664,
- ec2config.AMITypeAL2X8664GPU:
- x86Found = true
- case ec2config.AMITypeBottleRocketCPU:
- rocketFound = true
- }
- }
- if !x86Found && rocketFound {
- return fmt.Errorf("AddOnPrometheusGrafana.Enabled true but AddOnNodeGroups [x86Found %v, rocketFound %v]", x86Found, rocketFound)
- }
- }
- if cfg.IsEnabledAddOnManagedNodeGroups() {
- x86Found, rocketFound := false, false
- for _, asg := range cfg.AddOnManagedNodeGroups.MNGs {
- switch asg.AMIType {
- case eks.AMITypesAl2X8664,
- eks.AMITypesAl2X8664Gpu:
- x86Found = true
- case ec2config.AMITypeBottleRocketCPU:
- rocketFound = true
- }
- }
- if !x86Found && rocketFound {
- return fmt.Errorf("AddOnPrometheusGrafana.Enabled true but AddOnManagedNodeGroups [x86Found %v, rocketFound %v]", x86Found, rocketFound)
- }
- }
-
- if cfg.AddOnPrometheusGrafana.GrafanaAdminUserName == "" {
- cfg.AddOnPrometheusGrafana.GrafanaAdminUserName = randutil.String(10)
- }
- if cfg.AddOnPrometheusGrafana.GrafanaAdminPassword == "" {
- cfg.AddOnPrometheusGrafana.GrafanaAdminPassword = randutil.String(10)
- }
-
- return nil
-}
diff --git a/eksconfig/add-on-secrets-local.go b/eksconfig/add-on-secrets-local.go
deleted file mode 100644
index da159c55f..000000000
--- a/eksconfig/add-on-secrets-local.go
+++ /dev/null
@@ -1,366 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "path"
- "path/filepath"
- "strings"
-
- "github.com/aws/aws-k8s-tester/pkg/metrics"
- "github.com/aws/aws-k8s-tester/pkg/randutil"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnSecretsLocal defines parameters for EKS cluster
-// add-on "Secrets" local.
-// It generates loads from the local host machine.
-// Every object is written serially with no concurrency.
-// Use remote tester to write with concurrency.
-// The main use case is to write a large number of objects to fill up etcd database.
-// And measure latencies for secret encryption.
-type AddOnSecretsLocal struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // S3Dir is the S3 directory to store all test results.
- // It is under the bucket "eksconfig.Config.S3BucketName".
- S3Dir string `json:"s3-dir"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // Objects is the number of "Secret" objects to write/read.
- Objects int `json:"objects"`
- // ObjectSize is the "Secret" value size in bytes.
- ObjectSize int `json:"object-size"`
-
- // NamePrefix is the prefix of Secret name.
- // If multiple Secret loader is running,
- // this must be unique per worker to avoid name conflicts.
- NamePrefix string `json:"name-prefix"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- RequestsRawWritesJSONPath string `json:"requests-raw-writes-json-path" read-only:"true"`
- RequestsRawWritesJSONS3Key string `json:"requests-raw-writes-json-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsRawWritesCompareS3Dir is the s3 directory to store raw data points.
- // Used to comparison results.
- // ref. https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
- RequestsRawWritesCompareS3Dir string `json:"requests-raw-writes-compare-s3-dir"`
- RequestsRawWritesCompareAllJSONPath string `json:"requests-raw-writes-compare-all-json-path" read-only:"true"`
- RequestsRawWritesCompareAllJSONS3Key string `json:"requests-raw-writes-compare-all-json-s3-key" read-only:"true"`
- RequestsRawWritesCompareAllCSVPath string `json:"requests-raw-writes-compare-all-csv-path" read-only:"true"`
- RequestsRawWritesCompareAllCSVS3Key string `json:"requests-raw-writes-compare-all-csv-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWrites is the writes results.
- RequestsSummaryWrites metrics.RequestsSummary `json:"requests-summary-writes,omitempty" read-only:"true"`
- RequestsSummaryWritesJSONPath string `json:"requests-summary-writes-json-path" read-only:"true"`
- RequestsSummaryWritesJSONS3Key string `json:"requests-summary-writes-json-s3-key" read-only:"true"`
- RequestsSummaryWritesTablePath string `json:"requests-summary-writes-table-path" read-only:"true"`
- RequestsSummaryWritesTableS3Key string `json:"requests-summary-writes-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWritesCompareS3Dir is the S3 directory of previous/latest "RequestsSummary".
- // Specify the S3 key in the same bucket of "eksconfig.Config.S3BucketName".
- // Use for regression tests. Specify the value not bound to the cluster directory.
- // Different runs from different clusters reads and writes in this directory.
- RequestsSummaryWritesCompareS3Dir string `json:"requests-summary-writes-compare-s3-dir"`
- RequestsSummaryWritesCompare metrics.RequestsCompare `json:"requests-summary-writes-compare" read-only:"true"`
- RequestsSummaryWritesCompareJSONPath string `json:"requests-summary-writes-compare-json-path" read-only:"true"`
- RequestsSummaryWritesCompareJSONS3Key string `json:"requests-summary-writes-compare-json-s3-key" read-only:"true"`
- RequestsSummaryWritesCompareTablePath string `json:"requests-summary-writes-compare-table-path" read-only:"true"`
- RequestsSummaryWritesCompareTableS3Key string `json:"requests-summary-writes-compare-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- RequestsRawReadsJSONPath string `json:"requests-raw-reads-json-path" read-only:"true"`
- RequestsRawReadsJSONS3Key string `json:"requests-raw-reads-json-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsRawReadsCompareS3Dir is the s3 directory to store raw data points.
- // Used to comparison results.
- // ref. https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
- RequestsRawReadsCompareS3Dir string `json:"requests-raw-reads-compare-s3-dir"`
- RequestsRawReadsCompareAllJSONPath string `json:"requests-raw-reads-compare-all-json-path" read-only:"true"`
- RequestsRawReadsCompareAllJSONS3Key string `json:"requests-raw-reads-compare-all-json-s3-key" read-only:"true"`
- RequestsRawReadsCompareAllCSVPath string `json:"requests-raw-reads-compare-all-csv-path" read-only:"true"`
- RequestsRawReadsCompareAllCSVS3Key string `json:"requests-raw-reads-compare-all-csv-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryReads is the reads results.
- RequestsSummaryReads metrics.RequestsSummary `json:"requests-summary-reads,omitempty" read-only:"true"`
- RequestsSummaryReadsJSONPath string `json:"requests-summary-reads-json-path" read-only:"true"`
- RequestsSummaryReadsJSONS3Key string `json:"requests-summary-reads-json-s3-key" read-only:"true"`
- RequestsSummaryReadsTablePath string `json:"requests-summary-reads-table-path" read-only:"true"`
- RequestsSummaryReadsTableS3Key string `json:"requests-summary-reads-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryReadsCompareS3Dir is the S3 directory of previous/latest "RequestsSummary".
- // Specify the S3 key in the same bucket of "eksconfig.Config.S3BucketName".
- // Use for regression tests. Specify the value not bound to the cluster directory.
- // Different runs from different clusters reads and writes in this directory.
- RequestsSummaryReadsCompareS3Dir string `json:"requests-summary-reads-compare-s3-dir"`
- RequestsSummaryReadsCompare metrics.RequestsCompare `json:"requests-summary-reads-compare" read-only:"true"`
- RequestsSummaryReadsCompareJSONPath string `json:"requests-summary-reads-compare-json-path" read-only:"true"`
- RequestsSummaryReadsCompareJSONS3Key string `json:"requests-summary-reads-compare-json-s3-key" read-only:"true"`
- RequestsSummaryReadsCompareTablePath string `json:"requests-summary-reads-compare-table-path" read-only:"true"`
- RequestsSummaryReadsCompareTableS3Key string `json:"requests-summary-reads-compare-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-}
-
-// EnvironmentVariablePrefixAddOnSecretsLocal is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnSecretsLocal = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_SECRETS_LOCAL_"
-
-// IsEnabledAddOnSecretsLocal returns true if "AddOnSecretsLocal" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnSecretsLocal() bool {
- if cfg.AddOnSecretsLocal == nil {
- return false
- }
- if cfg.AddOnSecretsLocal.Enable {
- return true
- }
- cfg.AddOnSecretsLocal = nil
- return false
-}
-
-func getDefaultAddOnSecretsLocal() *AddOnSecretsLocal {
- return &AddOnSecretsLocal{
- Enable: false,
- Objects: 10,
- ObjectSize: 10 * 1024, // 10 KB
-
- // writes total 100 MB for "Secret" objects,
- // plus "Pod" objects, writes total 330 MB to etcd
- //
- // with 3 nodes, takes about 1.5 hour for all
- // these "Pod"s to complete
- //
- // Objects: 10000,
- // ObjectSize: 10 * 1024, // 10 KB
-
- NamePrefix: "secret" + randutil.String(5),
- }
-}
-
-func (cfg *Config) validateAddOnSecretsLocal() error {
- if !cfg.IsEnabledAddOnSecretsLocal() {
- return nil
- }
-
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnSecretsLocal.Enable true but no node group is enabled")
- }
-
- if cfg.AddOnSecretsLocal.S3Dir == "" {
- cfg.AddOnSecretsLocal.S3Dir = path.Join(cfg.Name, "add-on-secrets-local")
- }
-
- if cfg.AddOnSecretsLocal.Namespace == "" {
- cfg.AddOnSecretsLocal.Namespace = cfg.Name + "-secrets-local"
- }
-
- if cfg.AddOnSecretsLocal.Objects == 0 {
- cfg.AddOnSecretsLocal.Objects = 10
- }
- if cfg.AddOnSecretsLocal.ObjectSize == 0 {
- cfg.AddOnSecretsLocal.ObjectSize = 10 * 1024
- }
-
- if cfg.AddOnSecretsLocal.NamePrefix == "" {
- cfg.AddOnSecretsLocal.NamePrefix = "secret" + randutil.String(5)
- }
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsLocal.RequestsRawWritesJSONPath == "" {
- cfg.AddOnSecretsLocal.RequestsRawWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-local-requests-writes-raw.json"
- }
- if cfg.AddOnSecretsLocal.RequestsRawWritesJSONS3Key == "" {
- cfg.AddOnSecretsLocal.RequestsRawWritesJSONS3Key = path.Join(
- cfg.AddOnSecretsLocal.S3Dir,
- "requests-raw-writes",
- filepath.Base(cfg.AddOnSecretsLocal.RequestsRawWritesJSONPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsLocal.RequestsRawWritesCompareS3Dir == "" {
- cfg.AddOnSecretsLocal.RequestsRawWritesCompareS3Dir = path.Join("add-on-secrets-local", "requests-raw-writes-compare", cfg.Version)
- }
- if cfg.AddOnSecretsLocal.RequestsRawWritesCompareAllJSONPath == "" {
- cfg.AddOnSecretsLocal.RequestsRawWritesCompareAllJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-local-requests-raw-writes-compare-all.json"
- }
- if cfg.AddOnSecretsLocal.RequestsRawWritesCompareAllJSONS3Key == "" {
- cfg.AddOnSecretsLocal.RequestsRawWritesCompareAllJSONS3Key = path.Join(
- cfg.AddOnSecretsLocal.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnSecretsLocal.RequestsRawWritesCompareAllJSONPath),
- )
- }
- if cfg.AddOnSecretsLocal.RequestsRawWritesCompareAllCSVPath == "" {
- cfg.AddOnSecretsLocal.RequestsRawWritesCompareAllCSVPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-local-requests-raw-writes-compare-all.csv"
- }
- if cfg.AddOnSecretsLocal.RequestsRawWritesCompareAllCSVS3Key == "" {
- cfg.AddOnSecretsLocal.RequestsRawWritesCompareAllCSVS3Key = path.Join(
- cfg.AddOnSecretsLocal.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnSecretsLocal.RequestsRawWritesCompareAllCSVPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsLocal.RequestsSummaryWritesJSONPath == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-local-requests-summary-writes.json"
- }
- if cfg.AddOnSecretsLocal.RequestsSummaryWritesJSONS3Key == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryWritesJSONS3Key = path.Join(
- cfg.AddOnSecretsLocal.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnSecretsLocal.RequestsSummaryWritesJSONPath),
- )
- }
- if cfg.AddOnSecretsLocal.RequestsSummaryWritesTablePath == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryWritesTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-local-requests-summary-writes.txt"
- }
- if cfg.AddOnSecretsLocal.RequestsSummaryWritesTableS3Key == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryWritesTableS3Key = path.Join(
- cfg.AddOnSecretsLocal.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnSecretsLocal.RequestsSummaryWritesTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsLocal.RequestsSummaryWritesCompareS3Dir == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryWritesCompareS3Dir = path.Join("add-on-secrets-local", "requests-summary-writes-compare", cfg.Version)
- }
- if cfg.AddOnSecretsLocal.RequestsSummaryWritesCompareJSONPath == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryWritesCompareJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-local-requests-summary-writes-compare.json"
- }
- if cfg.AddOnSecretsLocal.RequestsSummaryWritesCompareJSONS3Key == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryWritesCompareJSONS3Key = path.Join(
- cfg.AddOnSecretsLocal.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnSecretsLocal.RequestsSummaryWritesCompareJSONPath),
- )
- }
- if cfg.AddOnSecretsLocal.RequestsSummaryWritesCompareTablePath == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryWritesCompareTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-local-requests-summary-writes-compare.txt"
- }
- if cfg.AddOnSecretsLocal.RequestsSummaryWritesCompareTableS3Key == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryWritesCompareTableS3Key = path.Join(
- cfg.AddOnSecretsLocal.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnSecretsLocal.RequestsSummaryWritesCompareTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsLocal.RequestsRawReadsJSONPath == "" {
- cfg.AddOnSecretsLocal.RequestsRawReadsJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-local-requests-raw-reads.json"
- }
- if cfg.AddOnSecretsLocal.RequestsRawReadsJSONS3Key == "" {
- cfg.AddOnSecretsLocal.RequestsRawReadsJSONS3Key = path.Join(
- cfg.AddOnSecretsLocal.S3Dir,
- "requests-raw-reads",
- filepath.Base(cfg.AddOnSecretsLocal.RequestsRawReadsJSONPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsLocal.RequestsRawReadsCompareS3Dir == "" {
- cfg.AddOnSecretsLocal.RequestsRawReadsCompareS3Dir = path.Join("add-on-secrets-local", "requests-raw-reads-compare", cfg.Version)
- }
- if cfg.AddOnSecretsLocal.RequestsRawReadsCompareAllJSONPath == "" {
- cfg.AddOnSecretsLocal.RequestsRawReadsCompareAllJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-local-requests-raw-reads-compare-all.json"
- }
- if cfg.AddOnSecretsLocal.RequestsRawReadsCompareAllJSONS3Key == "" {
- cfg.AddOnSecretsLocal.RequestsRawReadsCompareAllJSONS3Key = path.Join(
- cfg.AddOnSecretsLocal.S3Dir,
- "requests-raw-reads-compare-all",
- filepath.Base(cfg.AddOnSecretsLocal.RequestsRawReadsCompareAllJSONPath),
- )
- }
- if cfg.AddOnSecretsLocal.RequestsRawReadsCompareAllCSVPath == "" {
- cfg.AddOnSecretsLocal.RequestsRawReadsCompareAllCSVPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-local-requests-raw-reads-compare-all.csv"
- }
- if cfg.AddOnSecretsLocal.RequestsRawReadsCompareAllCSVS3Key == "" {
- cfg.AddOnSecretsLocal.RequestsRawReadsCompareAllCSVS3Key = path.Join(
- cfg.AddOnSecretsLocal.S3Dir,
- "requests-raw-reads-compare-all",
- filepath.Base(cfg.AddOnSecretsLocal.RequestsRawReadsCompareAllCSVPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsLocal.RequestsSummaryReadsJSONPath == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryReadsJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-local-requests-summary-reads.json"
- }
- if cfg.AddOnSecretsLocal.RequestsSummaryReadsJSONS3Key == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryReadsJSONS3Key = path.Join(
- cfg.AddOnSecretsLocal.S3Dir,
- "requests-summary-reads",
- filepath.Base(cfg.AddOnSecretsLocal.RequestsSummaryReadsJSONPath),
- )
- }
- if cfg.AddOnSecretsLocal.RequestsSummaryReadsTablePath == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryReadsTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-local-requests-summary-reads.txt"
- }
- if cfg.AddOnSecretsLocal.RequestsSummaryReadsTableS3Key == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryReadsTableS3Key = path.Join(
- cfg.AddOnSecretsLocal.S3Dir,
- "requests-summary-reads",
- filepath.Base(cfg.AddOnSecretsLocal.RequestsSummaryReadsTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsLocal.RequestsSummaryReadsCompareS3Dir == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryReadsCompareS3Dir = path.Join("add-on-secrets-local", "requests-summary-reads-compare", cfg.Version)
- }
- if cfg.AddOnSecretsLocal.RequestsSummaryReadsCompareJSONPath == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryReadsCompareJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-local-requests-summary-reads-compare.json"
- }
- if cfg.AddOnSecretsLocal.RequestsSummaryReadsCompareJSONS3Key == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryReadsCompareJSONS3Key = path.Join(
- cfg.AddOnSecretsLocal.S3Dir,
- "requests-summary-reads-compare",
- filepath.Base(cfg.AddOnSecretsLocal.RequestsSummaryReadsCompareJSONPath),
- )
- }
- if cfg.AddOnSecretsLocal.RequestsSummaryReadsCompareTablePath == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryReadsCompareTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-local-requests-summary-reads-compare.txt"
- }
- if cfg.AddOnSecretsLocal.RequestsSummaryReadsCompareTableS3Key == "" {
- cfg.AddOnSecretsLocal.RequestsSummaryReadsCompareTableS3Key = path.Join(
- cfg.AddOnSecretsLocal.S3Dir,
- "requests-summary-reads-compare",
- filepath.Base(cfg.AddOnSecretsLocal.RequestsSummaryReadsCompareTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- return nil
-}
diff --git a/eksconfig/add-on-secrets-remote.go b/eksconfig/add-on-secrets-remote.go
deleted file mode 100644
index 2b9c7890a..000000000
--- a/eksconfig/add-on-secrets-remote.go
+++ /dev/null
@@ -1,425 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "path"
- "path/filepath"
- "strings"
-
- "github.com/aws/aws-k8s-tester/pkg/metrics"
- "github.com/aws/aws-k8s-tester/pkg/randutil"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnSecretsRemote defines parameters for EKS cluster
-// add-on "Secrets" remote.
-// It generates loads from the remote workers (Pod) in the cluster.
-// Each worker writes serially with no concurrency.
-// Configure "DeploymentReplicas" accordingly to increase the concurrency.
-// The main use case is to write a large number of objects to fill up etcd database.
-// And measure latencies for secret encryption.
-type AddOnSecretsRemote struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // S3Dir is the S3 directory to store all test results.
- // It is under the bucket "eksconfig.Config.S3BucketName".
- S3Dir string `json:"s3-dir"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // RepositoryAccountID is the account ID for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryAccountID string `json:"repository-account-id,omitempty"`
- // RepositoryRegion is the ECR repository region to pull from.
- RepositoryRegion string `json:"repository-region,omitempty"`
- // RepositoryName is the repositoryName for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryName string `json:"repository-name,omitempty"`
- // RepositoryImageTag is the image tag for tester ECR image.
- // e.g. "latest" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester:latest"
- RepositoryImageTag string `json:"repository-image-tag,omitempty"`
-
- // Completes is the desired number of successfully finished pods.
- // Write QPS will be client QPS * replicas.
- // Read QPS will be client QPS * replicas.
- Completes int `json:"completes"`
- // Parallels is the the maximum desired number of pods the
- // job should run at any given time.
- // Write QPS will be client QPS * replicas.
- // Read QPS will be client QPS * replicas.
- Parallels int `json:"parallels"`
-
- // Objects is the number of "Secret" objects to write/read.
- Objects int `json:"objects"`
- // ObjectSize is the "Secret" value size in bytes.
- ObjectSize int `json:"object-size"`
-
- // NamePrefix is the prefix of Secret name.
- // If multiple Secret loader is running,
- // this must be unique per worker to avoid name conflicts.
- NamePrefix string `json:"name-prefix"`
-
- // RequestsSummaryWritesOutputNamePrefix is the output path name in "/var/log" directory, used in remote worker.
- RequestsSummaryWritesOutputNamePrefix string `json:"requests-summary-writes-output-name-prefix"`
- // RequestsSummaryReadsOutputNamePrefix is the output path name in "/var/log" directory, used in remote worker.
- RequestsSummaryReadsOutputNamePrefix string `json:"requests-summary-reads-output-name-prefix"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- RequestsRawWritesJSONPath string `json:"requests-raw-writes-json-path" read-only:"true"`
- RequestsRawWritesJSONS3Key string `json:"requests-raw-writes-json-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsRawWritesCompareS3Dir is the s3 directory to store raw data points.
- // Used to comparison results.
- // ref. https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
- RequestsRawWritesCompareS3Dir string `json:"requests-raw-writes-compare-s3-dir"`
- RequestsRawWritesCompareAllJSONPath string `json:"requests-raw-writes-compare-all-json-path" read-only:"true"`
- RequestsRawWritesCompareAllJSONS3Key string `json:"requests-raw-writes-compare-all-json-s3-key" read-only:"true"`
- RequestsRawWritesCompareAllCSVPath string `json:"requests-raw-writes-compare-all-csv-path" read-only:"true"`
- RequestsRawWritesCompareAllCSVS3Key string `json:"requests-raw-writes-compare-all-csv-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWrites is the writes results.
- RequestsSummaryWrites metrics.RequestsSummary `json:"requests-summary-writes,omitempty" read-only:"true"`
- RequestsSummaryWritesJSONPath string `json:"requests-summary-writes-json-path" read-only:"true"`
- RequestsSummaryWritesJSONS3Key string `json:"requests-summary-writes-json-s3-key" read-only:"true"`
- RequestsSummaryWritesTablePath string `json:"requests-summary-writes-table-path" read-only:"true"`
- RequestsSummaryWritesTableS3Key string `json:"requests-summary-writes-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWritesCompareS3Dir is the S3 directory of previous/latest "RequestsSummary".
- // Specify the S3 key in the same bucket of "eksconfig.Config.S3BucketName".
- // Use for regression tests. Specify the value not bound to the cluster directory.
- // Different runs from different clusters reads and writes in this directory.
- RequestsSummaryWritesCompareS3Dir string `json:"requests-summary-writes-compare-s3-dir"`
- RequestsSummaryWritesCompare metrics.RequestsCompare `json:"requests-summary-writes-compare" read-only:"true"`
- RequestsSummaryWritesCompareJSONPath string `json:"requests-summary-writes-compare-json-path" read-only:"true"`
- RequestsSummaryWritesCompareJSONS3Key string `json:"requests-summary-writes-compare-json-s3-key" read-only:"true"`
- RequestsSummaryWritesCompareTablePath string `json:"requests-summary-writes-compare-table-path" read-only:"true"`
- RequestsSummaryWritesCompareTableS3Key string `json:"requests-summary-writes-compare-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- RequestsRawReadsJSONPath string `json:"requests-raw-reads-json-path" read-only:"true"`
- RequestsRawReadsJSONS3Key string `json:"requests-raw-reads-json-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsRawReadsCompareS3Dir is the s3 directory to store raw data points.
- // Used to comparison results.
- // ref. https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
- RequestsRawReadsCompareS3Dir string `json:"requests-raw-reads-compare-s3-dir"`
- RequestsRawReadsCompareAllJSONPath string `json:"requests-raw-reads-compare-all-json-path" read-only:"true"`
- RequestsRawReadsCompareAllJSONS3Key string `json:"requests-raw-reads-compare-all-json-s3-key" read-only:"true"`
- RequestsRawReadsCompareAllCSVPath string `json:"requests-raw-reads-compare-all-csv-path" read-only:"true"`
- RequestsRawReadsCompareAllCSVS3Key string `json:"requests-raw-reads-compare-all-csv-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryReads is the reads results.
- RequestsSummaryReads metrics.RequestsSummary `json:"requests-summary-reads,omitempty" read-only:"true"`
- RequestsSummaryReadsJSONPath string `json:"requests-summary-reads-json-path" read-only:"true"`
- RequestsSummaryReadsJSONS3Key string `json:"requests-summary-reads-json-s3-key" read-only:"true"`
- RequestsSummaryReadsTablePath string `json:"requests-summary-reads-table-path" read-only:"true"`
- RequestsSummaryReadsTableS3Key string `json:"requests-summary-reads-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryReadsCompareS3Dir is the S3 directory of previous/latest "RequestsSummary".
- // Specify the S3 key in the same bucket of "eksconfig.Config.S3BucketName".
- // Use for regression tests. Specify the value not bound to the cluster directory.
- // Different runs from different clusters reads and writes in this directory.
- RequestsSummaryReadsCompareS3Dir string `json:"requests-summary-reads-compare-s3-dir"`
- RequestsSummaryReadsCompare metrics.RequestsCompare `json:"requests-summary-reads-compare" read-only:"true"`
- RequestsSummaryReadsCompareJSONPath string `json:"requests-summary-reads-compare-json-path" read-only:"true"`
- RequestsSummaryReadsCompareJSONS3Key string `json:"requests-summary-reads-compare-json-s3-key" read-only:"true"`
- RequestsSummaryReadsCompareTablePath string `json:"requests-summary-reads-compare-table-path" read-only:"true"`
- RequestsSummaryReadsCompareTableS3Key string `json:"requests-summary-reads-compare-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-}
-
-// EnvironmentVariablePrefixAddOnSecretsRemote is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnSecretsRemote = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_SECRETS_REMOTE_"
-
-// IsEnabledAddOnSecretsRemote returns true if "AddOnSecretsRemote" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnSecretsRemote() bool {
- if cfg.AddOnSecretsRemote == nil {
- return false
- }
- if cfg.AddOnSecretsRemote.Enable {
- return true
- }
- cfg.AddOnSecretsRemote = nil
- return false
-}
-
-func getDefaultAddOnSecretsRemote() *AddOnSecretsRemote {
- return &AddOnSecretsRemote{
- Enable: false,
- Completes: 5,
- Parallels: 5,
- Objects: 10,
- ObjectSize: 10 * 1024, // 10 KB
-
- // writes total 100 MB for "Secret" objects,
- // plus "Pod" objects, writes total 330 MB to etcd
- //
- // with 3 nodes, takes about 1.5 hour for all
- // these "Pod"s to complete
- //
- // Objects: 10000,
- // ObjectSize: 10 * 1024, // 10 KB
-
- NamePrefix: "secret" + randutil.String(5),
-
- RequestsSummaryWritesOutputNamePrefix: "secrets-writes-" + randutil.String(10),
- RequestsSummaryReadsOutputNamePrefix: "secrets-reads-" + randutil.String(10),
- }
-}
-
-func (cfg *Config) GetAddOnSecretsRemoteRepositoryRegion() string {
- if !cfg.IsEnabledAddOnSecretsRemote() {
- return cfg.Region
- }
- return cfg.AddOnSecretsRemote.RepositoryRegion
-}
-
-func (cfg *Config) validateAddOnSecretsRemote() error {
- if !cfg.IsEnabledAddOnSecretsRemote() {
- return nil
- }
-
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnSecretsRemote.Enable true but no node group is enabled")
- }
-
- if cfg.AddOnSecretsRemote.S3Dir == "" {
- cfg.AddOnSecretsRemote.S3Dir = path.Join(cfg.Name, "add-on-secrets-remote")
- }
-
- if cfg.AddOnSecretsRemote.Namespace == "" {
- cfg.AddOnSecretsRemote.Namespace = cfg.Name + "-secrets-remote"
- }
-
- if cfg.AddOnSecretsRemote.RepositoryAccountID == "" {
- return errors.New("AddOnSecretsRemote.RepositoryAccountID empty")
- }
- if cfg.AddOnSecretsRemote.RepositoryRegion == "" {
- cfg.AddOnSecretsRemote.RepositoryRegion = cfg.Region
- }
- if cfg.AddOnSecretsRemote.RepositoryName == "" {
- return errors.New("AddOnSecretsRemote.RepositoryName empty")
- }
- if cfg.AddOnSecretsRemote.RepositoryImageTag == "" {
- return errors.New("AddOnSecretsRemote.RepositoryImageTag empty")
- }
-
- if cfg.AddOnSecretsRemote.Objects == 0 {
- cfg.AddOnSecretsRemote.Objects = 10
- }
- if cfg.AddOnSecretsRemote.ObjectSize == 0 {
- cfg.AddOnSecretsRemote.ObjectSize = 10 * 1024
- }
-
- if cfg.AddOnSecretsRemote.NamePrefix == "" {
- cfg.AddOnSecretsRemote.NamePrefix = "secret" + randutil.String(5)
- }
-
- if cfg.AddOnSecretsRemote.RequestsSummaryWritesOutputNamePrefix == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryWritesOutputNamePrefix = "secrets-writes-" + randutil.String(10)
- }
- if cfg.AddOnSecretsRemote.RequestsSummaryReadsOutputNamePrefix == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryReadsOutputNamePrefix = "secrets-reads-" + randutil.String(10)
- }
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsRemote.RequestsRawWritesJSONPath == "" {
- cfg.AddOnSecretsRemote.RequestsRawWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-remote-requests-writes-raw.json"
- }
- if cfg.AddOnSecretsRemote.RequestsRawWritesJSONS3Key == "" {
- cfg.AddOnSecretsRemote.RequestsRawWritesJSONS3Key = path.Join(
- cfg.AddOnSecretsRemote.S3Dir,
- "requests-raw-writes",
- filepath.Base(cfg.AddOnSecretsRemote.RequestsRawWritesJSONPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsRemote.RequestsRawWritesCompareS3Dir == "" {
- cfg.AddOnSecretsRemote.RequestsRawWritesCompareS3Dir = path.Join("add-on-secrets-remote", "requests-raw-writes-compare", cfg.Version)
- }
- if cfg.AddOnSecretsRemote.RequestsRawWritesCompareAllJSONPath == "" {
- cfg.AddOnSecretsRemote.RequestsRawWritesCompareAllJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-remote-requests-raw-writes-compare-all.json"
- }
- if cfg.AddOnSecretsRemote.RequestsRawWritesCompareAllJSONS3Key == "" {
- cfg.AddOnSecretsRemote.RequestsRawWritesCompareAllJSONS3Key = path.Join(
- cfg.AddOnSecretsRemote.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnSecretsRemote.RequestsRawWritesCompareAllJSONPath),
- )
- }
- if cfg.AddOnSecretsRemote.RequestsRawWritesCompareAllCSVPath == "" {
- cfg.AddOnSecretsRemote.RequestsRawWritesCompareAllCSVPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-remote-requests-raw-writes-compare-all.csv"
- }
- if cfg.AddOnSecretsRemote.RequestsRawWritesCompareAllCSVS3Key == "" {
- cfg.AddOnSecretsRemote.RequestsRawWritesCompareAllCSVS3Key = path.Join(
- cfg.AddOnSecretsRemote.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnSecretsRemote.RequestsRawWritesCompareAllCSVPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsRemote.RequestsSummaryWritesJSONPath == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-remote-requests-summary-writes.json"
- }
- if cfg.AddOnSecretsRemote.RequestsSummaryWritesJSONS3Key == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryWritesJSONS3Key = path.Join(
- cfg.AddOnSecretsRemote.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnSecretsRemote.RequestsSummaryWritesJSONPath),
- )
- }
- if cfg.AddOnSecretsRemote.RequestsSummaryWritesTablePath == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryWritesTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-remote-requests-summary-writes.txt"
- }
- if cfg.AddOnSecretsRemote.RequestsSummaryWritesTableS3Key == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryWritesTableS3Key = path.Join(
- cfg.AddOnSecretsRemote.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnSecretsRemote.RequestsSummaryWritesTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsRemote.RequestsSummaryWritesCompareS3Dir == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryWritesCompareS3Dir = path.Join("add-on-secrets-remote", "requests-summary-writes-compare", cfg.Version)
- }
- if cfg.AddOnSecretsRemote.RequestsSummaryWritesCompareJSONPath == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryWritesCompareJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-remote-requests-summary-writes-compare.json"
- }
- if cfg.AddOnSecretsRemote.RequestsSummaryWritesCompareJSONS3Key == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryWritesCompareJSONS3Key = path.Join(
- cfg.AddOnSecretsRemote.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnSecretsRemote.RequestsSummaryWritesCompareJSONPath),
- )
- }
- if cfg.AddOnSecretsRemote.RequestsSummaryWritesCompareTablePath == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryWritesCompareTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-remote-requests-summary-writes-compare.txt"
- }
- if cfg.AddOnSecretsRemote.RequestsSummaryWritesCompareTableS3Key == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryWritesCompareTableS3Key = path.Join(
- cfg.AddOnSecretsRemote.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnSecretsRemote.RequestsSummaryWritesCompareTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsRemote.RequestsRawReadsJSONPath == "" {
- cfg.AddOnSecretsRemote.RequestsRawReadsJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-remote-requests-raw-reads.json"
- }
- if cfg.AddOnSecretsRemote.RequestsRawReadsJSONS3Key == "" {
- cfg.AddOnSecretsRemote.RequestsRawReadsJSONS3Key = path.Join(
- cfg.AddOnSecretsRemote.S3Dir,
- "requests-raw-reads",
- filepath.Base(cfg.AddOnSecretsRemote.RequestsRawReadsJSONPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsRemote.RequestsRawReadsCompareS3Dir == "" {
- cfg.AddOnSecretsRemote.RequestsRawReadsCompareS3Dir = path.Join("add-on-secrets-remote", "requests-raw-reads-compare", cfg.Version)
- }
- if cfg.AddOnSecretsRemote.RequestsRawReadsCompareAllJSONPath == "" {
- cfg.AddOnSecretsRemote.RequestsRawReadsCompareAllJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-remote-requests-raw-reads-compare-all.json"
- }
- if cfg.AddOnSecretsRemote.RequestsRawReadsCompareAllJSONS3Key == "" {
- cfg.AddOnSecretsRemote.RequestsRawReadsCompareAllJSONS3Key = path.Join(
- cfg.AddOnSecretsRemote.S3Dir,
- "requests-raw-reads-compare-all",
- filepath.Base(cfg.AddOnSecretsRemote.RequestsRawReadsCompareAllJSONPath),
- )
- }
- if cfg.AddOnSecretsRemote.RequestsRawReadsCompareAllCSVPath == "" {
- cfg.AddOnSecretsRemote.RequestsRawReadsCompareAllCSVPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-remote-requests-raw-reads-compare-all.csv"
- }
- if cfg.AddOnSecretsRemote.RequestsRawReadsCompareAllCSVS3Key == "" {
- cfg.AddOnSecretsRemote.RequestsRawReadsCompareAllCSVS3Key = path.Join(
- cfg.AddOnSecretsRemote.S3Dir,
- "requests-raw-reads-compare-all",
- filepath.Base(cfg.AddOnSecretsRemote.RequestsRawReadsCompareAllCSVPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsRemote.RequestsSummaryReadsJSONPath == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryReadsJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-remote-requests-summary-reads.json"
- }
- if cfg.AddOnSecretsRemote.RequestsSummaryReadsJSONS3Key == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryReadsJSONS3Key = path.Join(
- cfg.AddOnSecretsRemote.S3Dir,
- "requests-summary-reads",
- filepath.Base(cfg.AddOnSecretsRemote.RequestsSummaryReadsJSONPath),
- )
- }
- if cfg.AddOnSecretsRemote.RequestsSummaryReadsTablePath == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryReadsTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-remote-requests-summary-reads.txt"
- }
- if cfg.AddOnSecretsRemote.RequestsSummaryReadsTableS3Key == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryReadsTableS3Key = path.Join(
- cfg.AddOnSecretsRemote.S3Dir,
- "requests-summary-reads",
- filepath.Base(cfg.AddOnSecretsRemote.RequestsSummaryReadsTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnSecretsRemote.RequestsSummaryReadsCompareS3Dir == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryReadsCompareS3Dir = path.Join("add-on-secrets-remote", "requests-summary-reads-compare", cfg.Version)
- }
- if cfg.AddOnSecretsRemote.RequestsSummaryReadsCompareJSONPath == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryReadsCompareJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-remote-requests-summary-reads-compare.json"
- }
- if cfg.AddOnSecretsRemote.RequestsSummaryReadsCompareJSONS3Key == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryReadsCompareJSONS3Key = path.Join(
- cfg.AddOnSecretsRemote.S3Dir,
- "requests-summary-reads-compare",
- filepath.Base(cfg.AddOnSecretsRemote.RequestsSummaryReadsCompareJSONPath),
- )
- }
- if cfg.AddOnSecretsRemote.RequestsSummaryReadsCompareTablePath == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryReadsCompareTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-secrets-remote-requests-summary-reads-compare.txt"
- }
- if cfg.AddOnSecretsRemote.RequestsSummaryReadsCompareTableS3Key == "" {
- cfg.AddOnSecretsRemote.RequestsSummaryReadsCompareTableS3Key = path.Join(
- cfg.AddOnSecretsRemote.S3Dir,
- "requests-summary-reads-compare",
- filepath.Base(cfg.AddOnSecretsRemote.RequestsSummaryReadsCompareTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- return nil
-}
diff --git a/eksconfig/add-on-stresser-local.go b/eksconfig/add-on-stresser-local.go
deleted file mode 100644
index ddb7765f5..000000000
--- a/eksconfig/add-on-stresser-local.go
+++ /dev/null
@@ -1,343 +0,0 @@
-package eksconfig
-
-import (
- "path"
- "path/filepath"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/pkg/metrics"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnStresserLocal defines parameters for EKS cluster
-// add-on cluster loader local.
-// It generates loads from the local host machine.
-// Every request for write/read is sent serially with no concurrency.
-// Use remote tester to write and read with concurrency.
-type AddOnStresserLocal struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // S3Dir is the S3 directory to store all test results.
- // It is under the bucket "eksconfig.Config.S3BucketName".
- S3Dir string `json:"s3-dir"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // ObjectSize is the value size in bytes for write objects.
- ObjectSize int `json:"object-size"`
- // ListLimit is the maximum number of items in the list call.
- // Sets "metav1.ListOptions".Limit field.
- // 0 to list all.
- ListLimit int64 `json:"list-limit"`
- // Duration is the duration to run load testing.
- Duration time.Duration `json:"duration,omitempty"`
- DurationString string `json:"duration-string,omitempty" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- RequestsRawWritesJSONPath string `json:"requests-raw-writes-json-path" read-only:"true"`
- RequestsRawWritesJSONS3Key string `json:"requests-raw-writes-json-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsRawWritesCompareS3Dir is the s3 directory to store raw data points.
- // Used to comparison results.
- // ref. https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
- RequestsRawWritesCompareS3Dir string `json:"requests-raw-writes-compare-s3-dir"`
- RequestsRawWritesCompareAllJSONPath string `json:"requests-raw-writes-compare-all-json-path" read-only:"true"`
- RequestsRawWritesCompareAllJSONS3Key string `json:"requests-raw-writes-compare-all-json-s3-key" read-only:"true"`
- RequestsRawWritesCompareAllCSVPath string `json:"requests-raw-writes-compare-all-csv-path" read-only:"true"`
- RequestsRawWritesCompareAllCSVS3Key string `json:"requests-raw-writes-compare-all-csv-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWrites is the writes results.
- RequestsSummaryWrites metrics.RequestsSummary `json:"requests-summary-writes,omitempty" read-only:"true"`
- RequestsSummaryWritesJSONPath string `json:"requests-summary-writes-json-path" read-only:"true"`
- RequestsSummaryWritesJSONS3Key string `json:"requests-summary-writes-json-s3-key" read-only:"true"`
- RequestsSummaryWritesTablePath string `json:"requests-summary-writes-table-path" read-only:"true"`
- RequestsSummaryWritesTableS3Key string `json:"requests-summary-writes-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWritesCompareS3Dir is the S3 directory of previous/latest "RequestsSummary".
- // Specify the S3 key in the same bucket of "eksconfig.Config.S3BucketName".
- // Use for regression tests. Specify the value not bound to the cluster directory.
- // Different runs from different clusters reads and writes in this directory.
- RequestsSummaryWritesCompareS3Dir string `json:"requests-summary-writes-compare-s3-dir"`
- RequestsSummaryWritesCompare metrics.RequestsCompare `json:"requests-summary-writes-compare" read-only:"true"`
- RequestsSummaryWritesCompareJSONPath string `json:"requests-summary-writes-compare-json-path" read-only:"true"`
- RequestsSummaryWritesCompareJSONS3Key string `json:"requests-summary-writes-compare-json-s3-key" read-only:"true"`
- RequestsSummaryWritesCompareTablePath string `json:"requests-summary-writes-compare-table-path" read-only:"true"`
- RequestsSummaryWritesCompareTableS3Key string `json:"requests-summary-writes-compare-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- RequestsRawReadsJSONPath string `json:"requests-raw-reads-json-path" read-only:"true"`
- RequestsRawReadsJSONS3Key string `json:"requests-raw-reads-json-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsRawReadsCompareS3Dir is the s3 directory to store raw data points.
- // Used to comparison results.
- // ref. https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
- RequestsRawReadsCompareS3Dir string `json:"requests-raw-reads-compare-s3-dir"`
- RequestsRawReadsCompareAllJSONPath string `json:"requests-raw-reads-compare-all-json-path" read-only:"true"`
- RequestsRawReadsCompareAllJSONS3Key string `json:"requests-raw-reads-compare-all-json-s3-key" read-only:"true"`
- RequestsRawReadsCompareAllCSVPath string `json:"requests-raw-reads-compare-all-csv-path" read-only:"true"`
- RequestsRawReadsCompareAllCSVS3Key string `json:"requests-raw-reads-compare-all-csv-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryReads is the reads results.
- RequestsSummaryReads metrics.RequestsSummary `json:"requests-summary-reads,omitempty" read-only:"true"`
- RequestsSummaryReadsJSONPath string `json:"requests-summary-reads-json-path" read-only:"true"`
- RequestsSummaryReadsJSONS3Key string `json:"requests-summary-reads-json-s3-key" read-only:"true"`
- RequestsSummaryReadsTablePath string `json:"requests-summary-reads-table-path" read-only:"true"`
- RequestsSummaryReadsTableS3Key string `json:"requests-summary-reads-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryReadsCompareS3Dir is the S3 directory of previous/latest "RequestsSummary".
- // Specify the S3 key in the same bucket of "eksconfig.Config.S3BucketName".
- // Use for regression tests. Specify the value not bound to the cluster directory.
- // Different runs from different clusters reads and writes in this directory.
- RequestsSummaryReadsCompareS3Dir string `json:"requests-summary-reads-compare-s3-dir"`
- RequestsSummaryReadsCompare metrics.RequestsCompare `json:"requests-summary-reads-compare" read-only:"true"`
- RequestsSummaryReadsCompareJSONPath string `json:"requests-summary-reads-compare-json-path" read-only:"true"`
- RequestsSummaryReadsCompareJSONS3Key string `json:"requests-summary-reads-compare-json-s3-key" read-only:"true"`
- RequestsSummaryReadsCompareTablePath string `json:"requests-summary-reads-compare-table-path" read-only:"true"`
- RequestsSummaryReadsCompareTableS3Key string `json:"requests-summary-reads-compare-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-}
-
-// EnvironmentVariablePrefixAddOnStresserLocal is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnStresserLocal = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_STRESSER_LOCAL_"
-
-// IsEnabledAddOnStresserLocal returns true if "AddOnStresserLocal" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnStresserLocal() bool {
- if cfg.AddOnStresserLocal == nil {
- return false
- }
- if cfg.AddOnStresserLocal.Enable {
- return true
- }
- cfg.AddOnStresserLocal = nil
- return false
-}
-
-func getDefaultAddOnStresserLocal() *AddOnStresserLocal {
- return &AddOnStresserLocal{
- Enable: false,
- ObjectSize: 0,
- ListLimit: 0,
- Duration: time.Minute,
- }
-}
-
-func (cfg *Config) validateAddOnStresserLocal() error {
- if !cfg.IsEnabledAddOnStresserLocal() {
- return nil
- }
-
- if cfg.AddOnStresserLocal.S3Dir == "" {
- cfg.AddOnStresserLocal.S3Dir = path.Join(cfg.Name, "add-on-stresser-local")
- }
-
- if cfg.AddOnStresserLocal.Namespace == "" {
- cfg.AddOnStresserLocal.Namespace = cfg.Name + "-stresser-local"
- }
-
- if cfg.AddOnStresserLocal.Duration == time.Duration(0) {
- cfg.AddOnStresserLocal.Duration = time.Minute
- }
- cfg.AddOnStresserLocal.DurationString = cfg.AddOnStresserLocal.Duration.String()
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserLocal.RequestsRawWritesJSONPath == "" {
- cfg.AddOnStresserLocal.RequestsRawWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-local-requests-writes-raw.json"
- }
- if cfg.AddOnStresserLocal.RequestsRawWritesJSONS3Key == "" {
- cfg.AddOnStresserLocal.RequestsRawWritesJSONS3Key = path.Join(
- cfg.AddOnStresserLocal.S3Dir,
- "requests-raw-writes",
- filepath.Base(cfg.AddOnStresserLocal.RequestsRawWritesJSONPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserLocal.RequestsRawWritesCompareS3Dir == "" {
- cfg.AddOnStresserLocal.RequestsRawWritesCompareS3Dir = path.Join("add-on-stresser-local", "requests-raw-writes-compare", cfg.Version)
- }
- if cfg.AddOnStresserLocal.RequestsRawWritesCompareAllJSONPath == "" {
- cfg.AddOnStresserLocal.RequestsRawWritesCompareAllJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-local-requests-raw-writes-compare-all.json"
- }
- if cfg.AddOnStresserLocal.RequestsRawWritesCompareAllJSONS3Key == "" {
- cfg.AddOnStresserLocal.RequestsRawWritesCompareAllJSONS3Key = path.Join(
- cfg.AddOnStresserLocal.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnStresserLocal.RequestsRawWritesCompareAllJSONPath),
- )
- }
- if cfg.AddOnStresserLocal.RequestsRawWritesCompareAllCSVPath == "" {
- cfg.AddOnStresserLocal.RequestsRawWritesCompareAllCSVPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-local-requests-raw-writes-compare-all.csv"
- }
- if cfg.AddOnStresserLocal.RequestsRawWritesCompareAllCSVS3Key == "" {
- cfg.AddOnStresserLocal.RequestsRawWritesCompareAllCSVS3Key = path.Join(
- cfg.AddOnStresserLocal.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnStresserLocal.RequestsRawWritesCompareAllCSVPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserLocal.RequestsSummaryWritesJSONPath == "" {
- cfg.AddOnStresserLocal.RequestsSummaryWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-local-requests-summary-writes.json"
- }
- if cfg.AddOnStresserLocal.RequestsSummaryWritesJSONS3Key == "" {
- cfg.AddOnStresserLocal.RequestsSummaryWritesJSONS3Key = path.Join(
- cfg.AddOnStresserLocal.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnStresserLocal.RequestsSummaryWritesJSONPath),
- )
- }
- if cfg.AddOnStresserLocal.RequestsSummaryWritesTablePath == "" {
- cfg.AddOnStresserLocal.RequestsSummaryWritesTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-local-requests-summary-writes.txt"
- }
- if cfg.AddOnStresserLocal.RequestsSummaryWritesTableS3Key == "" {
- cfg.AddOnStresserLocal.RequestsSummaryWritesTableS3Key = path.Join(
- cfg.AddOnStresserLocal.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnStresserLocal.RequestsSummaryWritesTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserLocal.RequestsSummaryWritesCompareS3Dir == "" {
- cfg.AddOnStresserLocal.RequestsSummaryWritesCompareS3Dir = path.Join("add-on-stresser-local", "requests-summary-writes-compare", cfg.Version)
- }
- if cfg.AddOnStresserLocal.RequestsSummaryWritesCompareJSONPath == "" {
- cfg.AddOnStresserLocal.RequestsSummaryWritesCompareJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-local-requests-summary-writes-compare.json"
- }
- if cfg.AddOnStresserLocal.RequestsSummaryWritesCompareJSONS3Key == "" {
- cfg.AddOnStresserLocal.RequestsSummaryWritesCompareJSONS3Key = path.Join(
- cfg.AddOnStresserLocal.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnStresserLocal.RequestsSummaryWritesCompareJSONPath),
- )
- }
- if cfg.AddOnStresserLocal.RequestsSummaryWritesCompareTablePath == "" {
- cfg.AddOnStresserLocal.RequestsSummaryWritesCompareTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-local-requests-summary-writes-compare.txt"
- }
- if cfg.AddOnStresserLocal.RequestsSummaryWritesCompareTableS3Key == "" {
- cfg.AddOnStresserLocal.RequestsSummaryWritesCompareTableS3Key = path.Join(
- cfg.AddOnStresserLocal.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnStresserLocal.RequestsSummaryWritesCompareTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserLocal.RequestsRawReadsJSONPath == "" {
- cfg.AddOnStresserLocal.RequestsRawReadsJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-local-requests-raw-reads.json"
- }
- if cfg.AddOnStresserLocal.RequestsRawReadsJSONS3Key == "" {
- cfg.AddOnStresserLocal.RequestsRawReadsJSONS3Key = path.Join(
- cfg.AddOnStresserLocal.S3Dir,
- "requests-raw-reads",
- filepath.Base(cfg.AddOnStresserLocal.RequestsRawReadsJSONPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserLocal.RequestsRawReadsCompareS3Dir == "" {
- cfg.AddOnStresserLocal.RequestsRawReadsCompareS3Dir = path.Join("add-on-stresser-local", "requests-raw-reads-compare", cfg.Version)
- }
- if cfg.AddOnStresserLocal.RequestsRawReadsCompareAllJSONPath == "" {
- cfg.AddOnStresserLocal.RequestsRawReadsCompareAllJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-local-requests-raw-reads-compare-all.json"
- }
- if cfg.AddOnStresserLocal.RequestsRawReadsCompareAllJSONS3Key == "" {
- cfg.AddOnStresserLocal.RequestsRawReadsCompareAllJSONS3Key = path.Join(
- cfg.AddOnStresserLocal.S3Dir,
- "requests-raw-reads-compare-all",
- filepath.Base(cfg.AddOnStresserLocal.RequestsRawReadsCompareAllJSONPath),
- )
- }
- if cfg.AddOnStresserLocal.RequestsRawReadsCompareAllCSVPath == "" {
- cfg.AddOnStresserLocal.RequestsRawReadsCompareAllCSVPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-local-requests-raw-reads-compare-all.csv"
- }
- if cfg.AddOnStresserLocal.RequestsRawReadsCompareAllCSVS3Key == "" {
- cfg.AddOnStresserLocal.RequestsRawReadsCompareAllCSVS3Key = path.Join(
- cfg.AddOnStresserLocal.S3Dir,
- "requests-raw-reads-compare-all",
- filepath.Base(cfg.AddOnStresserLocal.RequestsRawReadsCompareAllCSVPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserLocal.RequestsSummaryReadsJSONPath == "" {
- cfg.AddOnStresserLocal.RequestsSummaryReadsJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-local-requests-summary-reads.json"
- }
- if cfg.AddOnStresserLocal.RequestsSummaryReadsJSONS3Key == "" {
- cfg.AddOnStresserLocal.RequestsSummaryReadsJSONS3Key = path.Join(
- cfg.AddOnStresserLocal.S3Dir,
- "requests-summary-reads",
- filepath.Base(cfg.AddOnStresserLocal.RequestsSummaryReadsJSONPath),
- )
- }
- if cfg.AddOnStresserLocal.RequestsSummaryReadsTablePath == "" {
- cfg.AddOnStresserLocal.RequestsSummaryReadsTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-local-requests-summary-reads.txt"
- }
- if cfg.AddOnStresserLocal.RequestsSummaryReadsTableS3Key == "" {
- cfg.AddOnStresserLocal.RequestsSummaryReadsTableS3Key = path.Join(
- cfg.AddOnStresserLocal.S3Dir,
- "requests-summary-reads",
- filepath.Base(cfg.AddOnStresserLocal.RequestsSummaryReadsTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserLocal.RequestsSummaryReadsCompareS3Dir == "" {
- cfg.AddOnStresserLocal.RequestsSummaryReadsCompareS3Dir = path.Join("add-on-stresser-local", "requests-summary-reads-compare", cfg.Version)
- }
- if cfg.AddOnStresserLocal.RequestsSummaryReadsCompareJSONPath == "" {
- cfg.AddOnStresserLocal.RequestsSummaryReadsCompareJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-local-requests-summary-reads-compare.json"
- }
- if cfg.AddOnStresserLocal.RequestsSummaryReadsCompareJSONS3Key == "" {
- cfg.AddOnStresserLocal.RequestsSummaryReadsCompareJSONS3Key = path.Join(
- cfg.AddOnStresserLocal.S3Dir,
- "requests-summary-reads-compare",
- filepath.Base(cfg.AddOnStresserLocal.RequestsSummaryReadsCompareJSONPath),
- )
- }
- if cfg.AddOnStresserLocal.RequestsSummaryReadsCompareTablePath == "" {
- cfg.AddOnStresserLocal.RequestsSummaryReadsCompareTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-local-requests-summary-reads-compare.txt"
- }
- if cfg.AddOnStresserLocal.RequestsSummaryReadsCompareTableS3Key == "" {
- cfg.AddOnStresserLocal.RequestsSummaryReadsCompareTableS3Key = path.Join(
- cfg.AddOnStresserLocal.S3Dir,
- "requests-summary-reads-compare",
- filepath.Base(cfg.AddOnStresserLocal.RequestsSummaryReadsCompareTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- return nil
-}
diff --git a/eksconfig/add-on-stresser-remote-v2.go b/eksconfig/add-on-stresser-remote-v2.go
deleted file mode 100644
index dd787dada..000000000
--- a/eksconfig/add-on-stresser-remote-v2.go
+++ /dev/null
@@ -1,147 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "time"
-
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-type AddOnStresserRemoteV2 struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // RepositoryAccountID is the account ID for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryAccountID string `json:"repository-account-id,omitempty"`
- // RepositoryRegion is the ECR repository region to pull from.
- RepositoryRegion string `json:"repository-region,omitempty"`
- // RepositoryName is the repositoryName for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryName string `json:"repository-name,omitempty"`
- // RepositoryImageTag is the image tag for tester ECR image.
- // e.g. "latest" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester:latest"
- RepositoryImageTag string `json:"repository-image-tag,omitempty"`
-
- // RepositoryBusyBoxName is the repositoryName for busybox ECR image.
- // e.g. "busybox" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/busybox:latest"
- RepositoryBusyBoxName string `json:"repository-busybox-name,omitempty"`
- // RepositoryBusyBoxImageTag is the image tag for busybox ECR image.
- // e.g. "latest" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/busybox:latest"
- RepositoryBusyBoxImageTag string `json:"repository-busybox-image-tag,omitempty"`
-
- // Schedule is the cron schedule (e.g. "*/5 * * * *").
- Schedule string `json:"schedule"`
- // Completes is the desired number of successfully finished pods.
- Completes int `json:"completes"`
- // Parallels is the the maximum desired number of pods the
- // job should run at any given time.
- Parallels int `json:"parallels"`
- // SuccessfulJobsHistoryLimit is the number of successful finished
- // jobs to retain. Defaults to 3.
- SuccessfulJobsHistoryLimit int32 `json:"successful-jobs-history-limit"`
- // FailedJobsHistoryLimit is the number of failed finished jobs
- // to retain. Defaults to 1.
- FailedJobsHistoryLimit int32 `json:"failed-jobs-history-limit"`
-
- // ObjectSize is the value size in bytes for write objects.
- // If 0, do not write anything.
- ObjectSize int `json:"object-size"`
-
- // Duration is the duration to run stress2 testing.
- Duration time.Duration `json:"duration,omitempty"`
- DurationString string `json:"duration-string,omitempty" read-only:"true"`
-
- // Coroutines is the number of concurrent go routines run per job
- Coroutines int `json:"coroutines"`
- // Secrets is the number of secrets generated per job
- Secrets int `json:"secrets"`
-}
-
-// EnvironmentVariablePrefixAddOnStresserRemoteV2 is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnStresserRemoteV2 = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_STRESSER_REMOTE_V2_"
-
-// IsEnabledAddOnStresserRemote returns true if "AddOnStresserRemote" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnStresserRemoteV2() bool {
- if cfg.AddOnStresserRemoteV2 == nil {
- return false
- }
- if cfg.AddOnStresserRemoteV2.Enable {
- return true
- }
- cfg.AddOnStresserRemoteV2 = nil
- return false
-}
-
-func getDefaultAddOnStresserRemoteV2() *AddOnStresserRemoteV2 {
- return &AddOnStresserRemoteV2{
- Enable: false,
- Schedule: "0 */6 * * *", // every 6 hours align with etcd defrag interval
- Completes: 10,
- Parallels: 10,
- SuccessfulJobsHistoryLimit: 3,
- FailedJobsHistoryLimit: 1,
- ObjectSize: 8, // 8 bytes
- Duration: 10 * time.Minute,
- Coroutines: 10,
- Secrets: 10,
- }
-}
-
-func (cfg *Config) GetAddOnStresserRemoteV2RepositoryRegion() string {
- if !cfg.IsEnabledAddOnStresserRemoteV2() {
- return cfg.Region
- }
- return cfg.AddOnStresserRemoteV2.RepositoryRegion
-}
-
-func (cfg *Config) validateAddOnStresserRemoteV2() error {
- if !cfg.IsEnabledAddOnStresserRemoteV2() {
- return nil
- }
-
- if cfg.AddOnStresserRemoteV2.Namespace == "" {
- cfg.AddOnStresserRemoteV2.Namespace = cfg.Name + "-stresser-remote-v2"
- }
-
- if cfg.AddOnStresserRemoteV2.RepositoryAccountID == "" {
- return errors.New("AddOnStresserRemoteV2.RepositoryAccountID empty")
- }
- if cfg.AddOnStresserRemoteV2.RepositoryRegion == "" {
- cfg.AddOnStresserRemoteV2.RepositoryRegion = cfg.Region
- }
- if cfg.AddOnStresserRemoteV2.RepositoryName == "" {
- return errors.New("AddOnStresserRemoteV2.RepositoryName empty")
- }
- if cfg.AddOnStresserRemoteV2.RepositoryImageTag == "" {
- return errors.New("AddOnStresserRemoteV2.RepositoryImageTag empty")
- }
- if cfg.AddOnStresserRemoteV2.RepositoryBusyBoxName == "" {
- return errors.New("AddOnStresserRemoteV2.RepositoryBusyBoxName empty")
- }
- if cfg.AddOnStresserRemoteV2.RepositoryBusyBoxImageTag == "" {
- return errors.New("AddOnStresserRemoteV2.RepositoryBusyBoxImageTag empty")
- }
-
- if cfg.AddOnStresserRemoteV2.Duration == time.Duration(0) {
- cfg.AddOnStresserRemoteV2.Duration = 10 * time.Minute
- }
- cfg.AddOnStresserRemoteV2.DurationString = cfg.AddOnStresserRemoteV2.Duration.String()
- if cfg.AddOnStresserRemoteV2.Coroutines <= 0 {
- cfg.AddOnStresserRemoteV2.Coroutines = 10
- }
- if cfg.AddOnStresserRemoteV2.Secrets >= 0 {
- cfg.AddOnStresserRemoteV2.Secrets = 10
- }
-
- return nil
-}
diff --git a/eksconfig/add-on-stresser-remote.go b/eksconfig/add-on-stresser-remote.go
deleted file mode 100644
index ccbcf76ec..000000000
--- a/eksconfig/add-on-stresser-remote.go
+++ /dev/null
@@ -1,409 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "path"
- "path/filepath"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/pkg/metrics"
- "github.com/aws/aws-k8s-tester/pkg/randutil"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
-)
-
-// AddOnStresserRemote defines parameters for EKS cluster
-// add-on cluster loader remote.
-// It generates loads from the remote workers (Pod) in the cluster.
-// Each worker writes and reads serially with no concurrency.
-// Configure "DeploymentReplicas" accordingly to increase the concurrency.
-// ref. https://github.com/kubernetes/perf-tests
-type AddOnStresserRemote struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // S3Dir is the S3 directory to store all test results.
- // It is under the bucket "eksconfig.Config.S3BucketName".
- S3Dir string `json:"s3-dir"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // RepositoryAccountID is the account ID for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryAccountID string `json:"repository-account-id,omitempty"`
- // RepositoryRegion is the ECR repository region to pull from.
- RepositoryRegion string `json:"repository-region,omitempty"`
- // RepositoryName is the repositoryName for tester ECR image.
- // e.g. "aws/aws-k8s-tester" for "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester"
- RepositoryName string `json:"repository-name,omitempty"`
- // RepositoryImageTag is the image tag for tester ECR image.
- // e.g. "latest" for image URI "[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com/aws/aws-k8s-tester:latest"
- RepositoryImageTag string `json:"repository-image-tag,omitempty"`
-
- // Completes is the desired number of successfully finished pods.
- // Write QPS will be client QPS * replicas.
- // Read QPS will be client QPS * replicas.
- Completes int `json:"completes"`
- // Parallels is the the maximum desired number of pods the
- // job should run at any given time.
- // Write QPS will be client QPS * replicas.
- // Read QPS will be client QPS * replicas.
- Parallels int `json:"parallels"`
-
- // ObjectSize is the value size in bytes for write objects.
- // If 0, do not write anything.
- ObjectSize int `json:"object-size"`
- // ListLimit is the maximum number of items in the list call.
- // Sets "metav1.ListOptions".Limit field.
- // 0 to list all.
- ListLimit int64 `json:"list-limit"`
- // Duration is the duration to run load testing.
- Duration time.Duration `json:"duration,omitempty"`
- DurationString string `json:"duration-string,omitempty" read-only:"true"`
-
- // RequestsSummaryWritesOutputNamePrefix is the output path name in "/var/log" directory, used in remote worker.
- RequestsSummaryWritesOutputNamePrefix string `json:"requests-summary-writes-output-name-prefix"`
- // RequestsSummaryReadsOutputNamePrefix is the output path name in "/var/log" directory, used in remote worker.
- RequestsSummaryReadsOutputNamePrefix string `json:"requests-summary-reads-output-name-prefix"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- RequestsRawWritesJSONPath string `json:"requests-raw-writes-json-path" read-only:"true"`
- RequestsRawWritesJSONS3Key string `json:"requests-raw-writes-json-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsRawWritesCompareS3Dir is the s3 directory to store raw data points.
- // Used to comparison results.
- // ref. https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
- RequestsRawWritesCompareS3Dir string `json:"requests-raw-writes-compare-s3-dir"`
- RequestsRawWritesCompareAllJSONPath string `json:"requests-raw-writes-compare-all-json-path" read-only:"true"`
- RequestsRawWritesCompareAllJSONS3Key string `json:"requests-raw-writes-compare-all-json-s3-key" read-only:"true"`
- RequestsRawWritesCompareAllCSVPath string `json:"requests-raw-writes-compare-all-csv-path" read-only:"true"`
- RequestsRawWritesCompareAllCSVS3Key string `json:"requests-raw-writes-compare-all-csv-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWrites is the writes results.
- RequestsSummaryWrites metrics.RequestsSummary `json:"requests-summary-writes,omitempty" read-only:"true"`
- RequestsSummaryWritesJSONPath string `json:"requests-summary-writes-json-path" read-only:"true"`
- RequestsSummaryWritesJSONS3Key string `json:"requests-summary-writes-json-s3-key" read-only:"true"`
- RequestsSummaryWritesTablePath string `json:"requests-summary-writes-table-path" read-only:"true"`
- RequestsSummaryWritesTableS3Key string `json:"requests-summary-writes-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryWritesCompareS3Dir is the S3 directory of previous/latest "RequestsSummary".
- // Specify the S3 key in the same bucket of "eksconfig.Config.S3BucketName".
- // Use for regression tests. Specify the value not bound to the cluster directory.
- // Different runs from different clusters reads and writes in this directory.
- RequestsSummaryWritesCompareS3Dir string `json:"requests-summary-writes-compare-s3-dir"`
- RequestsSummaryWritesCompare metrics.RequestsCompare `json:"requests-summary-writes-compare" read-only:"true"`
- RequestsSummaryWritesCompareJSONPath string `json:"requests-summary-writes-compare-json-path" read-only:"true"`
- RequestsSummaryWritesCompareJSONS3Key string `json:"requests-summary-writes-compare-json-s3-key" read-only:"true"`
- RequestsSummaryWritesCompareTablePath string `json:"requests-summary-writes-compare-table-path" read-only:"true"`
- RequestsSummaryWritesCompareTableS3Key string `json:"requests-summary-writes-compare-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- RequestsRawReadsJSONPath string `json:"requests-raw-reads-json-path" read-only:"true"`
- RequestsRawReadsJSONS3Key string `json:"requests-raw-reads-json-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsRawReadsCompareS3Dir is the s3 directory to store raw data points.
- // Used to comparison results.
- // ref. https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
- RequestsRawReadsCompareS3Dir string `json:"requests-raw-reads-compare-s3-dir"`
- RequestsRawReadsCompareAllJSONPath string `json:"requests-raw-reads-compare-all-json-path" read-only:"true"`
- RequestsRawReadsCompareAllJSONS3Key string `json:"requests-raw-reads-compare-all-json-s3-key" read-only:"true"`
- RequestsRawReadsCompareAllCSVPath string `json:"requests-raw-reads-compare-all-csv-path" read-only:"true"`
- RequestsRawReadsCompareAllCSVS3Key string `json:"requests-raw-reads-compare-all-csv-s3-key" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryReads is the reads results.
- RequestsSummaryReads metrics.RequestsSummary `json:"requests-summary-reads,omitempty" read-only:"true"`
- RequestsSummaryReadsJSONPath string `json:"requests-summary-reads-json-path" read-only:"true"`
- RequestsSummaryReadsJSONS3Key string `json:"requests-summary-reads-json-s3-key" read-only:"true"`
- RequestsSummaryReadsTablePath string `json:"requests-summary-reads-table-path" read-only:"true"`
- RequestsSummaryReadsTableS3Key string `json:"requests-summary-reads-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-
- // RequestsSummaryReadsCompareS3Dir is the S3 directory of previous/latest "RequestsSummary".
- // Specify the S3 key in the same bucket of "eksconfig.Config.S3BucketName".
- // Use for regression tests. Specify the value not bound to the cluster directory.
- // Different runs from different clusters reads and writes in this directory.
- RequestsSummaryReadsCompareS3Dir string `json:"requests-summary-reads-compare-s3-dir"`
- RequestsSummaryReadsCompare metrics.RequestsCompare `json:"requests-summary-reads-compare" read-only:"true"`
- RequestsSummaryReadsCompareJSONPath string `json:"requests-summary-reads-compare-json-path" read-only:"true"`
- RequestsSummaryReadsCompareJSONS3Key string `json:"requests-summary-reads-compare-json-s3-key" read-only:"true"`
- RequestsSummaryReadsCompareTablePath string `json:"requests-summary-reads-compare-table-path" read-only:"true"`
- RequestsSummaryReadsCompareTableS3Key string `json:"requests-summary-reads-compare-table-s3-path" read-only:"true"`
-
- //////////////////////////////////////////////////////////////////////////////
-}
-
-// EnvironmentVariablePrefixAddOnStresserRemote is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnStresserRemote = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_STRESSER_REMOTE_"
-
-// IsEnabledAddOnStresserRemote returns true if "AddOnStresserRemote" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnStresserRemote() bool {
- if cfg.AddOnStresserRemote == nil {
- return false
- }
- if cfg.AddOnStresserRemote.Enable {
- return true
- }
- cfg.AddOnStresserRemote = nil
- return false
-}
-
-func getDefaultAddOnStresserRemote() *AddOnStresserRemote {
- return &AddOnStresserRemote{
- Enable: false,
- Completes: 5,
- Parallels: 5,
- ObjectSize: 0,
- ListLimit: 0,
- Duration: time.Minute,
- RequestsSummaryWritesOutputNamePrefix: "stresser-writes-" + randutil.String(10),
- RequestsSummaryReadsOutputNamePrefix: "stresser-reads-" + randutil.String(10),
- }
-}
-
-func (cfg *Config) GetAddOnStresserRemoteRepositoryRegion() string {
- if !cfg.IsEnabledAddOnStresserRemote() {
- return cfg.Region
- }
- return cfg.AddOnStresserRemote.RepositoryRegion
-}
-
-func (cfg *Config) validateAddOnStresserRemote() error {
- if !cfg.IsEnabledAddOnStresserRemote() {
- return nil
- }
-
- if cfg.AddOnStresserRemote.S3Dir == "" {
- cfg.AddOnStresserRemote.S3Dir = path.Join(cfg.Name, "add-on-stresser-remote")
- }
-
- if cfg.AddOnStresserRemote.Namespace == "" {
- cfg.AddOnStresserRemote.Namespace = cfg.Name + "-stresser-remote"
- }
-
- if cfg.AddOnStresserRemote.RepositoryAccountID == "" {
- return errors.New("AddOnStresserRemote.RepositoryAccountID empty")
- }
- if cfg.AddOnStresserRemote.RepositoryRegion == "" {
- cfg.AddOnStresserRemote.RepositoryRegion = cfg.Region
- }
- if cfg.AddOnStresserRemote.RepositoryName == "" {
- return errors.New("AddOnStresserRemote.RepositoryName empty")
- }
- if cfg.AddOnStresserRemote.RepositoryImageTag == "" {
- return errors.New("AddOnStresserRemote.RepositoryImageTag empty")
- }
-
- if cfg.AddOnStresserRemote.Duration == time.Duration(0) {
- cfg.AddOnStresserRemote.Duration = time.Minute
- }
- cfg.AddOnStresserRemote.DurationString = cfg.AddOnStresserRemote.Duration.String()
-
- if cfg.AddOnStresserRemote.RequestsSummaryWritesOutputNamePrefix == "" {
- cfg.AddOnStresserRemote.RequestsSummaryWritesOutputNamePrefix = "stresser-writes-" + randutil.String(10)
- }
- if cfg.AddOnStresserRemote.RequestsSummaryReadsOutputNamePrefix == "" {
- cfg.AddOnStresserRemote.RequestsSummaryReadsOutputNamePrefix = "stresser-reads-" + randutil.String(10)
- }
-
- if cfg.AddOnStresserRemote.S3Dir == "" {
- cfg.AddOnStresserRemote.S3Dir = path.Join(cfg.Name, "add-on-stresser-remote")
- }
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserRemote.RequestsRawWritesJSONPath == "" {
- cfg.AddOnStresserRemote.RequestsRawWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-remote-requests-writes-raw.json"
- }
- if cfg.AddOnStresserRemote.RequestsRawWritesJSONS3Key == "" {
- cfg.AddOnStresserRemote.RequestsRawWritesJSONS3Key = path.Join(
- cfg.AddOnStresserRemote.S3Dir,
- "requests-raw-writes",
- filepath.Base(cfg.AddOnStresserRemote.RequestsRawWritesJSONPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserRemote.RequestsRawWritesCompareS3Dir == "" {
- cfg.AddOnStresserRemote.RequestsRawWritesCompareS3Dir = path.Join("add-on-stresser-remote", "requests-raw-writes-compare", cfg.Version)
- }
- if cfg.AddOnStresserRemote.RequestsRawWritesCompareAllJSONPath == "" {
- cfg.AddOnStresserRemote.RequestsRawWritesCompareAllJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-remote-requests-raw-writes-compare-all.json"
- }
- if cfg.AddOnStresserRemote.RequestsRawWritesCompareAllJSONS3Key == "" {
- cfg.AddOnStresserRemote.RequestsRawWritesCompareAllJSONS3Key = path.Join(
- cfg.AddOnStresserRemote.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnStresserRemote.RequestsRawWritesCompareAllJSONPath),
- )
- }
- if cfg.AddOnStresserRemote.RequestsRawWritesCompareAllCSVPath == "" {
- cfg.AddOnStresserRemote.RequestsRawWritesCompareAllCSVPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-remote-requests-raw-writes-compare-all.csv"
- }
- if cfg.AddOnStresserRemote.RequestsRawWritesCompareAllCSVS3Key == "" {
- cfg.AddOnStresserRemote.RequestsRawWritesCompareAllCSVS3Key = path.Join(
- cfg.AddOnStresserRemote.S3Dir,
- "requests-raw-writes-compare-all",
- filepath.Base(cfg.AddOnStresserRemote.RequestsRawWritesCompareAllCSVPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserRemote.RequestsSummaryWritesJSONPath == "" {
- cfg.AddOnStresserRemote.RequestsSummaryWritesJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-remote-requests-summary-writes.json"
- }
- if cfg.AddOnStresserRemote.RequestsSummaryWritesJSONS3Key == "" {
- cfg.AddOnStresserRemote.RequestsSummaryWritesJSONS3Key = path.Join(
- cfg.AddOnStresserRemote.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnStresserRemote.RequestsSummaryWritesJSONPath),
- )
- }
- if cfg.AddOnStresserRemote.RequestsSummaryWritesTablePath == "" {
- cfg.AddOnStresserRemote.RequestsSummaryWritesTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-remote-requests-summary-writes.txt"
- }
- if cfg.AddOnStresserRemote.RequestsSummaryWritesTableS3Key == "" {
- cfg.AddOnStresserRemote.RequestsSummaryWritesTableS3Key = path.Join(
- cfg.AddOnStresserRemote.S3Dir,
- "requests-summary-writes",
- filepath.Base(cfg.AddOnStresserRemote.RequestsSummaryWritesTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserRemote.RequestsSummaryWritesCompareS3Dir == "" {
- cfg.AddOnStresserRemote.RequestsSummaryWritesCompareS3Dir = path.Join("add-on-stresser-remote", "requests-summary-writes-compare", cfg.Version)
- }
- if cfg.AddOnStresserRemote.RequestsSummaryWritesCompareJSONPath == "" {
- cfg.AddOnStresserRemote.RequestsSummaryWritesCompareJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-remote-requests-summary-writes-compare.json"
- }
- if cfg.AddOnStresserRemote.RequestsSummaryWritesCompareJSONS3Key == "" {
- cfg.AddOnStresserRemote.RequestsSummaryWritesCompareJSONS3Key = path.Join(
- cfg.AddOnStresserRemote.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnStresserRemote.RequestsSummaryWritesCompareJSONPath),
- )
- }
- if cfg.AddOnStresserRemote.RequestsSummaryWritesCompareTablePath == "" {
- cfg.AddOnStresserRemote.RequestsSummaryWritesCompareTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-remote-requests-summary-writes-compare.txt"
- }
- if cfg.AddOnStresserRemote.RequestsSummaryWritesCompareTableS3Key == "" {
- cfg.AddOnStresserRemote.RequestsSummaryWritesCompareTableS3Key = path.Join(
- cfg.AddOnStresserRemote.S3Dir,
- "requests-summary-writes-compare",
- filepath.Base(cfg.AddOnStresserRemote.RequestsSummaryWritesCompareTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserRemote.RequestsRawReadsJSONPath == "" {
- cfg.AddOnStresserRemote.RequestsRawReadsJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-remote-requests-raw-reads.json"
- }
- if cfg.AddOnStresserRemote.RequestsRawReadsJSONS3Key == "" {
- cfg.AddOnStresserRemote.RequestsRawReadsJSONS3Key = path.Join(
- cfg.AddOnStresserRemote.S3Dir,
- "requests-raw-reads",
- filepath.Base(cfg.AddOnStresserRemote.RequestsRawReadsJSONPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserRemote.RequestsRawReadsCompareS3Dir == "" {
- cfg.AddOnStresserRemote.RequestsRawReadsCompareS3Dir = path.Join("add-on-stresser-remote", "requests-raw-reads-compare", cfg.Version)
- }
- if cfg.AddOnStresserRemote.RequestsRawReadsCompareAllJSONPath == "" {
- cfg.AddOnStresserRemote.RequestsRawReadsCompareAllJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-remote-requests-raw-reads-compare-all.json"
- }
- if cfg.AddOnStresserRemote.RequestsRawReadsCompareAllJSONS3Key == "" {
- cfg.AddOnStresserRemote.RequestsRawReadsCompareAllJSONS3Key = path.Join(
- cfg.AddOnStresserRemote.S3Dir,
- "requests-raw-reads-compare-all",
- filepath.Base(cfg.AddOnStresserRemote.RequestsRawReadsCompareAllJSONPath),
- )
- }
- if cfg.AddOnStresserRemote.RequestsRawReadsCompareAllCSVPath == "" {
- cfg.AddOnStresserRemote.RequestsRawReadsCompareAllCSVPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-remote-requests-raw-reads-compare-all.csv"
- }
- if cfg.AddOnStresserRemote.RequestsRawReadsCompareAllCSVS3Key == "" {
- cfg.AddOnStresserRemote.RequestsRawReadsCompareAllCSVS3Key = path.Join(
- cfg.AddOnStresserRemote.S3Dir,
- "requests-raw-reads-compare-all",
- filepath.Base(cfg.AddOnStresserRemote.RequestsRawReadsCompareAllCSVPath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserRemote.RequestsSummaryReadsJSONPath == "" {
- cfg.AddOnStresserRemote.RequestsSummaryReadsJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-remote-requests-summary-reads.json"
- }
- if cfg.AddOnStresserRemote.RequestsSummaryReadsJSONS3Key == "" {
- cfg.AddOnStresserRemote.RequestsSummaryReadsJSONS3Key = path.Join(
- cfg.AddOnStresserRemote.S3Dir,
- "requests-summary-reads",
- filepath.Base(cfg.AddOnStresserRemote.RequestsSummaryReadsJSONPath),
- )
- }
- if cfg.AddOnStresserRemote.RequestsSummaryReadsTablePath == "" {
- cfg.AddOnStresserRemote.RequestsSummaryReadsTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-remote-requests-summary-reads.txt"
- }
- if cfg.AddOnStresserRemote.RequestsSummaryReadsTableS3Key == "" {
- cfg.AddOnStresserRemote.RequestsSummaryReadsTableS3Key = path.Join(
- cfg.AddOnStresserRemote.S3Dir,
- "requests-summary-reads",
- filepath.Base(cfg.AddOnStresserRemote.RequestsSummaryReadsTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////
- if cfg.AddOnStresserRemote.RequestsSummaryReadsCompareS3Dir == "" {
- cfg.AddOnStresserRemote.RequestsSummaryReadsCompareS3Dir = path.Join("add-on-stresser-remote", "requests-summary-reads-compare", cfg.Version)
- }
- if cfg.AddOnStresserRemote.RequestsSummaryReadsCompareJSONPath == "" {
- cfg.AddOnStresserRemote.RequestsSummaryReadsCompareJSONPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-remote-requests-summary-reads-compare.json"
- }
- if cfg.AddOnStresserRemote.RequestsSummaryReadsCompareJSONS3Key == "" {
- cfg.AddOnStresserRemote.RequestsSummaryReadsCompareJSONS3Key = path.Join(
- cfg.AddOnStresserRemote.S3Dir,
- "requests-summary-reads-compare",
- filepath.Base(cfg.AddOnStresserRemote.RequestsSummaryReadsCompareJSONPath),
- )
- }
- if cfg.AddOnStresserRemote.RequestsSummaryReadsCompareTablePath == "" {
- cfg.AddOnStresserRemote.RequestsSummaryReadsCompareTablePath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + "-stresser-remote-requests-summary-reads-compare.txt"
- }
- if cfg.AddOnStresserRemote.RequestsSummaryReadsCompareTableS3Key == "" {
- cfg.AddOnStresserRemote.RequestsSummaryReadsCompareTableS3Key = path.Join(
- cfg.AddOnStresserRemote.S3Dir,
- "requests-summary-reads-compare",
- filepath.Base(cfg.AddOnStresserRemote.RequestsSummaryReadsCompareTablePath),
- )
- }
- //////////////////////////////////////////////////////////////////////////////
-
- return nil
-}
diff --git a/eksconfig/add-on-wordpress.go b/eksconfig/add-on-wordpress.go
deleted file mode 100644
index ceed3310d..000000000
--- a/eksconfig/add-on-wordpress.go
+++ /dev/null
@@ -1,128 +0,0 @@
-package eksconfig
-
-import (
- "errors"
- "fmt"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/pkg/randutil"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- "github.com/aws/aws-sdk-go/service/eks"
-)
-
-// AddOnWordpress defines parameters for EKS cluster
-// add-on WordPress.
-// ref. https://github.com/helm/charts/blob/master/stable/wordpress/requirements.yaml
-// ref. https://github.com/helm/charts/tree/master/stable/mariadb
-// ref. https://github.com/bitnami/charts/tree/master/bitnami/wordpress/#installing-the-chart
-type AddOnWordpress struct {
- // Enable is 'true' to create this add-on.
- Enable bool `json:"enable"`
- // Created is true when the resource has been created.
- // Used for delete operations.
- Created bool `json:"created" read-only:"true"`
-
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // Namespace is the namespace to create objects in.
- Namespace string `json:"namespace"`
-
- // UserName is the user name.
- // ref. https://github.com/helm/charts/tree/master/stable/wordpress
- UserName string `json:"user-name"`
- // Password is the user password.
- // ref. https://github.com/helm/charts/tree/master/stable/wordpress
- Password string `json:"password"`
-
- // NLBARN is the ARN of the NLB created from the service.
- NLBARN string `json:"nlb-arn" read-only:"true"`
- // NLBName is the name of the NLB created from the service.
- NLBName string `json:"nlb-name" read-only:"true"`
- // URL is the host name for WordPress service.
- URL string `json:"url" read-only:"true"`
-}
-
-// EnvironmentVariablePrefixAddOnWordpress is the environment variable prefix used for "eksconfig".
-const EnvironmentVariablePrefixAddOnWordpress = AWS_K8S_TESTER_EKS_PREFIX + "ADD_ON_WORDPRESS_"
-
-// IsEnabledAddOnWordpress returns true if "AddOnWordpress" is enabled.
-// Otherwise, nil the field for "omitempty".
-func (cfg *Config) IsEnabledAddOnWordpress() bool {
- if cfg.AddOnWordpress == nil {
- return false
- }
- if cfg.AddOnWordpress.Enable {
- return true
- }
- cfg.AddOnWordpress = nil
- return false
-}
-
-func getDefaultAddOnWordpress() *AddOnWordpress {
- return &AddOnWordpress{
- Enable: false,
- UserName: "user",
- Password: "",
- }
-}
-
-func (cfg *Config) validateAddOnWordpress() error {
- if !cfg.IsEnabledAddOnWordpress() {
- return nil
- }
- if !cfg.IsEnabledAddOnCSIEBS() {
- return errors.New("AddOnWordpress.Enable true but IsEnabledAddOnCSIEBS.Enable false")
- }
- if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
- return errors.New("AddOnWordpress.Enable true but no node group is enabled")
- }
-
- // TODO: PVC not working on BottleRocket
- // do not assign mariadb to Bottlerocket
- // e.g. MountVolume.MountDevice failed for volume "pvc-8e035a13-4d33-472f-a4c0-f36c7d39d170" : executable file not found in $PATH
- // e.g. Unable to mount volumes for pod "wordpress-84c567b89b-2jgh5_eks-2020042114-exclusivea3i-wordpress(d02336a3-1799-4b08-9f15-b90871f6a2f0)": timeout expired waiting for volumes to attach or mount for pod "eks-2020042114-exclusivea3i-wordpress"/"wordpress-84c567b89b-2jgh5". list of unmounted volumes=[wordpress-data]. list of unattached volumes=[wordpress-data default-token-7bdc2]
- // TODO: fix CSI EBS https://github.com/bottlerocket-os/bottlerocket/issues/877
- if cfg.IsEnabledAddOnNodeGroups() {
- x86Found, rocketFound := false, false
- for _, asg := range cfg.AddOnNodeGroups.ASGs {
- switch asg.AMIType {
- case ec2config.AMITypeAL2X8664,
- ec2config.AMITypeAL2X8664GPU:
- x86Found = true
- case ec2config.AMITypeBottleRocketCPU:
- rocketFound = true
- }
- }
- if !x86Found && rocketFound {
- return fmt.Errorf("AddOnWordpress.Enabled true but AddOnNodeGroups [x86Found %v, rocketFound %v]", x86Found, rocketFound)
- }
- }
- if cfg.IsEnabledAddOnManagedNodeGroups() {
- x86Found, rocketFound := false, false
- for _, asg := range cfg.AddOnManagedNodeGroups.MNGs {
- switch asg.AMIType {
- case eks.AMITypesAl2X8664,
- eks.AMITypesAl2X8664Gpu:
- x86Found = true
- case ec2config.AMITypeBottleRocketCPU:
- rocketFound = true
- }
- }
- if !x86Found && rocketFound {
- return fmt.Errorf("AddOnWordpress.Enabled true but AddOnManagedNodeGroups [x86Found %v, rocketFound %v]", x86Found, rocketFound)
- }
- }
-
- if cfg.AddOnWordpress.Namespace == "" {
- cfg.AddOnWordpress.Namespace = cfg.Name + "-wordpress"
- }
- if cfg.AddOnWordpress.UserName == "" {
- cfg.AddOnWordpress.UserName = "user"
- }
- if cfg.AddOnWordpress.Password == "" {
- cfg.AddOnWordpress.Password = randutil.String(10)
- }
-
- return nil
-}
diff --git a/eksconfig/addon.go b/eksconfig/addon.go
deleted file mode 100644
index b230af7b0..000000000
--- a/eksconfig/addon.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package eksconfig
-
-// Addon is an interface for configuration initialization
-type Addon interface {
- // Validate the addon against the config.
- Validate(cfg *Config) error
- // Default the addon with the config.
- Default(cfg *Config)
-}
diff --git a/eksconfig/artifacts/clusterloader2-testing-load-config.yaml b/eksconfig/artifacts/clusterloader2-testing-load-config.yaml
deleted file mode 100644
index 42257c37d..000000000
--- a/eksconfig/artifacts/clusterloader2-testing-load-config.yaml
+++ /dev/null
@@ -1,701 +0,0 @@
-# ASSUMPTIONS:
-# - Underlying cluster should have 100+ nodes.
-# - Number of nodes should be divisible by NODES_PER_NAMESPACE (default 100).
-# - The number of created SVCs is half the number of created Deployments.
-# - Only half of Deployments will be assigned 1-1 to existing SVCs.
-
-#Constants
-{{$NODE_MODE := DefaultParam .NODE_MODE "allnodes"}}
-{{$NODES_PER_NAMESPACE := DefaultParam .NODES_PER_NAMESPACE 100}}
-{{$PODS_PER_NODE := DefaultParam .PODS_PER_NODE 30}}
-{{$LOAD_TEST_THROUGHPUT := DefaultParam .CL2_LOAD_TEST_THROUGHPUT 10}}
-{{$BIG_GROUP_SIZE := DefaultParam .BIG_GROUP_SIZE 250}}
-{{$MEDIUM_GROUP_SIZE := DefaultParam .MEDIUM_GROUP_SIZE 30}}
-{{$SMALL_GROUP_SIZE := DefaultParam .SMALL_GROUP_SIZE 5}}
-{{$SMALL_STATEFUL_SETS_PER_NAMESPACE := DefaultParam .SMALL_STATEFUL_SETS_PER_NAMESPACE 1}}
-{{$MEDIUM_STATEFUL_SETS_PER_NAMESPACE := DefaultParam .MEDIUM_STATEFUL_SETS_PER_NAMESPACE 1}}
-{{$ENABLE_CHAOSMONKEY := DefaultParam .ENABLE_CHAOSMONKEY false}}
-{{$PROMETHEUS_SCRAPE_KUBE_PROXY := DefaultParam .PROMETHEUS_SCRAPE_KUBE_PROXY true}}
-{{$ENABLE_PROMETHEUS_API_RESPONSIVENESS := DefaultParam .ENABLE_PROMETHEUS_API_RESPONSIVENESS false}}
-{{$ENABLE_PVS := DefaultParam .CL2_ENABLE_PVS true}}
-{{$ENABLE_NETWORKPOLICIES := DefaultParam .CL2_ENABLE_NETWORKPOLICIES false}}
-{{$ENABLE_SYSTEM_POD_METRICS:= DefaultParam .ENABLE_SYSTEM_POD_METRICS true}}
-{{$USE_SIMPLE_LATENCY_QUERY := DefaultParam .USE_SIMPLE_LATENCY_QUERY false}}
-{{$ENABLE_RESTART_COUNT_CHECK := DefaultParam .ENABLE_RESTART_COUNT_CHECK false}}
-{{$RESTART_COUNT_THRESHOLD_OVERRIDES:= DefaultParam .RESTART_COUNT_THRESHOLD_OVERRIDES ""}}
-{{$ALLOWED_SLOW_API_CALLS := DefaultParam .CL2_ALLOWED_SLOW_API_CALLS 0}}
-{{$CUSTOM_API_CALL_THRESHOLDS := DefaultParam .CUSTOM_API_CALL_THRESHOLDS ""}}
-#Variables
-{{$namespaces := DivideInt .Nodes $NODES_PER_NAMESPACE}}
-{{$totalPods := MultiplyInt $namespaces $NODES_PER_NAMESPACE $PODS_PER_NODE}}
-{{$podsPerNamespace := DivideInt $totalPods $namespaces}}
-{{$saturationTime := DivideInt $totalPods $LOAD_TEST_THROUGHPUT}}
-# bigDeployments - 1/4 of namespace pods should be in big Deployments.
-{{$bigDeploymentsPerNamespace := DivideInt $podsPerNamespace (MultiplyInt 4 $BIG_GROUP_SIZE)}}
-# mediumDeployments - 1/4 of namespace pods should be in medium Deployments.
-{{$mediumDeploymentsPerNamespace := DivideInt $podsPerNamespace (MultiplyInt 4 $MEDIUM_GROUP_SIZE)}}
-# smallDeployments - 1/2 of namespace pods should be in small Deployments.
-{{$smallDeploymentsPerNamespace := DivideInt $podsPerNamespace (MultiplyInt 2 $SMALL_GROUP_SIZE)}}
-# Reduce the number of small and medium deployments per namespace
-{{$smallDeploymentsPerNamespace := SubtractInt $smallDeploymentsPerNamespace $SMALL_STATEFUL_SETS_PER_NAMESPACE}}
-{{$mediumDeploymentsPerNamespace := SubtractInt $mediumDeploymentsPerNamespace $MEDIUM_STATEFUL_SETS_PER_NAMESPACE}}
-
-# Reduce the number of small, medium, big deployments per namespace.
-{{$smallDeploymentsPerNamespace := SubtractInt $smallDeploymentsPerNamespace 1}}
-{{$mediumDeploymentsPerNamespace := SubtractInt $mediumDeploymentsPerNamespace 1}}
-{{$bigDeploymentsPerNamespace := SubtractInt $bigDeploymentsPerNamespace 1}}
-
-# Probe measurements shared parameter
-{{$PROBE_MEASUREMENTS_CHECK_PROBES_READY_TIMEOUT := DefaultParam .CL2_PROBE_MEASUREMENTS_CHECK_PROBES_READY_TIMEOUT "5m"}}
-
-name: load
-namespace:
- number: {{$namespaces}}
-tuningSets:
-- name: Sequence
- parallelismLimitedLoad:
- parallelismLimit: 1
-- name: RandomizedSaturationTimeLimited
- RandomizedTimeLimitedLoad:
- timeLimit: {{$saturationTime}}s
-- name: RandomizedScalingTimeLimited
- RandomizedTimeLimitedLoad:
- # The expected number of created/deleted pods is totalPods/4 when scaling,
- # as each RS changes its size from X to a uniform random value in [X/2, 3X/2].
- # To match 10 [pods/s] requirement, we need to divide saturationTime by 4.
- timeLimit: {{DivideInt $saturationTime 4}}s
-{{if $ENABLE_CHAOSMONKEY}}
-chaosMonkey:
- nodeFailure:
- failureRate: 0.01
- interval: 5m
- jitterFactor: 2.0
- simulatedDowntime: 10m
-{{end}}
-steps:
-- name: Starting measurements
- measurements:
- - Identifier: APIResponsivenessPrometheus
- Method: APIResponsivenessPrometheus
- Params:
- action: start
- - Identifier: APIResponsivenessPrometheusSimple
- Method: APIResponsivenessPrometheus
- Params:
- action: start
- - Identifier: PodStartupLatency
- Method: PodStartupLatency
- Params:
- action: start
- labelSelector: group = load
- threshold: 1h
- - Identifier: InClusterNetworkLatency
- Method: InClusterNetworkLatency
- Params:
- action: start
- checkProbesReadyTimeout: {{$PROBE_MEASUREMENTS_CHECK_PROBES_READY_TIMEOUT}}
- replicasPerProbe: {{AddInt 2 (DivideInt .Nodes 100)}}
- - Identifier: DnsLookupLatency
- Method: DnsLookupLatency
- Params:
- action: start
- checkProbesReadyTimeout: {{$PROBE_MEASUREMENTS_CHECK_PROBES_READY_TIMEOUT}}
- replicasPerProbe: {{AddInt 2 (DivideInt .Nodes 100)}}
- {{if $PROMETHEUS_SCRAPE_KUBE_PROXY}}
- - Identifier: NetworkProgrammingLatency
- Method: NetworkProgrammingLatency
- Params:
- action: start
- {{end}}
- - Identifier: TestMetrics
- Method: TestMetrics
- Params:
- action: start
- nodeMode: {{$NODE_MODE}}
- systemPodMetricsEnabled: {{$ENABLE_SYSTEM_POD_METRICS}}
- restartCountThresholdOverrides: {{YamlQuote $RESTART_COUNT_THRESHOLD_OVERRIDES 4}}
- enableRestartCountCheck: {{$ENABLE_RESTART_COUNT_CHECK}}
-
-- name: Creating SVCs
- phases:
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: {{DivideInt (AddInt $bigDeploymentsPerNamespace 1) 2}}
- tuningSet: Sequence
- objectBundle:
- - basename: big-service
- objectTemplatePath: service.yaml
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: {{DivideInt (AddInt $mediumDeploymentsPerNamespace 1) 2}}
- tuningSet: Sequence
- objectBundle:
- - basename: medium-service
- objectTemplatePath: service.yaml
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: {{DivideInt (AddInt $smallDeploymentsPerNamespace 1) 2}}
- tuningSet: Sequence
- objectBundle:
- - basename: small-service
- objectTemplatePath: service.yaml
-
-- name: Creating PriorityClass for DaemonSets
- phases:
- - replicasPerNamespace: 1
- tuningSet: Sequence
- objectBundle:
- - basename: daemonset-priorityclass
- objectTemplatePath: daemonset-priorityclass.yaml
-
-- name: Starting measurement for waiting for pods
- measurements:
- - Identifier: WaitForRunningDeployments
- Method: WaitForControlledPodsRunning
- Params:
- action: start
- apiVersion: apps/v1
- kind: Deployment
- labelSelector: group = load
- operationTimeout: 15m
- - Identifier: WaitForRunningStatefulSets
- Method: WaitForControlledPodsRunning
- Params:
- action: start
- apiVersion: apps/v1
- kind: StatefulSet
- labelSelector: group = load
- operationTimeout: 15m
- - Identifier: WaitForRunningDaemonSets
- Method: WaitForControlledPodsRunning
- Params:
- action: start
- apiVersion: apps/v1
- kind: DaemonSet
- labelSelector: group = load
- operationTimeout: 15m
- - Identifier: WaitForRunningJobs
- Method: WaitForControlledPodsRunning
- Params:
- action: start
- apiVersion: batch/v1
- kind: Job
- labelSelector: group = load
- operationTimeout: 15m
-
-- name: Creating objects
- phases:
- - namespaceRange:
- min: 1
- max: 1
- replicasPerNamespace: 1
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: daemonset
- objectTemplatePath: daemonset.yaml
- templateFillMap:
- Image: k8s.gcr.io/pause:3.0
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: {{$bigDeploymentsPerNamespace}}
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: big-deployment
- objectTemplatePath: configmap.yaml
- - basename: big-deployment
- objectTemplatePath: secret.yaml
- {{if $ENABLE_NETWORKPOLICIES}}
- - basename: big-deployment
- objectTemplatePath: networkpolicy.yaml
- {{end}}
- - basename: big-deployment
- objectTemplatePath: deployment.yaml
- templateFillMap:
- ReplicasMin: {{$BIG_GROUP_SIZE}}
- ReplicasMax: {{$BIG_GROUP_SIZE}}
- SvcName: big-service
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: {{$mediumDeploymentsPerNamespace}}
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: medium-deployment
- objectTemplatePath: configmap.yaml
- - basename: medium-deployment
- objectTemplatePath: secret.yaml
- {{if $ENABLE_NETWORKPOLICIES}}
- - basename: medium-deployment
- objectTemplatePath: networkpolicy.yaml
- {{end}}
- - basename: medium-deployment
- objectTemplatePath: deployment.yaml
- templateFillMap:
- ReplicasMin: {{$MEDIUM_GROUP_SIZE}}
- ReplicasMax: {{$MEDIUM_GROUP_SIZE}}
- SvcName: medium-service
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: {{$smallDeploymentsPerNamespace}}
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: small-deployment
- objectTemplatePath: configmap.yaml
- - basename: small-deployment
- objectTemplatePath: secret.yaml
- {{if $ENABLE_NETWORKPOLICIES}}
- - basename: small-deployment
- objectTemplatePath: networkpolicy.yaml
- {{end}}
- - basename: small-deployment
- objectTemplatePath: deployment.yaml
- templateFillMap:
- ReplicasMin: {{$SMALL_GROUP_SIZE}}
- ReplicasMax: {{$SMALL_GROUP_SIZE}}
- SvcName: small-service
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: {{$SMALL_STATEFUL_SETS_PER_NAMESPACE}}
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: small-statefulset
- objectTemplatePath: statefulset_service.yaml
- - basename: small-statefulset
- objectTemplatePath: statefulset.yaml
- templateFillMap:
- ReplicasMin: {{$SMALL_GROUP_SIZE}}
- ReplicasMax: {{$SMALL_GROUP_SIZE}}
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: {{$MEDIUM_STATEFUL_SETS_PER_NAMESPACE}}
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: medium-statefulset
- objectTemplatePath: statefulset_service.yaml
- - basename: medium-statefulset
- objectTemplatePath: statefulset.yaml
- templateFillMap:
- ReplicasMin: {{$MEDIUM_GROUP_SIZE}}
- ReplicasMax: {{$MEDIUM_GROUP_SIZE}}
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 1
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: small-job
- objectTemplatePath: job.yaml
- templateFillMap:
- ReplicasMin: {{$SMALL_GROUP_SIZE}}
- ReplicasMax: {{$SMALL_GROUP_SIZE}}
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 1
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: medium-job
- objectTemplatePath: job.yaml
- templateFillMap:
- ReplicasMin: {{$MEDIUM_GROUP_SIZE}}
- ReplicasMax: {{$MEDIUM_GROUP_SIZE}}
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 1
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: big-job
- objectTemplatePath: job.yaml
- templateFillMap:
- ReplicasMin: {{$BIG_GROUP_SIZE}}
- ReplicasMax: {{$BIG_GROUP_SIZE}}
-
-- name: Waiting for pods to be running
- measurements:
- - Identifier: WaitForRunningDeployments
- Method: WaitForControlledPodsRunning
- Params:
- action: gather
- - Identifier: WaitForRunningStatefulSets
- Method: WaitForControlledPodsRunning
- Params:
- action: gather
- - Identifier: WaitForRunningDaemonSets
- Method: WaitForControlledPodsRunning
- Params:
- action: gather
- - Identifier: WaitForRunningJobs
- Method: WaitForControlledPodsRunning
- Params:
- action: gather
-
-- name: Scaling and updating objects
- phases:
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: {{$bigDeploymentsPerNamespace}}
- tuningSet: RandomizedScalingTimeLimited
- objectBundle:
- - basename: big-deployment
- objectTemplatePath: deployment.yaml
- templateFillMap:
- ReplicasMin: {{MultiplyInt $BIG_GROUP_SIZE 0.5}}
- ReplicasMax: {{MultiplyInt $BIG_GROUP_SIZE 1.5}}
- SvcName: big-service
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: {{$mediumDeploymentsPerNamespace}}
- tuningSet: RandomizedScalingTimeLimited
- objectBundle:
- - basename: medium-deployment
- objectTemplatePath: deployment.yaml
- templateFillMap:
- ReplicasMin: {{MultiplyInt $MEDIUM_GROUP_SIZE 0.5}}
- ReplicasMax: {{MultiplyInt $MEDIUM_GROUP_SIZE 1.5}}
- SvcName: medium-service
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: {{$smallDeploymentsPerNamespace}}
- tuningSet: RandomizedScalingTimeLimited
- objectBundle:
- - basename: small-deployment
- objectTemplatePath: deployment.yaml
- templateFillMap:
- ReplicasMin: {{MultiplyInt $SMALL_GROUP_SIZE 0.5}}
- ReplicasMax: {{MultiplyInt $SMALL_GROUP_SIZE 1.5}}
- SvcName: small-service
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: {{$SMALL_STATEFUL_SETS_PER_NAMESPACE}}
- tuningSet: RandomizedScalingTimeLimited
- objectBundle:
- - basename: small-statefulset
- objectTemplatePath: statefulset.yaml
- templateFillMap:
- ReplicasMin: {{MultiplyInt $SMALL_GROUP_SIZE 0.5}}
- ReplicasMax: {{MultiplyInt $SMALL_GROUP_SIZE 1.5}}
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: {{$MEDIUM_STATEFUL_SETS_PER_NAMESPACE}}
- tuningSet: RandomizedScalingTimeLimited
- objectBundle:
- - basename: medium-statefulset
- objectTemplatePath: statefulset.yaml
- templateFillMap:
- ReplicasMin: {{MultiplyInt $MEDIUM_GROUP_SIZE 0.5}}
- ReplicasMax: {{MultiplyInt $MEDIUM_GROUP_SIZE 1.5}}
- - namespaceRange:
- min: 1
- max: 1
- replicasPerNamespace: 1
- tuningSet: RandomizedScalingTimeLimited
- objectBundle:
- - basename: daemonset
- objectTemplatePath: daemonset.yaml
- templateFillMap:
- Image: k8s.gcr.io/pause:3.1
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 1
- tuningSet: RandomizedScalingTimeLimited
- objectBundle:
- - basename: small-job
- objectTemplatePath: job.yaml
- templateFillMap:
- ReplicasMin: {{MultiplyInt $SMALL_GROUP_SIZE 0.5}}
- ReplicasMax: {{MultiplyInt $SMALL_GROUP_SIZE 1.5}}
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 1
- tuningSet: RandomizedScalingTimeLimited
- objectBundle:
- - basename: medium-job
- objectTemplatePath: job.yaml
- templateFillMap:
- ReplicasMin: {{MultiplyInt $MEDIUM_GROUP_SIZE 0.5}}
- ReplicasMax: {{MultiplyInt $MEDIUM_GROUP_SIZE 1.5}}
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 1
- tuningSet: RandomizedScalingTimeLimited
- objectBundle:
- - basename: big-job
- objectTemplatePath: job.yaml
- templateFillMap:
- ReplicasMin: {{MultiplyInt $BIG_GROUP_SIZE 0.5}}
- ReplicasMax: {{MultiplyInt $BIG_GROUP_SIZE 1.5}}
-
-- name: Waiting for objects to become scaled
- measurements:
- - Identifier: WaitForRunningDeployments
- Method: WaitForControlledPodsRunning
- Params:
- action: gather
- - Identifier: WaitForRunningStatefulSets
- Method: WaitForControlledPodsRunning
- Params:
- action: gather
- - Identifier: WaitForRunningDaemonSets
- Method: WaitForControlledPodsRunning
- Params:
- action: gather
- - Identifier: WaitForRunningJobs
- Method: WaitForControlledPodsRunning
- Params:
- action: gather
-
-- name: Deleting objects
- phases:
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 0
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: big-deployment
- objectTemplatePath: deployment.yaml
- - basename: big-deployment
- objectTemplatePath: configmap.yaml
- - basename: big-deployment
- objectTemplatePath: secret.yaml
- {{if $ENABLE_NETWORKPOLICIES}}
- - basename: big-deployment
- objectTemplatePath: networkpolicy.yaml
- {{end}}
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 0
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: medium-deployment
- objectTemplatePath: deployment.yaml
- - basename: medium-deployment
- objectTemplatePath: configmap.yaml
- - basename: medium-deployment
- objectTemplatePath: secret.yaml
- {{if $ENABLE_NETWORKPOLICIES}}
- - basename: medium-deployment
- objectTemplatePath: networkpolicy.yaml
- {{end}}
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 0
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: small-deployment
- objectTemplatePath: deployment.yaml
- - basename: small-deployment
- objectTemplatePath: configmap.yaml
- - basename: small-deployment
- objectTemplatePath: secret.yaml
- {{if $ENABLE_NETWORKPOLICIES}}
- - basename: small-deployment
- objectTemplatePath: networkpolicy.yaml
- {{end}}
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 0
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: small-statefulset
- objectTemplatePath: statefulset.yaml
- - basename: small-statefulset
- objectTemplatePath: statefulset_service.yaml
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 0
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: medium-statefulset
- objectTemplatePath: statefulset.yaml
- - basename: medium-statefulset
- objectTemplatePath: statefulset_service.yaml
- - namespaceRange:
- min: 1
- max: 1
- replicasPerNamespace: 0
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: daemonset
- objectTemplatePath: daemonset.yaml
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 0
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: small-job
- objectTemplatePath: job.yaml
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 0
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: medium-job
- objectTemplatePath: job.yaml
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 0
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- - basename: big-job
- objectTemplatePath: job.yaml
- # If both StatefulSets and PVs were enabled we need to delete PVs manually.
- {{if $ENABLE_PVS}}
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 0
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- {{range $ssIndex := Loop $SMALL_STATEFUL_SETS_PER_NAMESPACE}}
- - basename: pv-small-statefulset-{{$ssIndex}}
- objectTemplatePath: pvc.yaml
- listUnknownObjectOptions:
- labelSelector:
- matchLabels:
- name: small-statefulset-{{$ssIndex}}
- {{end}}
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 0
- tuningSet: RandomizedSaturationTimeLimited
- objectBundle:
- {{range $ssIndex := Loop $MEDIUM_STATEFUL_SETS_PER_NAMESPACE}}
- - basename: pv-medium-statefulset-{{$ssIndex}}
- objectTemplatePath: pvc.yaml
- listUnknownObjectOptions:
- labelSelector:
- matchLabels:
- name: medium-statefulset-{{$ssIndex}}
- {{end}}
- {{end}}
-
-- name: Waiting for pods to be deleted
- measurements:
- - Identifier: WaitForRunningDeployments
- Method: WaitForControlledPodsRunning
- Params:
- action: gather
- - Identifier: WaitForRunningStatefulSets
- Method: WaitForControlledPodsRunning
- Params:
- action: gather
- - Identifier: WaitForRunningDaemonSets
- Method: WaitForControlledPodsRunning
- Params:
- action: gather
- - Identifier: WaitForRunningJobs
- Method: WaitForControlledPodsRunning
- Params:
- action: gather
- {{if $ENABLE_PVS}}
- - Identifier: WaitForPVCsToBeDeleted
- Method: WaitForBoundPVCs
- Params:
- desiredPVCCount: 0
- labelSelector: group = load
- timeout: 15m
- {{end}}
-
-- name: Deleting PriorityClass for DaemonSets
- phases:
- - replicasPerNamespace: 0
- tuningSet: Sequence
- objectBundle:
- - basename: daemonset-priorityclass
- objectTemplatePath: daemonset-priorityclass.yaml
-
-- name: Deleting SVCs
- phases:
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 0
- tuningSet: Sequence
- objectBundle:
- - basename: big-service
- objectTemplatePath: service.yaml
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 0
- tuningSet: Sequence
- objectBundle:
- - basename: medium-service
- objectTemplatePath: service.yaml
- - namespaceRange:
- min: 1
- max: {{$namespaces}}
- replicasPerNamespace: 0
- tuningSet: Sequence
- objectBundle:
- - basename: small-service
- objectTemplatePath: service.yaml
-
-- name: Collecting measurements
- measurements:
- - Identifier: APIResponsivenessPrometheusSimple
- Method: APIResponsivenessPrometheus
- Params:
- action: gather
- enableViolations: true
- useSimpleLatencyQuery: true
- summaryName: APIResponsivenessPrometheus_simple
- allowedSlowCalls: {{$ALLOWED_SLOW_API_CALLS}}
- {{if not $USE_SIMPLE_LATENCY_QUERY}}
- - Identifier: APIResponsivenessPrometheus
- Method: APIResponsivenessPrometheus
- Params:
- action: gather
- allowedSlowCalls: {{$ALLOWED_SLOW_API_CALLS}}
- customThresholds: {{YamlQuote $CUSTOM_API_CALL_THRESHOLDS 4}}
- {{end}}
- - Identifier: PodStartupLatency
- Method: PodStartupLatency
- Params:
- action: gather
- - Identifier: InClusterNetworkLatency
- Method: InClusterNetworkLatency
- Params:
- action: gather
- - Identifier: DnsLookupLatency
- Method: DnsLookupLatency
- Params:
- action: gather
- {{if $PROMETHEUS_SCRAPE_KUBE_PROXY}}
- - Identifier: NetworkProgrammingLatency
- Method: NetworkProgrammingLatency
- Params:
- action: gather
- {{end}}
- - Identifier: TestMetrics
- Method: TestMetrics
- Params:
- action: gather
- systemPodMetricsEnabled: {{$ENABLE_SYSTEM_POD_METRICS}}
- restartCountThresholdOverrides: {{YamlQuote $RESTART_COUNT_THRESHOLD_OVERRIDES 4}}
- enableRestartCountCheck: {{$ENABLE_RESTART_COUNT_CHECK}}
diff --git a/eksconfig/cluster-loader.go b/eksconfig/cluster-loader.go
deleted file mode 100644
index 61b36dc71..000000000
--- a/eksconfig/cluster-loader.go
+++ /dev/null
@@ -1,42 +0,0 @@
-package eksconfig
-
-import (
- "fmt"
-)
-
-// ClusterLoaderSpec defines the spec for the Addon
-type ClusterLoaderSpec struct {
- Image string `json:"image,omitempty"`
- Nodes int32 `json:"nodes,omitempty"`
- TestConfigUris []string `json:"testConfigUris,omitempty"`
- TestOverrides []string `json:"testOverrides,omitempty"`
- // Specifies which instance type the clusterloader2 pod will be able to be scheduled on
- // Leaving this empty will allow it to be scheduled on any instance type
- Affinities map[string][]string `json:"affinities,omitempty"`
-}
-
-// ClusterLoaderStatus defines the status for the Addon
-type ClusterLoaderStatus struct {
- AddonStatus `json:",inline"`
-}
-
-// Validate installs the addon
-func (spec *ClusterLoaderSpec) Validate(cfg *Config) error {
- if spec.Nodes <= 0 {
- return fmt.Errorf("ClusterLoaderSpec.Nodes must be greater than 0")
- }
- if len(spec.TestConfigUris) == 0 {
- return fmt.Errorf("ClusterLoaderSpec.TestConfigUris array must have length greater than 0")
- }
- return nil
-}
-
-// Default installs the addon
-func (spec *ClusterLoaderSpec) Default(cfg *Config) {
- if spec.Image == "" {
- spec.Image = "197575167141.dkr.ecr.us-west-2.amazonaws.com/clusterloader2:latest"
- }
- if spec.Affinities == nil {
- spec.Affinities = make(map[string][]string)
- }
-}
diff --git a/eksconfig/clusterautoscaler.go b/eksconfig/clusterautoscaler.go
deleted file mode 100644
index e1b50e5b2..000000000
--- a/eksconfig/clusterautoscaler.go
+++ /dev/null
@@ -1,80 +0,0 @@
-package eksconfig
-
-import (
- corev1 "k8s.io/api/core/v1"
- "k8s.io/apimachinery/pkg/api/resource"
-)
-
-// ClusterAutoscalerSpec defines the spec for the Addon
-type ClusterAutoscalerSpec struct {
- Image string `json:"image,omitempty"`
- CloudProvider CloudProvider `json:"cloudProvider,omitempty"`
- MinNodes int `json:"minNodes,omitempty"`
- MaxNodes int `json:"maxNodes,omitempty"`
- Resources corev1.ResourceRequirements `json:"resources,omitempty"`
- ScaleDownDelay string `json:"scaleDownDelay,omitempty"`
-}
-
-// CloudProvider enum for ClusterAutoscaler
-type CloudProvider string
-
-// CloudProvider options
-const (
- CloudProviderKubemark CloudProvider = "kubemark"
- CloudProviderAWS CloudProvider = "aws"
-)
-
-// ClusterAutoscalerStatus defines the status for the Addon
-type ClusterAutoscalerStatus struct {
- AddonStatus `json:",inline"`
-}
-
-// Validate installs the addon
-func (spec *ClusterAutoscalerSpec) Validate(cfg *Config) error {
- return nil
-}
-
-// Default installs the addon
-func (spec *ClusterAutoscalerSpec) Default(cfg *Config) {
- if spec.CloudProvider == "" {
- spec.CloudProvider = CloudProviderKubemark
- }
- if spec.MinNodes == 0 {
- spec.MinNodes = 1
- }
- if spec.MaxNodes == 0 {
- spec.MaxNodes = 100
- }
- if spec.Resources.Requests == nil {
- spec.Resources.Requests = corev1.ResourceList{}
- }
- if spec.Resources.Limits == nil {
- spec.Resources.Limits = corev1.ResourceList{}
- }
- if spec.Resources.Limits.Cpu().IsZero() {
- spec.Resources.Limits[corev1.ResourceCPU] = resource.MustParse("200M")
- }
- if spec.Resources.Limits.Memory().IsZero() {
- spec.Resources.Limits[corev1.ResourceMemory] = resource.MustParse("1Gi")
- }
- if spec.Resources.Requests.Cpu().IsZero() {
- spec.Resources.Requests[corev1.ResourceCPU] = resource.MustParse("200M")
- }
- if spec.Resources.Requests.Memory().IsZero() {
- spec.Resources.Requests[corev1.ResourceMemory] = resource.MustParse("1Gi")
- }
- if spec.ScaleDownDelay == "" {
- spec.ScaleDownDelay = "30s"
- }
- spec.Image = spec.defaultImage()
-}
-
-func (spec *ClusterAutoscalerSpec) defaultImage() string {
- if spec.Image != "" {
- return spec.Image
- }
- if spec.CloudProvider == CloudProviderKubemark {
- return "197575167141.dkr.ecr.us-west-2.amazonaws.com/cluster-autoscaler-kubemark:latest"
- }
- return "k8s.gcr.io/cluster-autoscaler:v1.14.7"
-}
diff --git a/eksconfig/config.go b/eksconfig/config.go
deleted file mode 100644
index 05427a208..000000000
--- a/eksconfig/config.go
+++ /dev/null
@@ -1,1473 +0,0 @@
-// Package eksconfig defines EKS test configuration.
-package eksconfig
-
-import (
- "bytes"
- "errors"
- "fmt"
- "io/ioutil"
- "os"
- "os/exec"
- "path/filepath"
- "reflect"
- "regexp"
- "runtime"
- "strconv"
- "strings"
- "sync"
- "text/template"
- "time"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/pkg/fileutil"
- "github.com/aws/aws-k8s-tester/pkg/logutil"
- "github.com/aws/aws-k8s-tester/pkg/randutil"
- "github.com/aws/aws-k8s-tester/pkg/terminal"
- aws_ec2_v2_types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
- "github.com/mitchellh/colorstring"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "sigs.k8s.io/yaml" // must use "sigs.k8s.io/yaml"
-)
-
-// Config defines EKS configuration.
-type Config struct {
- mu *sync.RWMutex
-
- // TODO, Migrate metadata fields to here
- metav1.TypeMeta `json:",inline"`
- metav1.ObjectMeta `json:"metadata,omitempty"`
-
- // Name is the cluster name.
- // If empty, deployer auto-populates it.
- Name string `json:"name"`
- // Partition is the AWS partition for EKS deployment region.
- // If empty, set default partition "aws".
- Partition string `json:"partition"`
- // Region is the AWS geographic area for EKS deployment.
- // If empty, set default region.
- Region string `json:"region"`
- // AvailabilityZoneNames lists the availability zones for the specified region.
- // ref. https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html
- AvailabilityZoneNames []string `json:"availability-zone-names,omitempty" read-only:"true"`
-
- // ConfigPath is the configuration file path.
- // Deployer is expected to update this file with latest status.
- ConfigPath string `json:"config-path,omitempty"`
- // KubectlCommandsOutputPath is the output path for kubectl commands.
- KubectlCommandsOutputPath string `json:"kubectl-commands-output-path,omitempty"`
- // RemoteAccessCommandsOutputPath is the output path for ssh commands.
- RemoteAccessCommandsOutputPath string `json:"remote-access-commands-output-path,omitempty"`
-
- // LogColor is true to output logs in color.
- LogColor bool `json:"log-color"`
- // LogColorOverride is not empty to override "LogColor" setting.
- // If not empty, the automatic color check is not even run and use this value instead.
- // For instance, github action worker might not support color device,
- // thus exiting color check with the exit code 1.
- // Useful to output in color in HTML based log outputs (e.g., Prow).
- // Useful to skip terminal color check when there is no color device (e.g., Github action worker).
- LogColorOverride string `json:"log-color-override"`
-
- // LogLevel configures log level. Only supports debug, info, warn, error, panic, or fatal. Default 'info'.
- LogLevel string `json:"log-level"`
- // LogOutputs is a list of log outputs. Valid values are 'default', 'stderr', 'stdout', or file names.
- // Logs are appended to the existing file, if any.
- // Multiple values are accepted. If empty, it sets to 'default', which outputs to stderr.
- // See https://pkg.go.dev/go.uber.org/zap#Open and https://pkg.go.dev/go.uber.org/zap#Config for more details.
- LogOutputs []string `json:"log-outputs,omitempty"`
-
- // AWSCLIPath is the path for AWS CLI path.
- // Required for 'aws eks update-kubeconfig'.
- AWSCLIPath string `json:"aws-cli-path,omitempty"`
-
- // KubectlPath is the path to download the "kubectl".
- KubectlPath string `json:"kubectl-path,omitempty"`
- // KubectlDownloadURL is the download URL to download "kubectl" binary from.
- KubectlDownloadURL string `json:"kubectl-download-url,omitempty"`
- // KubeConfigPath is the file path of KUBECONFIG for the EKS cluster.
- // If empty, auto-generate one.
- // Deployer is expected to delete this on cluster tear down.
- KubeConfigPath string `json:"kubeconfig-path,omitempty"`
-
- // AWSIAMAuthenticatorPath is the path to aws-iam-authenticator.
- AWSIAMAuthenticatorPath string `json:"aws-iam-authenticator-path,omitempty"`
- // AWSIAMAuthenticatorDownloadURL is the download URL to download "aws-iam-authenticator" binary from.
- AWSIAMAuthenticatorDownloadURL string `json:"aws-iam-authenticator-download-url,omitempty"`
- // AuthenticationAPIVersion is the API version used for authenticating the client.
- AuthenticationAPIVersion string `json:"authentication-api-version,omitempty"`
-
- // OnFailureDelete is true to delete all resources on creation fail.
- OnFailureDelete bool `json:"on-failure-delete"`
- // OnFailureDeleteWaitSeconds is the seconds to wait before deleting
- // all resources on creation fail.
- OnFailureDeleteWaitSeconds uint64 `json:"on-failure-delete-wait-seconds"`
-
- // CommandAfterCreateCluster is the command to execute after creating clusters.
- // Currently supported variables are:
- // - "GetRef.Name" for cluster name
- // - "GetRef.ClusterARN" for cluster ARN
- CommandAfterCreateCluster string `json:"command-after-create-cluster"`
- CommandAfterCreateClusterOutputPath string `json:"command-after-create-cluster-output-path" read-only:"true"`
- CommandAfterCreateClusterTimeout time.Duration `json:"command-after-create-cluster-timeout"`
- CommandAfterCreateClusterTimeoutString string `json:"command-after-create-cluster-timeout-string" read-only:"true"`
- // CommandAfterCreateAddOns is the command to execute after creating clusters and add-ons.
- // Currently supported variables are:
- // - "GetRef.Name" for cluster name
- // - "GetRef.ClusterARN" for cluster ARN
- CommandAfterCreateAddOns string `json:"command-after-create-add-ons"`
- CommandAfterCreateAddOnsOutputPath string `json:"command-after-create-add-ons-output-path" read-only:"true"`
- CommandAfterCreateAddOnsTimeout time.Duration `json:"command-after-create-add-ons-timeout"`
- CommandAfterCreateAddOnsTimeoutString string `json:"command-after-create-add-ons-timeout-string" read-only:"true"`
-
- // CWNamespace is the CloudWatch namespace to put metric datum.
- CWNamespace string `json:"cw-namespace"`
-
- // SkipDeleteClusterAndNodes is true to skip EKS "cluster" and "nodes" deletion.
- // The created EKS "cluster" and all resources created "before" cluster are kept.
- // For example, CMK key, VPC, IAM role are not deleted if cluster is to be kept.
- // All node groups and managed node groups are kept.
- // Use this to use existing clusters to create/delete add-ons.
- SkipDeleteClusterAndNodes bool `json:"skip-delete-cluster-and-nodes"`
-
- S3 *S3 `json:"s3"`
- Encryption *Encryption `json:"encryption"`
- Role *Role `json:"role"`
- VPC *VPC `json:"vpc"`
-
- // Tags defines EKS create cluster tags.
- Tags map[string]string `json:"tags"`
- // RequestHeaderKey defines EKS create cluster request header key.
- RequestHeaderKey string `json:"request-header-key"`
- // RequestHeaderValue defines EKS create cluster request header value.
- RequestHeaderValue string `json:"request-header-value"`
-
- // ResolverURL defines an AWS resolver endpoint for EKS API.
- // Must be left empty to use production EKS service.
- ResolverURL string `json:"resolver-url"`
- // SigningName is the EKS create request signing name.
- SigningName string `json:"signing-name"`
-
- // Version is the version of EKS Kubernetes "cluster".
- // If empty, set default version.
- Version string `json:"version"`
- VersionValue float64 `json:"version-value" read-only:"true"`
-
- // EKS internal only
- // If empty, use default kube-controller-manager and kube-scheduler qps and burst
- // ref. https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/
- // ref. https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/
-
- // KubeAPIServerMaxRequestsInflight is the EKS kube-apiserver max-requests-inflight
- // The maximum number of non-mutating requests in flight at a given time. When the server exceeds this, it rejects requests. Zero for no limit.
- // --max-requests-inflight int Default: 400
- KubeAPIServerMaxRequestsInflight string `json:"kube-apiserver-max-requests-inflight"`
- // KubeControllerManagerQPS is the EKS kube-controller-manager qps
- // --kube-api-qps float32 Default: 20
- KubeControllerManagerQPS string `json:"kube-controller-manager-qps,omitempty"`
- // KubeControllerManagerBurst is the EKS kube-controller-manager burst
- // --kube-api-burst int32 Default: 30
- KubeControllerManagerBurst string `json:"kube-controller-manager-burst,omitempty"`
- // KubeSchedulerQPS is the internal EKS kube-scheduler qps
- // --kube-api-qps float32 Default: 50
- KubeSchedulerQPS string `json:"kube-scheduler-qps,omitempty"`
- // KubeSchedulerBurst is the internal EKS kube-scheduler burst
- // --kube-api-burst int32 Default: 100
- KubeSchedulerBurst string `json:"kube-scheduler-burst,omitempty"`
- // FEUpdateMasterFlagsURL is the internal EKS update master flags endpoint
- FEUpdateMasterFlagsURL string `json:"fe-update-master-flags-url,omitempty"`
-
- // RemoteAccessKeyCreate is true to create the remote SSH access private key.
- RemoteAccessKeyCreate bool `json:"remote-access-key-create"`
- // RemoteAccessKeyName is the key name for node group SSH EC2 key pair.
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
- // ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html
- RemoteAccessKeyName string `json:"remote-access-key-name,omitempty"`
- // RemoteAccessPrivateKeyPath is the file path to store node group key pair private key.
- // Thus, deployer must delete the private key right after node group creation.
- // MAKE SURE PRIVATE KEY NEVER GETS UPLOADED TO CLOUD STORAGE AND DELETE AFTER USE!!!
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
- // ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html
- RemoteAccessPrivateKeyPath string `json:"remote-access-private-key-path,omitempty"`
-
- // Clients is the number of kubernetes clients to create.
- // Default is 1.
- // This field is used for "eks/stresser" tester. Configure accordingly.
- // Rate limit is done via "k8s.io/client-go/util/flowcontrol.NewTokenBucketRateLimiter".
- Clients int `json:"clients"`
- // ClientQPS is the QPS for kubernetes client.
- // To use while talking with kubernetes apiserver.
- //
- // Kubernetes client DefaultQPS is 5.
- // Kubernetes client DefaultBurst is 10.
- // ref. https://github.com/kubernetes/kubernetes/blob/4d0e86f0b8d1eae00a202009858c8739e4c9402e/staging/src/k8s.io/client-go/rest/config.go#L43-L46
- //
- // kube-apiserver default inflight requests limits are:
- // FLAG: --max-mutating-requests-inflight="200"
- // FLAG: --max-requests-inflight="400"
- // ref. https://github.com/kubernetes/kubernetes/blob/4d0e86f0b8d1eae00a202009858c8739e4c9402e/staging/src/k8s.io/apiserver/pkg/server/config.go#L300-L301
- //
- // This field is used for "eks/stresser" tester. Configure accordingly.
- // Rate limit is done via "k8s.io/client-go/util/flowcontrol.NewTokenBucketRateLimiter".
- ClientQPS float32 `json:"client-qps"`
- // ClientBurst is the burst for kubernetes client.
- // To use while talking with kubernetes apiserver
- //
- // Kubernetes client DefaultQPS is 5.
- // Kubernetes client DefaultBurst is 10.
- // ref. https://github.com/kubernetes/kubernetes/blob/4d0e86f0b8d1eae00a202009858c8739e4c9402e/staging/src/k8s.io/client-go/rest/config.go#L43-L46
- //
- // kube-apiserver default inflight requests limits are:
- // FLAG: --max-mutating-requests-inflight="200"
- // FLAG: --max-requests-inflight="400"
- // ref. https://github.com/kubernetes/kubernetes/blob/4d0e86f0b8d1eae00a202009858c8739e4c9402e/staging/src/k8s.io/apiserver/pkg/server/config.go#L300-L301
- //
- // This field is used for "eks/stresser" tester. Configure accordingly.
- // Rate limit is done via "k8s.io/client-go/util/flowcontrol.NewTokenBucketRateLimiter".
- ClientBurst int `json:"client-burst"`
- // ClientTimeout is the client timeout.
- ClientTimeout time.Duration `json:"client-timeout"`
- ClientTimeoutString string `json:"client-timeout-string,omitempty" read-only:"true"`
-
- //
- //
- //
- //
- //
- //
- //
- //
- //
- //
- //
- //
- //
- //
- //
- //
-
- // AddOnCNIVPC defines parameters for https://github.com/aws/amazon-vpc-cni-k8s.
- AddOnCNIVPC *AddOnCNIVPC `json:"add-on-cni-vpc"`
- // AddOnNodeGroups defines EKS "Node Group"
- // creation parameters.
- AddOnNodeGroups *AddOnNodeGroups `json:"add-on-node-groups,omitempty"`
- // AddOnManagedNodeGroups defines EKS "Managed Node Group"
- // creation parameters. If empty, it will use default values.
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html
- // ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html
- AddOnManagedNodeGroups *AddOnManagedNodeGroups `json:"add-on-managed-node-groups,omitempty"`
-
- // TotalNodes is the total number of nodes from all node groups.
- TotalNodes int32 `json:"total-nodes" read-only:"true"`
-
- // AddOnCWAgent defines parameters for EKS cluster
- // add-on Fluentd.
- AddOnCWAgent *AddOnCWAgent `json:"add-on-cw-agent,omitempty"`
- // AddOnFluentd defines parameters for EKS cluster
- // add-on Fluentd.
- AddOnFluentd *AddOnFluentd `json:"add-on-fluentd,omitempty"`
- // AddOnMetricsServer defines parameters for EKS cluster
- // add-on metrics server.
- AddOnMetricsServer *AddOnMetricsServer `json:"add-on-metrics-server,omitempty"`
-
- // AddOnConformance defines parameters for EKS cluster
- // add-on Conformance.
- AddOnConformance *AddOnConformance `json:"add-on-conformance,omitempty"`
-
- // AddOnAppMesh defines parameters for EKS cluster
- // add-on "EKS App Mesh Integration".
- AddOnAppMesh *AddOnAppMesh `json:"add-on-app-mesh,omitempty"`
- // AddOnCSIEBS defines parameters for EKS cluster
- // add-on AWS EBS CSI Driver.
- AddOnCSIEBS *AddOnCSIEBS `json:"add-on-csi-ebs,omitempty"`
- // AddOnKubernetesDashboard defines parameters for EKS cluster
- // add-on Dashboard.
- AddOnKubernetesDashboard *AddOnKubernetesDashboard `json:"add-on-kubernetes-dashboard,omitempty"`
- // AddOnPrometheusGrafana defines parameters for EKS cluster
- // add-on Prometheus/Grafana.
- AddOnPrometheusGrafana *AddOnPrometheusGrafana `json:"add-on-prometheus-grafana,omitempty"`
-
- // AddOnPHPApache defines parameters for EKS cluster
- // add-on PHP Apache.
- AddOnPHPApache *AddOnPHPApache `json:"add-on-php-apache,omitempty"`
- // AddOnNLBHelloWorld defines parameters for EKS cluster
- // add-on NLB hello-world service.
- AddOnNLBHelloWorld *AddOnNLBHelloWorld `json:"add-on-nlb-hello-world,omitempty"`
- // AddOnNLBGuestbook defines parameters for EKS cluster
- // add-on NLB guestbook service.
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/eks-guestbook.html
- AddOnNLBGuestbook *AddOnNLBGuestbook `json:"add-on-nlb-guestbook,omitempty"`
- // AddOnALB2048 defines parameters for EKS cluster
- // add-on ALB 2048 service.
- AddOnALB2048 *AddOnALB2048 `json:"add-on-alb-2048,omitempty"`
- // AddOnJobsPi defines parameters for EKS cluster
- // add-on Job with pi Perl command.
- AddOnJobsPi *AddOnJobsPi `json:"add-on-jobs-pi,omitempty"`
- // AddOnJobsEcho defines parameters for EKS cluster
- // add-on Job with echo.
- AddOnJobsEcho *AddOnJobsEcho `json:"add-on-jobs-echo,omitempty"`
- // AddOnCronJobs defines parameters for EKS cluster
- // add-on with CronJob.
- AddOnCronJobs *AddOnCronJobs `json:"add-on-cron-jobs,omitempty"`
-
- // AddOnCSRsLocal defines parameters for EKS cluster
- // add-on "CertificateSigningRequest" local.
- // It generates loads from the local host machine.
- AddOnCSRsLocal *AddOnCSRsLocal `json:"add-on-csrs-local,omitempty"`
- // AddOnCSRsRemote defines parameters for EKS cluster
- // add-on "CertificateSigningRequest" remote.
- // It generates loads from the remote workers (Pod) in the cluster.
- AddOnCSRsRemote *AddOnCSRsRemote `json:"add-on-csrs-remote,omitempty"`
-
- // AddOnConfigmapsLocal defines parameters for EKS cluster
- // add-on "ConfigMap" local.
- // It generates loads from the local host machine.
- AddOnConfigmapsLocal *AddOnConfigmapsLocal `json:"add-on-configmaps-local,omitempty"`
- // AddOnConfigmapsRemote defines parameters for EKS cluster
- // add-on "ConfigMap" remote.
- // It generates loads from the remote workers (Pod) in the cluster.
- AddOnConfigmapsRemote *AddOnConfigmapsRemote `json:"add-on-configmaps-remote,omitempty"`
-
- // AddOnSecretsLocal defines parameters for EKS cluster
- // add-on "Secrets" local.
- // It generates loads from the local host machine.
- AddOnSecretsLocal *AddOnSecretsLocal `json:"add-on-secrets-local,omitempty"`
- // AddOnSecretsRemote defines parameters for EKS cluster
- // add-on "Secrets" remote.
- // It generates loads from the remote workers (Pod) in the cluster.
- AddOnSecretsRemote *AddOnSecretsRemote `json:"add-on-secrets-remote,omitempty"`
-
- // AddOnFargate defines parameters for EKS cluster
- // add-on "EKS on Fargate".
- AddOnFargate *AddOnFargate `json:"add-on-fargate,omitempty"`
- // AddOnIRSA defines parameters for EKS cluster
- // add-on "IAM Roles for Service Accounts (IRSA)".
- AddOnIRSA *AddOnIRSA `json:"add-on-irsa,omitempty"`
- // AddOnIRSAFargate defines parameters for EKS cluster
- // add-on "IAM Roles for Service Accounts (IRSA)" with Fargate.
- AddOnIRSAFargate *AddOnIRSAFargate `json:"add-on-irsa-fargate,omitempty"`
- // AddOnWordpress defines parameters for EKS cluster
- // add-on WordPress.
- AddOnWordpress *AddOnWordpress `json:"add-on-wordpress,omitempty"`
- // AddOnJupyterHub defines parameters for EKS cluster
- // add-on JupyterHub.
- AddOnJupyterHub *AddOnJupyterHub `json:"add-on-jupyter-hub,omitempty"`
- // AddOnKubeflow defines parameters for EKS cluster
- // add-on Kubeflow.
- AddOnKubeflow *AddOnKubeflow `json:"add-on-kubeflow,omitempty"`
- // AddOnCUDAVectorAdd defines parameters for EKS cluster
- // add-on cuda-vector-add.
- AddOnCUDAVectorAdd *AddOnCUDAVectorAdd `json:"add-on-cuda-vector-add,omitempty"`
-
- // AddOnClusterLoaderLocal defines parameters for EKS cluster
- // add-on cluster loader local.
- // It generates loads from the local host machine.
- // ref. https://github.com/kubernetes/perf-tests/tree/master/clusterloader2
- AddOnClusterLoaderLocal *AddOnClusterLoaderLocal `json:"add-on-cluster-loader-local,omitempty"`
- // AddOnClusterLoaderRemote defines parameters for EKS cluster
- // add-on cluster loader remote.
- // It generates loads from the remote host machine.
- // ref. https://github.com/kubernetes/perf-tests/tree/master/clusterloader2
- AddOnClusterLoaderRemote *AddOnClusterLoaderRemote `json:"add-on-cluster-loader-remote,omitempty"`
-
- // AddOnStresserLocal defines parameters for EKS cluster
- // add-on cluster loader local.
- // It generates loads from the local host machine.
- // ref. https://github.com/kubernetes/perf-tests
- AddOnStresserLocal *AddOnStresserLocal `json:"add-on-stresser-local,omitempty"`
- // AddOnStresserRemote defines parameters for EKS cluster
- // add-on cluster loader remote.
- // It generates loads from the remote workers (Pod) in the cluster.
- // ref. https://github.com/kubernetes/perf-tests
- AddOnStresserRemote *AddOnStresserRemote `json:"add-on-stresser-remote,omitempty"`
- // AddOnStresserRemoteV2 defines parameters for EKS cluster
- // add-on cluster loader remote v2.
- // It generates loads from the remote workers (Pod) in the cluster.
- // ref. https://github.com/kubernetes/perf-tests
- AddOnStresserRemoteV2 *AddOnStresserRemoteV2 `json:"add-on-stresser-remote-v2,omitempty"`
-
- // AddOnClusterVersionUpgrade defines parameters
- // for EKS cluster version upgrade add-on.
- AddOnClusterVersionUpgrade *AddOnClusterVersionUpgrade `json:"add-on-cluster-version-upgrade,omitempty"`
-
- // AddOnAmiSoftLockupIssue454 defines parameters
- // for testing the AMI soft lockup issue.
- AddOnAmiSoftLockupIssue454 *AddOnAmiSoftLockupIssue454 `json:"add-on-ami-soft-lockup-issue-454,omitempty"`
-
- // Spec contains addons and other configuration
- // Note: New addons should be implemented inside spec
- Spec Spec `json:"spec,omitempty"`
-
- // Status represents the current status of AWS resources.
- // Status is read-only.
- // Status cannot be configured via environmental variables.
- Status *Status `json:"status,omitempty" read-only:"true"`
-}
-
-// Colorize prints colorized input, if color output is supported.
-func (c Config) Colorize(input string) string {
- colorize := colorstring.Colorize{
- Colors: colorstring.DefaultColors,
- Disable: !c.LogColor,
- Reset: true,
- }
- return colorize.Color(input)
-}
-
-type S3 struct {
- // BucketCreate is true to auto-create S3 bucket.
- BucketCreate bool `json:"bucket-create"`
- // BucketCreateKeep is true to not delete auto-created S3 bucket.
- // The created S3 bucket is kept.
- BucketCreateKeep bool `json:"bucket-create-keep"`
- // BucketName is the name of cluster S3.
- BucketName string `json:"bucket-name"`
- // BucketLifecycleExpirationDays is expiration in days for the lifecycle of the object.
- BucketLifecycleExpirationDays int64 `json:"bucket-lifecycle-expiration-days"`
-}
-
-func getDefaultS3() *S3 {
- return &S3{
- BucketName: "",
- BucketCreate: true,
- BucketCreateKeep: true,
- BucketLifecycleExpirationDays: 0,
- }
-}
-
-type Encryption struct {
- // CMKCreate is true to auto-create and delete KMS CMK
- // for encryption feature.
- CMKCreate bool `json:"cmk-create"`
- // CMKARN is the KMS CMK ARN for encryption feature.
- // If not empty, the cluster is created with encryption feature
- // enabled.
- CMKARN string `json:"cmk-arn"`
-}
-
-func getDefaultEncryption() *Encryption {
- return &Encryption{
- CMKCreate: true,
- }
-}
-
-type Role struct {
- // Name is the name of cluster role.
- Name string `json:"name"`
- // Create is true to auto-create and delete cluster role.
- Create bool `json:"create"`
- // ARN is the role ARN that EKS uses to create AWS resources for Kubernetes.
- // By default, it's empty which triggers tester to create one.
- ARN string `json:"arn"`
-
- // ServicePrincipals is the EKS Role Service Principals
- ServicePrincipals []string `json:"service-principals"`
- // ManagedPolicyARNs is EKS Role managed policy ARNs.
- ManagedPolicyARNs []string `json:"managed-policy-arns"`
-
- // PolicyName is the name of the policy.
- PolicyName string `json:"policy-name" read-only:"true"`
- // PolicyARN is the attached policy ARN.
- PolicyARN string `json:"policy-arn" read-only:"true"`
-
- // InstanceProfileName is the instance profile name for the node group.
- InstanceProfileName string `json:"instance-profile-name" read-only:"true"`
- // InstanceProfileARN is the instance profile ARN for the node group.
- InstanceProfileARN string `json:"instance-profile-arn" read-only:"true"`
-}
-
-func getDefaultRole() *Role {
- return &Role{
- Create: true,
- ServicePrincipals: []string{
- "ec2.amazonaws.com",
- "eks.amazonaws.com",
- "eks-fargate-pods.amazonaws.com",
- },
- // TODO: scope this down
- ManagedPolicyARNs: []string{
- "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy",
- "arn:aws:iam::aws:policy/AmazonSSMFullAccess",
- "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
- "arn:aws:iam::aws:policy/ElasticLoadBalancingFullAccess",
- "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy",
- "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", // required for MNG
- "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", // required for MNG
- "arn:aws:iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy",
- },
- }
-}
-
-type VPC struct {
- // Create is true to auto-create and delete VPC.
- Create bool `json:"create"`
- // ID is the VPC ID for cluster creation.
- // If not empty, VPC is reused and not deleted.
- // If empty, VPC is created anew and deleted on cluster deletion.
- ID string `json:"id"`
- SecurityGroupID string `json:"security-group-id" read-only:"true"`
-
- // CIDRs is the list of CIDR blocks with IP range (CIDR notation) for the primary VPC Block.
- // Must be a valid RFC 1918 CIDR range.
- CIDRs []string `json:"cidrs"`
-
- // PublicSubnetCIDRs is the CIDR blocks for public subnets.
- PublicSubnetCIDRs []string `json:"public-subnet-cidrs"`
- PublicSubnetIDs []string `json:"public-subnet-ids" read-only:"true"`
- InternetGatewayID string `json:"internet-gateway-id" read-only:"true"`
- PublicRouteTableID string `json:"public-route-table-id" read-only:"true"`
- PublicSubnetRouteTableAssociationIDs []string `json:"public-subnet-route-table-association-ids" read-only:"true"`
- EIPAllocationIDs []string `json:"eip-allocation-ids" read-only:"true"`
- NATGatewayIDs []string `json:"nat-gateway-ids" read-only:"true"`
-
- // PrivateSubnetCIDRs is the CIDR blocks for private subnets.
- PrivateSubnetCIDRs []string `json:"private-subnet-cidrs,omitempty"`
- PrivateSubnetIDs []string `json:"private-subnet-ids" read-only:"true"`
- PrivateRouteTableIDs []string `json:"private-route-table-ids" read-only:"true"`
- PrivateSubnetRouteTableAssociationIDs []string `json:"private-subnet-route-table-association-ids" read-only:"true"`
-
- // DHCPOptionsDomainName is used to complete unqualified DNS hostnames for VPC.
- // ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-dhcp-options.html
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html
- DHCPOptionsDomainName string `json:"dhcp-options-domain-name,omitempty"`
- // DHCPOptionsDomainNameServers is a list of strings.
- // The IPv4 addresses of up to four domain name servers, or AmazonProvidedDNS, for VPC.
- // ref. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-dhcp-options.html
- // ref. https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html
- DHCPOptionsDomainNameServers []string `json:"dhcp-options-domain-name-servers,omitempty"`
- DHCPOptionsID string `json:"dhcp-options-id,omitempty" read-only:"true"`
-
- // NodeGroupSecurityGroupName is the name of the node security group.
- NodeGroupSecurityGroupName string `json:"node-group-security-group-name" read-only:"true"`
- // NodeGroupSecurityGroupID is the security group ID for the node group.
- NodeGroupSecurityGroupID string `json:"node-group-security-group-id" read-only:"true"`
-}
-
-func getDefaultVPC() *VPC {
- return &VPC{
- Create: true,
- CIDRs: []string{
- "10.0.0.0/16",
- "10.1.0.0/16",
- "10.2.0.0/16",
- "10.3.0.0/16",
- },
- PublicSubnetCIDRs: []string{
- "10.0.0.0/16",
- "10.1.0.0/16",
- "10.2.0.0/16",
- },
- PrivateSubnetCIDRs: []string{
- "10.3.0.0/17",
- "10.3.128.0/17",
- },
- }
-}
-
-// Load loads configuration from YAML.
-// Useful when injecting shared configuration via ConfigMap.
-//
-// Example usage:
-//
-// import "github.com/aws/aws-k8s-tester/eksconfig"
-// cfg := eksconfig.Load("test.yaml")
-// err := cfg.ValidateAndSetDefaults()
-//
-// Do not set default values in this function.
-// "ValidateAndSetDefaults" must be called separately,
-// to prevent overwriting previous data when loaded from disks.
-func Load(p string) (cfg *Config, err error) {
- var d []byte
- d, err = ioutil.ReadFile(p)
- if err != nil {
- return nil, err
- }
- cfg = new(Config)
- if err = yaml.Unmarshal(d, cfg, yaml.DisallowUnknownFields); err != nil {
- return nil, err
- }
-
- cfg.mu = new(sync.RWMutex)
- if cfg.ConfigPath != p {
- cfg.ConfigPath = p
- }
- var ap string
- ap, err = filepath.Abs(p)
- if err != nil {
- return nil, err
- }
- cfg.ConfigPath = ap
- if serr := cfg.unsafeSync(); serr != nil {
- fmt.Fprintf(os.Stderr, "[WARN] failed to sync config files %v\n", serr)
- }
-
- return cfg, nil
-}
-
-// EvaluateCommandRefs updates "CommandAfterCreateCluster" and "CommandAfterCreateAddOns".
-// currently, only support "GetRef.Name" and "GetRef.ClusterARN"
-func (cfg *Config) EvaluateCommandRefs() error {
- cfg.mu.Lock()
- err := cfg.evaluateCommandRefs()
- cfg.mu.Unlock()
- return err
-}
-
-func (cfg *Config) evaluateCommandRefs() error {
- if cfg.CommandAfterCreateCluster != "" {
- ss := strings.Split(cfg.CommandAfterCreateCluster, " ")
- p, err := exec.LookPath(ss[0])
- if err != nil {
- return fmt.Errorf("%q does not exist (%v)", ss[0], err)
- }
- ss[0] = p
- cfg.CommandAfterCreateCluster = strings.Join(ss, " ")
- }
-
- if cfg.CommandAfterCreateAddOns != "" {
- ss := strings.Split(cfg.CommandAfterCreateAddOns, " ")
- p, err := exec.LookPath(ss[0])
- if err != nil {
- return fmt.Errorf("%q does not exist (%v)", ss[0], err)
- }
- ss[0] = p
- cfg.CommandAfterCreateAddOns = strings.Join(ss, " ")
- }
-
- if cfg.Name != "" && strings.Contains(cfg.CommandAfterCreateCluster, "GetRef.Name") {
- cfg.CommandAfterCreateCluster = strings.ReplaceAll(cfg.CommandAfterCreateCluster, "GetRef.Name", cfg.Name)
- }
- if cfg.Status != nil && cfg.Status.ClusterARN != "" && strings.Contains(cfg.CommandAfterCreateCluster, "GetRef.ClusterARN") {
- cfg.CommandAfterCreateCluster = strings.ReplaceAll(cfg.CommandAfterCreateCluster, "GetRef.ClusterARN", cfg.Status.ClusterARN)
- }
-
- if cfg.Name != "" && strings.Contains(cfg.CommandAfterCreateAddOns, "GetRef.Name") {
- cfg.CommandAfterCreateAddOns = strings.ReplaceAll(cfg.CommandAfterCreateAddOns, "GetRef.Name", cfg.Name)
- }
- if cfg.Status != nil && cfg.Status.ClusterARN != "" && strings.Contains(cfg.CommandAfterCreateAddOns, "GetRef.ClusterARN") {
- cfg.CommandAfterCreateAddOns = strings.ReplaceAll(cfg.CommandAfterCreateAddOns, "GetRef.ClusterARN", cfg.Status.ClusterARN)
- }
- if serr := cfg.unsafeSync(); serr != nil {
- fmt.Fprintf(os.Stderr, "[WARN] failed to sync config files %v\n", serr)
- }
-
- return nil
-}
-
-// Sync persists current configuration and states to disk.
-// Every call overwrites the previous contents if any.
-func (cfg *Config) Sync() (err error) {
- cfg.mu.Lock()
- defer cfg.mu.Unlock()
- return cfg.unsafeSync()
-}
-
-func (cfg *Config) unsafeSync() (err error) {
- var p string
- if cfg.ConfigPath != "" && !filepath.IsAbs(cfg.ConfigPath) {
- p, err = filepath.Abs(cfg.ConfigPath)
- if err != nil {
- return fmt.Errorf("failed to 'filepath.Abs(%s)' %v", cfg.ConfigPath, err)
- }
- cfg.ConfigPath = p
- }
- if cfg.KubeConfigPath != "" && !filepath.IsAbs(cfg.KubeConfigPath) {
- p, err = filepath.Abs(cfg.KubeConfigPath)
- if err != nil {
- return fmt.Errorf("failed to 'filepath.Abs(%s)' %v", cfg.KubeConfigPath, err)
- }
- cfg.KubeConfigPath = p
- }
-
- var d []byte
- d, err = yaml.Marshal(cfg)
- if err != nil {
- return fmt.Errorf("failed to 'yaml.Marshal' %v", err)
- }
- err = ioutil.WriteFile(cfg.ConfigPath, d, 0600)
- if err != nil {
- return fmt.Errorf("failed to write file %q (%v)", cfg.ConfigPath, err)
- }
-
- if cfg.RemoteAccessCommandsOutputPath != "" {
- err = ioutil.WriteFile(cfg.RemoteAccessCommandsOutputPath, []byte(cmdTop+cfg.unsafeSSHCommands()), 0600)
- if err != nil {
- return fmt.Errorf("failed to write RemoteAccessCommandsOutputPath %q (%v)", cfg.RemoteAccessCommandsOutputPath, err)
- }
- }
-
- if cfg.KubectlCommandsOutputPath != "" {
- err = ioutil.WriteFile(cfg.KubectlCommandsOutputPath, []byte(cmdTop+cfg.KubectlCommands()), 0600)
- if err != nil {
- return fmt.Errorf("failed to write KubectlCommandsOutputPath %q (%v)", cfg.KubectlCommandsOutputPath, err)
- }
- }
-
- return nil
-}
-
-const cmdTop = `#!/bin/bash
-set -e
-set -x
-
-`
-
-// KubectlCommand returns the kubectl command.
-func (cfg *Config) KubectlCommand() string {
- return fmt.Sprintf("%s --kubeconfig=%s", cfg.KubectlPath, cfg.KubeConfigPath)
-}
-
-// KubectlCommands returns the various kubectl commands.
-func (cfg *Config) KubectlCommands() (s string) {
- if cfg.KubeConfigPath == "" {
- return ""
- }
- tpl := template.Must(template.New("kubectlTmpl").Parse(kubectlTmpl))
- buf := bytes.NewBuffer(nil)
- if err := tpl.Execute(buf, struct {
- KubeConfigPath string
- KubectlCommand string
- KubernetesDashboardEnabled bool
- KubernetesDashboardURL string
- KubernetesDashboardAuthenticationToken string
- }{
- cfg.KubeConfigPath,
- cfg.KubectlCommand(),
- cfg.IsEnabledAddOnKubernetesDashboard(),
- cfg.getAddOnKubernetesDashboardURL(),
- cfg.getAddOnKubernetesDashboardAuthenticationToken(),
- }); err != nil {
- return ""
- }
- return buf.String()
-}
-
-const kubectlTmpl = `
-###########################
-# kubectl commands
-export KUBECONFIG={{ .KubeConfigPath }}
-export KUBECTL="{{ .KubectlCommand }}"
-
-{{ .KubectlCommand }} version
-{{ .KubectlCommand }} cluster-info
-{{ .KubectlCommand }} get cs
-{{ .KubectlCommand }} --namespace=kube-system get pods
-{{ .KubectlCommand }} --namespace=kube-system get ds
-{{ .KubectlCommand }} get pods
-{{ .KubectlCommand }} get csr -o=yaml
-{{ .KubectlCommand }} get nodes --show-labels -o=wide
-{{ .KubectlCommand }} get nodes -o=wide
-###########################
-{{ if .KubernetesDashboardEnabled }}
-###########################
-{{ .KubectlCommand }} proxy
-
-# Kubernetes Dashboard URL:
-{{ .KubernetesDashboardURL }}
-
-# Kubernetes Dashboard Authentication Token:
-{{ .KubernetesDashboardAuthenticationToken }}
-###########################
-{{ end }}
-`
-
-// SSHCommands returns the SSH commands.
-func (cfg *Config) SSHCommands() string {
- cfg.mu.RLock()
- defer cfg.mu.RUnlock()
- return cfg.unsafeSSHCommands()
-}
-
-func (cfg *Config) unsafeSSHCommands() (s string) {
- buf := bytes.NewBuffer(nil)
- buf.WriteByte('\n')
-
- if cfg.AddOnNodeGroups != nil && cfg.AddOnNodeGroups.Enable {
- for name, cur := range cfg.AddOnNodeGroups.ASGs {
- if len(cur.Instances) == 0 {
- buf.WriteString(fmt.Sprintf("no ASG instances found for node group %s\n", name))
- continue
- }
- buf.WriteString("ASG name from node group \"" + name + "\":\n")
- asg := &ec2config.ASG{
- Name: name,
- Instances: cur.Instances,
- }
- buf.WriteString(asg.SSHCommands(cfg.Region, cfg.RemoteAccessPrivateKeyPath, cfg.AddOnNodeGroups.ASGs[name].RemoteAccessUserName))
- buf.WriteString("\n\n")
- }
- }
-
- if cfg.AddOnManagedNodeGroups != nil && cfg.AddOnManagedNodeGroups.Enable {
- for name, cur := range cfg.AddOnManagedNodeGroups.MNGs {
- if len(cur.Instances) == 0 {
- buf.WriteString(fmt.Sprintf("no ASG instances found for managed node group %s\n", name))
- continue
- }
- buf.WriteString("ASG name from managed node group \"" + name + "\":\n")
- asg := &ec2config.ASG{
- Name: name,
- Instances: cur.Instances,
- }
- buf.WriteString(asg.SSHCommands(cfg.Region, cfg.RemoteAccessPrivateKeyPath, cfg.AddOnManagedNodeGroups.MNGs[name].RemoteAccessUserName))
- buf.WriteString("\n\n")
- }
- }
-
- return buf.String()
-}
-
-const (
- // DefaultClients is the default number of clients to create.
- DefaultClients = 2
- // DefaultClientQPS is the default client QPS.
- DefaultClientQPS float32 = 10
- // DefaultClientBurst is the default client burst.
- DefaultClientBurst = 20
- // DefaultClientTimeout is the default client timeout.
- DefaultClientTimeout = 15 * time.Second
-
- DefaultCommandAfterCreateClusterTimeout = 3 * time.Minute
- DefaultCommandAfterCreateAddOnsTimeout = 3 * time.Minute
-
- // DefaultNodeInstanceTypeCPU is the default EC2 instance type for CPU worker node.
- DefaultNodeInstanceTypeCPU = "c5.xlarge"
- // DefaultNodeInstanceTypeARMCPU is the default EC2 instance type for ARM CPU worker node.
- DefaultNodeInstanceTypeARMCPU = "c6g.xlarge"
- // DefaultNodeInstanceTypeGPU is the default EC2 instance type for GPU worker node.
- DefaultNodeInstanceTypeGPU = "p3.8xlarge"
-
- // DefaultNodeVolumeSize is the default EC2 instance volume size for a worker node.
- DefaultNodeVolumeSize = 40
-
- // DefaultNodeVolumeType is the default EC2 instance volume type for a worker node.
- DefaultNodeVolumeType = aws_ec2_v2_types.VolumeTypeGp3
-
- // NGsMaxLimit is the maximum number of "Node Group"s per a EKS cluster.
- NGsMaxLimit = 10
- // NGMaxLimit is the maximum number of nodes per a "Node Group".
- NGMaxLimit = 5000
-
- // MNGsMaxLimit is the maximum number of "Managed Node Group"s per a EKS cluster.
- MNGsMaxLimit = 10
- // MNGMaxLimit is the maximum number of nodes per a "Managed Node Group".
- MNGMaxLimit = 100
-)
-
-// NewDefault returns a default configuration.
-// - empty string creates a non-nil object for pointer-type field
-// - omitting an entire field returns nil value
-// - make sure to check both
-func NewDefault() *Config {
- name := fmt.Sprintf("eks-%s-%s", getTS()[:10], randutil.String(12))
- if v := os.Getenv(AWS_K8S_TESTER_EKS_PREFIX + "NAME"); v != "" {
- name = v
- }
-
- cfg := Config{
- mu: new(sync.RWMutex),
-
- Name: name,
- Partition: "aws",
- Region: "us-west-2",
-
- // to be auto-generated
- ConfigPath: "",
- KubectlCommandsOutputPath: "",
- KubeConfigPath: "",
- AWSCLIPath: "",
-
- LogColor: true,
- LogColorOverride: "",
-
- LogLevel: logutil.DefaultLogLevel,
- // default, stderr, stdout, or file name
- // log file named with cluster name will be added automatically
- LogOutputs: []string{"stderr"},
-
- // https://github.com/kubernetes/kubernetes/tags
- // https://kubernetes.io/docs/tasks/tools/install-kubectl/
- // https://docs.aws.amazon.com/eks/latest/userguide/install-kubectl.html
- KubectlPath: "/tmp/kubectl-test-v1.20.7",
- KubectlDownloadURL: "https://storage.googleapis.com/kubernetes-release/release/v1.20.7/bin/linux/amd64/kubectl",
-
- OnFailureDelete: true,
- OnFailureDeleteWaitSeconds: 120,
-
- CWNamespace: "aws-k8s-tester-eks",
-
- SkipDeleteClusterAndNodes: false,
-
- S3: getDefaultS3(),
- Encryption: getDefaultEncryption(),
- Role: getDefaultRole(),
- VPC: getDefaultVPC(),
-
- SigningName: "eks",
- Version: "1.27",
-
- RemoteAccessKeyCreate: true,
- // keep in-sync with the default value in https://pkg.go.dev/k8s.io/kubernetes/test/e2e/framework#GetSigner
- // RemoteAccessPrivateKeyPath: filepath.Join(homedir.HomeDir(), ".ssh", "kube_aws_rsa"),
- RemoteAccessPrivateKeyPath: filepath.Join(os.TempDir(), randutil.String(15)+".insecure.key"),
-
- // Kubernetes client DefaultQPS is 5.
- // Kubernetes client DefaultBurst is 10.
- // ref. https://github.com/kubernetes/kubernetes/blob/4d0e86f0b8d1eae00a202009858c8739e4c9402e/staging/src/k8s.io/client-go/rest/config.go#L43-L46
- //
- // kube-apiserver default inflight requests limits are:
- // FLAG: --max-mutating-requests-inflight="200"
- // FLAG: --max-requests-inflight="400"
- // ref. https://github.com/kubernetes/kubernetes/blob/4d0e86f0b8d1eae00a202009858c8739e4c9402e/staging/src/k8s.io/apiserver/pkg/server/config.go#L300-L301
- //
- Clients: DefaultClients,
- ClientQPS: DefaultClientQPS,
- ClientBurst: DefaultClientBurst,
-
- AddOnCNIVPC: getDefaultAddOnCNIVPC(),
- AddOnNodeGroups: getDefaultAddOnNodeGroups(name),
- AddOnManagedNodeGroups: getDefaultAddOnManagedNodeGroups(name),
-
- AddOnCWAgent: getDefaultAddOnCWAgent(),
- AddOnFluentd: getDefaultAddOnFluentd(),
- AddOnMetricsServer: getDefaultAddOnMetricsServer(),
-
- AddOnConformance: getDefaultAddOnConformance(),
-
- AddOnAppMesh: getDefaultAddOnAppMesh(),
- AddOnCSIEBS: getDefaultAddOnCSIEBS(),
- AddOnKubernetesDashboard: getDefaultAddOnKubernetesDashboard(),
- AddOnPrometheusGrafana: getDefaultAddOnPrometheusGrafana(),
- AddOnPHPApache: getDefaultAddOnPHPApache(),
- AddOnNLBHelloWorld: getDefaultAddOnNLBHelloWorld(),
- AddOnNLBGuestbook: getDefaultAddOnNLBGuestbook(),
- AddOnALB2048: getDefaultAddOnALB2048(),
- AddOnJobsPi: getDefaultAddOnJobsPi(),
- AddOnJobsEcho: getDefaultAddOnJobsEcho(),
- AddOnCronJobs: getDefaultAddOnCronJobs(),
- AddOnCSRsLocal: getDefaultAddOnCSRsLocal(),
- AddOnCSRsRemote: getDefaultAddOnCSRsRemote(),
- AddOnConfigmapsLocal: getDefaultAddOnConfigmapsLocal(),
- AddOnConfigmapsRemote: getDefaultAddOnConfigmapsRemote(),
- AddOnSecretsLocal: getDefaultAddOnSecretsLocal(),
- AddOnSecretsRemote: getDefaultAddOnSecretsRemote(),
- AddOnFargate: getDefaultAddOnFargate(),
- AddOnIRSA: getDefaultAddOnIRSA(),
- AddOnIRSAFargate: getDefaultAddOnIRSAFargate(),
- AddOnWordpress: getDefaultAddOnWordpress(),
- AddOnJupyterHub: getDefaultAddOnJupyterHub(),
- AddOnKubeflow: getDefaultAddOnKubeflow(),
- AddOnCUDAVectorAdd: getDefaultAddOnCUDAVectorAdd(),
- AddOnClusterLoaderLocal: getDefaultAddOnClusterLoaderLocal(),
- AddOnClusterLoaderRemote: getDefaultAddOnClusterLoaderRemote(),
- AddOnStresserLocal: getDefaultAddOnStresserLocal(),
- AddOnStresserRemote: getDefaultAddOnStresserRemote(),
- AddOnStresserRemoteV2: getDefaultAddOnStresserRemoteV2(),
- AddOnClusterVersionUpgrade: getDefaultAddOnClusterVersionUpgrade(),
- AddOnAmiSoftLockupIssue454: getDefaultAddOnAmiSoftLockupIssue454(),
-
- // read-only
- Status: &Status{
- Up: false,
- PrivateDNSToNodeInfo: make(map[string]NodeInfo),
- DeletedResources: make(map[string]string),
- },
- }
-
- // https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-welcome.html
- // pip3 install awscli --no-cache-dir --upgrade
- var err error
- cfg.AWSCLIPath, err = exec.LookPath("aws")
- if err != nil {
- fmt.Fprintf(os.Stderr, "[WARN] aws CLI is not installed (%v); required for 'aws eks update-kubeconfig'\n", err)
- }
-
- // TODO: check for ARM/x86
- if runtime.GOOS == "darwin" {
- cfg.KubectlDownloadURL = strings.Replace(cfg.KubectlDownloadURL, "linux", "darwin", -1)
- }
-
- return &cfg
-}
-
-// ValidateAndSetDefaults returns an error for invalid configurations.
-// And updates empty fields with default values.
-// At the end, it writes populated YAML to aws-k8s-tester config path.
-// "read-only" fields cannot be set, causing errors.
-func (cfg *Config) ValidateAndSetDefaults() error {
- if cfg.mu == nil {
- cfg.mu = new(sync.RWMutex)
- }
- cfg.mu.Lock()
- defer func() {
- if serr := cfg.unsafeSync(); serr != nil {
- fmt.Fprintf(os.Stderr, "[WARN] failed to sync config files %v\n", serr)
- }
- cfg.mu.Unlock()
- }()
-
- // generically defaults and validates addons that are members of cfg.Spec
- spec := reflect.ValueOf(cfg.Spec)
- for i := 0; i < spec.NumField(); i++ {
- // skip if the field does not implement Addon or is Nil
- if addon, ok := spec.Field(i).Interface().(Addon); ok && !reflect.ValueOf(addon).IsNil() {
- addon.Default(cfg)
- if err := addon.Validate(cfg); err != nil {
- return fmt.Errorf("failed to validate %s, %v", reflect.ValueOf(addon).Type(), err)
- }
- }
- }
-
- if err := cfg.validateConfig(); err != nil {
- return fmt.Errorf("validateConfig failed [%v]", err)
- }
- if err := cfg.validateAddOnNodeGroups(); err != nil {
- return fmt.Errorf("validateAddOnNodeGroups failed [%v]", err)
- }
- if err := cfg.validateAddOnManagedNodeGroups(); err != nil {
- return fmt.Errorf("validateAddOnManagedNodeGroups failed [%v]", err)
- }
-
- if err := cfg.validateAddOnCNIVPC(); err != nil {
- return fmt.Errorf("validateAddOnCNIVPC failed [%v]", err)
- }
-
- total := int32(0)
- if cfg.IsEnabledAddOnNodeGroups() {
- for _, cur := range cfg.AddOnNodeGroups.ASGs {
- total += cur.ASGDesiredCapacity
- }
- }
- if cfg.IsEnabledAddOnManagedNodeGroups() {
- for _, cur := range cfg.AddOnManagedNodeGroups.MNGs {
- total += int32(cur.ASGDesiredCapacity)
- }
- }
- cfg.TotalNodes = total
-
- if err := cfg.validateAddOnCWAgent(); err != nil {
- return fmt.Errorf("validateAddOnCWAgent failed [%v]", err)
- }
- if err := cfg.validateAddOnFluentd(); err != nil {
- return fmt.Errorf("validateAddOnFluentd failed [%v]", err)
- }
- if err := cfg.validateAddOnMetricsServer(); err != nil {
- return fmt.Errorf("validateAddOnMetricsServer failed [%v]", err)
- }
-
- if err := cfg.validateAddOnConformance(); err != nil {
- return fmt.Errorf("validateAddOnConformance failed [%v]", err)
- }
-
- if err := cfg.validateAddOnAppMesh(); err != nil {
- return fmt.Errorf("validateAddOnAppMesh failed [%v]", err)
- }
- if err := cfg.validateAddOnCSIEBS(); err != nil {
- return fmt.Errorf("validateAddOnCSIEBS failed [%v]", err)
- }
-
- if err := cfg.validateAddOnKubernetesDashboard(); err != nil {
- return fmt.Errorf("validateAddOnKubernetesDashboard failed [%v]", err)
- }
- if err := cfg.validateAddOnPrometheusGrafana(); err != nil {
- return fmt.Errorf("validateAddOnPrometheusGrafana failed [%v]", err)
- }
- if err := cfg.validateAddOnPHPApache(); err != nil {
- return fmt.Errorf("validateAddOnPHPApache failed [%v]", err)
- }
- if err := cfg.validateAddOnNLBHelloWorld(); err != nil {
- return fmt.Errorf("validateAddOnNLBHelloWorld failed [%v]", err)
- }
- if err := cfg.validateAddOnNLBGuestbook(); err != nil {
- return fmt.Errorf("validateAddOnNLBGuestbook failed [%v]", err)
- }
- if err := cfg.validateAddOnALB2048(); err != nil {
- return fmt.Errorf("validateAddOnALB2048 failed [%v]", err)
- }
- if err := cfg.validateAddOnJobsPi(); err != nil {
- return fmt.Errorf("validateAddOnJobsPi failed [%v]", err)
- }
- if err := cfg.validateAddOnJobsEcho(); err != nil {
- return fmt.Errorf("validateAddOnJobsEcho failed [%v]", err)
- }
- if err := cfg.validateAddOnCronJobs(); err != nil {
- return fmt.Errorf("validateAddOnCronJobs failed [%v]", err)
- }
-
- if err := cfg.validateAddOnCSRsLocal(); err != nil {
- return fmt.Errorf("validateAddOnCSRsLocal failed [%v]", err)
- }
- if err := cfg.validateAddOnCSRsRemote(); err != nil {
- return fmt.Errorf("validateAddOnCSRsRemote failed [%v]", err)
- }
-
- if err := cfg.validateAddOnConfigmapsLocal(); err != nil {
- return fmt.Errorf("validateAddOnConfigmapsLocal failed [%v]", err)
- }
- if err := cfg.validateAddOnConfigmapsRemote(); err != nil {
- return fmt.Errorf("validateAddOnConfigmapsRemote failed [%v]", err)
- }
-
- if err := cfg.validateAddOnSecretsLocal(); err != nil {
- return fmt.Errorf("validateAddOnSecretsLocal failed [%v]", err)
- }
- if err := cfg.validateAddOnSecretsRemote(); err != nil {
- return fmt.Errorf("validateAddOnSecretsRemote failed [%v]", err)
- }
-
- if err := cfg.validateAddOnFargate(); err != nil {
- return fmt.Errorf("validateAddOnFargate failed [%v]", err)
- }
- if err := cfg.validateAddOnIRSA(); err != nil {
- return fmt.Errorf("validateAddOnIRSA failed [%v]", err)
- }
- if err := cfg.validateAddOnIRSAFargate(); err != nil {
- return fmt.Errorf("validateIRSAAddOnFargate failed [%v]", err)
- }
- if err := cfg.validateAddOnWordpress(); err != nil {
- return fmt.Errorf("validateAddOnWordpress failed [%v]", err)
- }
- if err := cfg.validateAddOnJupyterHub(); err != nil {
- return fmt.Errorf("validateAddOnJupyterHub failed [%v]", err)
- }
- if err := cfg.validateAddOnKubeflow(); err != nil {
- return fmt.Errorf("validateAddOnKubeflow failed [%v]", err)
- }
- if err := cfg.validateAddOnCUDAVectorAdd(); err != nil {
- return fmt.Errorf("validateAddOnCUDAVectorAdd failed [%v]", err)
- }
-
- if err := cfg.validateAddOnClusterLoaderLocal(); err != nil {
- return fmt.Errorf("validateAddOnClusterLoaderLocal failed [%v]", err)
- }
- if err := cfg.validateAddOnClusterLoaderRemote(); err != nil {
- return fmt.Errorf("validateAddOnClusterLoaderRemote failed [%v]", err)
- }
-
- if err := cfg.validateAddOnStresserLocal(); err != nil {
- return fmt.Errorf("validateAddOnStresserLocal failed [%v]", err)
- }
- if err := cfg.validateAddOnStresserRemote(); err != nil {
- return fmt.Errorf("validateAddOnStresserRemote failed [%v]", err)
- }
- if err := cfg.validateAddOnStresserRemoteV2(); err != nil {
- return fmt.Errorf("validateAddOnStresserRemoteV2 failed [%v]", err)
- }
-
- if err := cfg.validateAddOnClusterVersionUpgrade(); err != nil {
- return fmt.Errorf("validateAddOnClusterVersionUpgrade failed [%v]", err)
- }
-
- if err := cfg.validateAddOnAmiSoftLockupIssue454(); err != nil {
- return fmt.Errorf("validateAddOnClusterVersionUpgrade failed [%v]", err)
- }
-
- return nil
-}
-
-// endpoints package no longer exists in the AWS SDK for Go V2
-// "github.com/aws/aws-sdk-go/aws/endpoints" is deprecated...
-// the check will be done in "eks" with AWS API call
-// ref. https://aws.github.io/aws-sdk-go-v2/docs/migrating/
-func (cfg *Config) validateConfig() error {
- if len(cfg.Name) == 0 {
- return errors.New("name is empty")
- }
- if cfg.Name != strings.ToLower(cfg.Name) {
- return fmt.Errorf("name %q must be in lower-case", cfg.Name)
- }
-
- if cfg.LogColorOverride == "" {
- _, cerr := terminal.IsColor()
- if cfg.LogColor && cerr != nil {
- cfg.LogColor = false
- fmt.Fprintf(os.Stderr, "[WARN] LogColor is set to 'false' due to error %+v", cerr)
- }
- } else {
- // non-empty override, don't even run "terminal.IsColor"
- ov, perr := strconv.ParseBool(cfg.LogColorOverride)
- if perr != nil {
- return fmt.Errorf("failed to parse LogColorOverride %q (%v)", cfg.LogColorOverride, perr)
- }
- cfg.LogColor = ov
- fmt.Fprintf(os.Stderr, "[WARN] LogColor is overwritten with %q", cfg.LogColorOverride)
- }
-
- if len(cfg.LogOutputs) == 0 {
- return errors.New("LogOutputs is not empty")
- }
-
- if cfg.Clients == 0 {
- cfg.Clients = DefaultClients
- }
- if cfg.ClientQPS == 0 {
- cfg.ClientQPS = DefaultClientQPS
- }
- if cfg.ClientBurst == 0 {
- cfg.ClientBurst = DefaultClientBurst
- }
- if cfg.ClientTimeout == time.Duration(0) {
- cfg.ClientTimeout = DefaultClientTimeout
- }
- cfg.ClientTimeoutString = cfg.ClientTimeout.String()
-
- if cfg.ConfigPath == "" {
- rootDir, err := os.Getwd()
- if err != nil {
- rootDir = filepath.Join(os.TempDir(), cfg.Name)
- if err := os.MkdirAll(rootDir, 0700); err != nil {
- return err
- }
- }
- cfg.ConfigPath = filepath.Join(rootDir, cfg.Name+".yaml")
- var p string
- p, err = filepath.Abs(cfg.ConfigPath)
- if err != nil {
- panic(err)
- }
- cfg.ConfigPath = p
- }
- if err := os.MkdirAll(filepath.Dir(cfg.ConfigPath), 0700); err != nil {
- return err
- }
- if err := fileutil.IsDirWriteable(filepath.Dir(cfg.ConfigPath)); err != nil {
- return err
- }
-
- if len(cfg.LogOutputs) == 1 && (cfg.LogOutputs[0] == "stderr" || cfg.LogOutputs[0] == "stdout") {
- cfg.LogOutputs = append(cfg.LogOutputs, strings.ReplaceAll(cfg.ConfigPath, ".yaml", "")+".log")
- }
- logFilePath := ""
- for _, fpath := range cfg.LogOutputs {
- if filepath.Ext(fpath) == ".log" {
- logFilePath = fpath
- break
- }
- }
- if logFilePath == "" {
- return fmt.Errorf("*.log file not found in %q", cfg.LogOutputs)
- }
-
- if cfg.Version == "" {
- return errors.New("empty Parameters.Version")
- }
- var err error
- cfg.VersionValue, err = strconv.ParseFloat(cfg.Version, 64)
- if err != nil {
- return fmt.Errorf("cannot parse Parameters.Version %q (%v)", cfg.Version, err)
- }
-
- if len(cfg.Role.ServicePrincipals) == 0 {
- return errors.New("empty Role.ServicePrincipals")
- }
- if len(cfg.Role.ManagedPolicyARNs) == 0 {
- return errors.New("empty Role.ManagedPolicyARNs")
- }
- // e.g.,
- // "api error LimitExceeded: Cannot exceed quota for PoliciesPerRole: 10"
- if len(cfg.Role.ManagedPolicyARNs) > 9 {
- return fmt.Errorf("too many ManagedPolicyARNs %q", cfg.Role.ManagedPolicyARNs)
- }
-
- found := false
- for _, v := range cfg.Role.ServicePrincipals {
- if v == "eks.amazonaws.com" {
- found = true
- break
- }
- }
- if !found {
- return fmt.Errorf("Role.ServicePrincipals missing 'eks.amazonaws.com' (%q)", cfg.Role.ServicePrincipals)
- }
- found = false
- for _, v := range cfg.Role.ManagedPolicyARNs {
- if v == "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy" {
- found = true
- break
- }
- }
- if !found {
- return fmt.Errorf("Role.ManagedPolicyARNs missing 'arn:aws:iam::aws:policy/AmazonEKSClusterPolicy' (%q)", cfg.Role.ManagedPolicyARNs)
- }
-
- switch cfg.Role.Create {
- case true: // need create one, or already created
- if cfg.Role.Name == "" {
- cfg.Role.Name = cfg.Name + "-role"
- }
- case false: // use existing one
- if cfg.Role.ARN == "" {
- return fmt.Errorf("Role.Create false; expect non-empty RoleARN but got %q", cfg.Role.ARN)
- }
- if cfg.Role.Name == "" {
- cfg.Role.Name = getNameFromARN(cfg.Role.ARN)
- }
- }
- if cfg.Role.PolicyName == "" {
- cfg.Role.PolicyName = cfg.Name + "-policy"
- }
-
- switch cfg.VPC.Create {
- case true: // need create one, or already created
- // just ignore...
- // could be populated from previous run
- // do not error, so long as VPCCreate false, VPC won't be deleted
- case false: // use existing one
- if cfg.VPC.ID == "" {
- return fmt.Errorf("RoleCreate false; expect non-empty VPCID but got %q", cfg.VPC.ID)
- }
- }
-
- if cfg.VPC.NodeGroupSecurityGroupName == "" {
- cfg.VPC.NodeGroupSecurityGroupName = cfg.Name + "-node-group-security-group"
- }
- if len(cfg.VPC.PublicSubnetCIDRs) < 2 {
- return fmt.Errorf("unexpected number of VPC.PublicSubnetCIDRs %v (expected at least 2)", cfg.VPC.PublicSubnetCIDRs)
- }
-
- switch cfg.Encryption.CMKCreate {
- case true: // need create one, or already created
- // just ignore...
- // could be populated from previous run
- // do not error, so long as EncryptionCMKCreate false, CMK won't be deleted
- case false: // use existing one
- }
-
- switch cfg.RemoteAccessKeyCreate {
- case true: // need create one, or already created
- if cfg.RemoteAccessKeyName == "" {
- cfg.RemoteAccessKeyName = cfg.Name + "-remote-access-key"
- }
- if cfg.RemoteAccessPrivateKeyPath == "" {
- cfg.RemoteAccessPrivateKeyPath = filepath.Join(os.TempDir(), randutil.String(10)+".insecure.key")
- }
-
- case false: // use existing one
- if cfg.RemoteAccessKeyName == "" {
- return fmt.Errorf("RemoteAccessKeyCreate false; expect non-empty RemoteAccessKeyName but got %q", cfg.RemoteAccessKeyName)
- }
- if cfg.RemoteAccessPrivateKeyPath == "" {
- return fmt.Errorf("RemoteAccessKeyCreate false; expect non-empty RemoteAccessPrivateKeyPath but got %q", cfg.RemoteAccessPrivateKeyPath)
- }
- if !fileutil.Exist(cfg.RemoteAccessPrivateKeyPath) {
- return fmt.Errorf("RemoteAccessPrivateKeyPath %q does not exist", cfg.RemoteAccessPrivateKeyPath)
- }
- }
- keyDir := filepath.Dir(cfg.RemoteAccessPrivateKeyPath)
- if err := fileutil.IsDirWriteable(keyDir); err != nil {
- return err
- }
-
- if cfg.KubectlCommandsOutputPath == "" {
- cfg.KubectlCommandsOutputPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + ".kubectl.sh"
- }
- if filepath.Ext(cfg.KubectlCommandsOutputPath) != ".sh" {
- cfg.KubectlCommandsOutputPath = cfg.KubectlCommandsOutputPath + ".sh"
- }
- if err := fileutil.IsDirWriteable(filepath.Dir(cfg.KubectlCommandsOutputPath)); err != nil {
- return err
- }
- if cfg.RemoteAccessCommandsOutputPath == "" {
- cfg.RemoteAccessCommandsOutputPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + ".ssh.sh"
- }
- if filepath.Ext(cfg.RemoteAccessCommandsOutputPath) != ".sh" {
- cfg.RemoteAccessCommandsOutputPath = cfg.RemoteAccessCommandsOutputPath + ".sh"
- }
- if err := fileutil.IsDirWriteable(filepath.Dir(cfg.RemoteAccessCommandsOutputPath)); err != nil {
- return err
- }
-
- if cfg.CommandAfterCreateClusterOutputPath == "" {
- cfg.CommandAfterCreateClusterOutputPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + ".after-create-cluster.out.log"
- }
- if filepath.Ext(cfg.CommandAfterCreateClusterOutputPath) != ".log" {
- cfg.CommandAfterCreateClusterOutputPath = cfg.CommandAfterCreateClusterOutputPath + ".log"
- }
- if cfg.CommandAfterCreateClusterTimeout == time.Duration(0) {
- cfg.CommandAfterCreateClusterTimeout = DefaultCommandAfterCreateClusterTimeout
- }
- if err := fileutil.IsDirWriteable(filepath.Dir(cfg.CommandAfterCreateClusterOutputPath)); err != nil {
- return err
- }
- cfg.CommandAfterCreateClusterTimeoutString = cfg.CommandAfterCreateClusterTimeout.String()
-
- if cfg.CommandAfterCreateAddOnsOutputPath == "" {
- cfg.CommandAfterCreateAddOnsOutputPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + ".after-create-add-ons.out.log"
- }
- if filepath.Ext(cfg.CommandAfterCreateAddOnsOutputPath) != ".log" {
- cfg.CommandAfterCreateAddOnsOutputPath = cfg.CommandAfterCreateAddOnsOutputPath + ".log"
- }
- if err := fileutil.IsDirWriteable(filepath.Dir(cfg.CommandAfterCreateAddOnsOutputPath)); err != nil {
- return err
- }
- if cfg.CommandAfterCreateAddOnsTimeout == time.Duration(0) {
- cfg.CommandAfterCreateAddOnsTimeout = DefaultCommandAfterCreateAddOnsTimeout
- }
- cfg.CommandAfterCreateAddOnsTimeoutString = cfg.CommandAfterCreateAddOnsTimeout.String()
-
- if cfg.KubeConfigPath == "" {
- cfg.KubeConfigPath = strings.ReplaceAll(cfg.ConfigPath, ".yaml", "") + ".kubeconfig.yaml"
- }
- if err := fileutil.IsDirWriteable(filepath.Dir(cfg.KubeConfigPath)); err != nil {
- return err
- }
-
- if cfg.KubectlPath == "" && cfg.KubectlDownloadURL == "" {
- return errors.New("empty KubectlPath and KubectlDownloadURL")
- }
- if !strings.ContainsAny(cfg.KubectlDownloadURL, runtime.GOOS) {
- return fmt.Errorf("kubectl-download-url %q build OS mismatch, expected %q", cfg.KubectlDownloadURL, runtime.GOOS)
- }
-
- if err := cfg.evaluateCommandRefs(); err != nil {
- return err
- }
-
- switch cfg.S3.BucketCreate {
- case true: // need create one, or already created
- if cfg.S3.BucketName == "" {
- cfg.S3.BucketName = cfg.Name + "-s3-bucket"
- }
- if cfg.S3.BucketLifecycleExpirationDays > 0 && cfg.S3.BucketLifecycleExpirationDays < 3 {
- cfg.S3.BucketLifecycleExpirationDays = 3
- }
- case false: // use existing one
- if cfg.S3.BucketName == "" {
- return errors.New("empty S3BucketName")
- }
- }
-
- if cfg.CWNamespace == "" {
- cfg.CWNamespace = "aws-k8s-tester-eks"
- }
-
- if cfg.Status == nil {
- cfg.Status = &Status{
- Up: false,
- PrivateDNSToNodeInfo: make(map[string]NodeInfo),
- }
- }
- return nil
-}
-
-// only letters and numbers
-var regex = regexp.MustCompile("[^a-zA-Z0-9]+")
-
-// get "role-eks" from "arn:aws:iam::123:role/role-eks"
-func getNameFromARN(arn string) string {
- if ss := strings.Split(arn, "/"); len(ss) > 0 {
- arn = ss[len(ss)-1]
- }
- return arn
-}
-
-func getTS() string {
- now := time.Now()
- return fmt.Sprintf(
- "%04d%02d%02d%02d%02d",
- now.Year(),
- int(now.Month()),
- now.Day(),
- now.Hour(),
- now.Second(),
- )
-}
diff --git a/eksconfig/default.yaml b/eksconfig/default.yaml
deleted file mode 100644
index 389f58065..000000000
--- a/eksconfig/default.yaml
+++ /dev/null
@@ -1,148 +0,0 @@
-add-on-cni-vpc: null
-aws-cli-path: /home/leegyuho/.local/bin/aws
-client-burst: 20
-client-qps: 10
-client-timeout: 15000000000
-client-timeout-string: 15s
-clients: 2
-command-after-create-add-ons: ""
-command-after-create-add-ons-output-path: /home/leegyuho/go/src/github.com/aws/aws-k8s-tester/eksconfig/default.after-create-add-ons.out.log
-command-after-create-add-ons-timeout: 180000000000
-command-after-create-add-ons-timeout-string: 3m0s
-command-after-create-cluster: ""
-command-after-create-cluster-output-path: /home/leegyuho/go/src/github.com/aws/aws-k8s-tester/eksconfig/default.after-create-cluster.out.log
-command-after-create-cluster-timeout: 180000000000
-command-after-create-cluster-timeout-string: 3m0s
-config-path: /home/leegyuho/go/src/github.com/aws/aws-k8s-tester/eksconfig/default.yaml
-cw-namespace: aws-k8s-tester-eks
-encryption:
- cmk-arn: ""
- cmk-create: true
-kube-apiserver-max-requests-inflight: ""
-kubeconfig-path: /home/leegyuho/go/src/github.com/aws/aws-k8s-tester/eksconfig/default.kubeconfig.yaml
-kubectl-commands-output-path: /home/leegyuho/go/src/github.com/aws/aws-k8s-tester/eksconfig/default.kubectl.sh
-kubectl-download-url: https://storage.googleapis.com/kubernetes-release/release/v1.20.7/bin/linux/amd64/kubectl
-kubectl-path: /tmp/kubectl-test-v1.20.7
-log-color: true
-log-color-override: ""
-log-level: info
-log-outputs:
-- stderr
-- /home/leegyuho/go/src/github.com/aws/aws-k8s-tester/eksconfig/default.log
-metadata:
- creationTimestamp: null
-name: eks-2021091520-tropical1f5d
-on-failure-delete: true
-on-failure-delete-wait-seconds: 120
-partition: aws
-region: us-west-2
-remote-access-commands-output-path: /home/leegyuho/go/src/github.com/aws/aws-k8s-tester/eksconfig/default.ssh.sh
-remote-access-key-create: true
-remote-access-key-name: eks-2021091520-tropical1f5d-remote-access-key
-remote-access-private-key-path: /tmp/atlasubjsfmv0hb.insecure.key
-request-header-key: ""
-request-header-value: ""
-resolver-url: ""
-role:
- arn: ""
- create: true
- instance-profile-arn: ""
- instance-profile-name: ""
- managed-policy-arns:
- - arn:aws:iam::aws:policy/AmazonEKSClusterPolicy
- - arn:aws:iam::aws:policy/AmazonSSMFullAccess
- - arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy
- - arn:aws:iam::aws:policy/ElasticLoadBalancingFullAccess
- - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy
- - arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy
- - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
- - arn:aws:iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy
- name: eks-2021091520-tropical1f5d-role
- policy-arn: ""
- policy-name: eks-2021091520-tropical1f5d-policy
- service-principals:
- - ec2.amazonaws.com
- - eks.amazonaws.com
- - eks-fargate-pods.amazonaws.com
-s3:
- bucket-create: true
- bucket-create-keep: true
- bucket-lifecycle-expiration-days: 0
- bucket-name: eks-2021091520-tropical1f5d-s3-bucket
-signing-name: eks
-skip-delete-cluster-and-nodes: false
-spec: {}
-status:
- aws-account-id: ""
- aws-credential-path: ""
- aws-iam-role-arn: ""
- aws-user-id: ""
- cluster-api-server-endpoint: ""
- cluster-arn: ""
- cluster-ca: ""
- cluster-ca-decoded: ""
- cluster-oidc-issuer-arn: ""
- cluster-oidc-issuer-ca-thumbprint: ""
- cluster-oidc-issuer-host-path: ""
- cluster-oidc-issuer-url: ""
- cluster-status: null
- cluster-status-current: ""
- deleted-resources: {}
- private-dns-to-node-info: {}
- server-version-info:
- buildDate: ""
- compiler: ""
- gitCommit: ""
- gitTreeState: ""
- gitVersion: ""
- goVersion: ""
- major: ""
- minor: ""
- platform: ""
- version-value: 0
- time-frame-create:
- complete-utc: "0001-01-01T00:00:00Z"
- complete-utc-rfc3339-nano: ""
- start-utc: "0001-01-01T00:00:00Z"
- start-utc-rfc3339-nano: ""
- took: 0
- took-string: ""
- time-frame-delete:
- complete-utc: "0001-01-01T00:00:00Z"
- complete-utc-rfc3339-nano: ""
- start-utc: "0001-01-01T00:00:00Z"
- start-utc-rfc3339-nano: ""
- took: 0
- took-string: ""
- up: false
-tags: null
-total-nodes: 0
-version: "1.27"
-version-value: 1.27
-vpc:
- cidrs:
- - 10.0.0.0/16
- - 10.1.0.0/16
- - 10.2.0.0/16
- - 10.3.0.0/16
- create: true
- eip-allocation-ids: null
- id: ""
- internet-gateway-id: ""
- nat-gateway-ids: null
- node-group-security-group-id: ""
- node-group-security-group-name: eks-2021091520-tropical1f5d-node-group-security-group
- private-route-table-ids: null
- private-subnet-cidrs:
- - 10.3.0.0/17
- - 10.3.128.0/17
- private-subnet-ids: null
- private-subnet-route-table-association-ids: null
- public-route-table-id: ""
- public-subnet-cidrs:
- - 10.0.0.0/16
- - 10.1.0.0/16
- - 10.2.0.0/16
- public-subnet-ids: null
- public-subnet-route-table-association-ids: null
- security-group-id: ""
diff --git a/eksconfig/env.go b/eksconfig/env.go
deleted file mode 100644
index 602068bb0..000000000
--- a/eksconfig/env.go
+++ /dev/null
@@ -1,773 +0,0 @@
-package eksconfig
-
-import (
- "encoding/json"
- "fmt"
- "os"
- "reflect"
- "strconv"
- "strings"
- "time"
-
- "sigs.k8s.io/yaml"
-)
-
-const (
- // AWS_K8S_TESTER_EKS_PREFIX is the environment variable prefix used for "eksconfig".
- AWS_K8S_TESTER_EKS_PREFIX = "AWS_K8S_TESTER_EKS_"
- AWS_K8S_TESTER_EKS_S3_PREFIX = AWS_K8S_TESTER_EKS_PREFIX + "S3_"
- AWS_K8S_TESTER_EKS_ENCRYPTION_PREFIX = AWS_K8S_TESTER_EKS_PREFIX + "ENCRYPTION_"
- AWS_K8S_TESTER_EKS_ROLE_PREFIX = AWS_K8S_TESTER_EKS_PREFIX + "ROLE_"
- AWS_K8S_TESTER_EKS_VPC_PREFIX = AWS_K8S_TESTER_EKS_PREFIX + "VPC_"
-)
-
-// UpdateFromEnvs updates fields from environmental variables.
-// Empty values are ignored and do not overwrite fields with empty values.
-// WARNING: The environmental variable value always overwrites current field
-// values if there's a conflict.
-func (cfg *Config) UpdateFromEnvs() (err error) {
- cfg.mu.Lock()
- defer func() {
- cfg.unsafeSync()
- cfg.mu.Unlock()
- }()
-
- if env, ok := os.LookupEnv("AWS_K8S_TESTER_EKS_CONFIG"); ok {
- mu := cfg.mu
- if err = yaml.Unmarshal([]byte(env), cfg, yaml.DisallowUnknownFields); err != nil {
- return err
- }
- cfg.mu = mu
- }
-
- var vv interface{}
- vv, err = parseEnvs(AWS_K8S_TESTER_EKS_PREFIX, cfg)
- if err != nil {
- return err
- }
- if av, ok := vv.(*Config); ok {
- a1, b1 := av.Role, av.VPC
- cfg = av
- a2, b2 := cfg.Role, cfg.VPC
- if !reflect.DeepEqual(a1, a2) {
- return fmt.Errorf("Role overwritten [before %+v, after %+v]", a1, a2)
- }
- if !reflect.DeepEqual(b1, b2) {
- return fmt.Errorf("VPC overwritten [before %+v, after %+v]", b1, b2)
- }
- } else {
- return fmt.Errorf("expected *Config, got %T", vv)
- }
-
- if cfg.S3 == nil {
- cfg.S3 = &S3{}
- }
- vv, err = parseEnvs(AWS_K8S_TESTER_EKS_S3_PREFIX, cfg.S3)
- if err != nil {
- return err
- }
- if av, ok := vv.(*S3); ok {
- cfg.S3 = av
- } else {
- return fmt.Errorf("expected *S3, got %T", vv)
- }
-
- if cfg.Encryption == nil {
- cfg.Encryption = &Encryption{}
- }
- vv, err = parseEnvs(AWS_K8S_TESTER_EKS_ENCRYPTION_PREFIX, cfg.Encryption)
- if err != nil {
- return err
- }
- if av, ok := vv.(*Encryption); ok {
- cfg.Encryption = av
- } else {
- return fmt.Errorf("expected *Encryption, got %T", vv)
- }
-
- if cfg.Role == nil {
- cfg.Role = &Role{}
- }
- vv, err = parseEnvs(AWS_K8S_TESTER_EKS_ROLE_PREFIX, cfg.Role)
- if err != nil {
- return err
- }
- if av, ok := vv.(*Role); ok {
- cfg.Role = av
- } else {
- return fmt.Errorf("expected *Role, got %T", vv)
- }
-
- if cfg.VPC == nil {
- cfg.VPC = &VPC{}
- }
- vv, err = parseEnvs(AWS_K8S_TESTER_EKS_VPC_PREFIX, cfg.VPC)
- if err != nil {
- return err
- }
- if av, ok := vv.(*VPC); ok {
- cfg.VPC = av
- } else {
- return fmt.Errorf("expected *VPC, got %T", vv)
- }
-
- if cfg.AddOnCNIVPC == nil {
- cfg.AddOnCNIVPC = &AddOnCNIVPC{}
- }
- vv, err = parseEnvs(AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_PREFIX, cfg.AddOnCNIVPC)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnCNIVPC); ok {
- cfg.AddOnCNIVPC = av
- } else {
- return fmt.Errorf("expected *AddOnCNIVPC, got %T", vv)
- }
-
- if cfg.AddOnNodeGroups == nil {
- cfg.AddOnNodeGroups = &AddOnNodeGroups{}
- }
- vv, err = parseEnvs(AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_PREFIX, cfg.AddOnNodeGroups)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnNodeGroups); ok {
- cfg.AddOnNodeGroups = av
- } else {
- return fmt.Errorf("expected *AddOnNodeGroups, got %T", vv)
- }
-
- if cfg.AddOnNodeGroups.Role == nil {
- cfg.AddOnNodeGroups.Role = &Role{}
- }
- vv, err = parseEnvs(AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_PREFIX+"ROLE_", cfg.AddOnNodeGroups.Role)
- if err != nil {
- return err
- }
- if av, ok := vv.(*Role); ok {
- cfg.AddOnNodeGroups.Role = av
- } else {
- return fmt.Errorf("expected *AddOnNodeGroups.Role, got %T", vv)
- }
-
- if cfg.AddOnManagedNodeGroups == nil {
- cfg.AddOnManagedNodeGroups = &AddOnManagedNodeGroups{}
- }
- vv, err = parseEnvs(AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_PREFIX, cfg.AddOnManagedNodeGroups)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnManagedNodeGroups); ok {
- cfg.AddOnManagedNodeGroups = av
- } else {
- return fmt.Errorf("expected *AddOnManagedNodeGroups, got %T", vv)
- }
-
- if cfg.AddOnManagedNodeGroups.Role == nil {
- cfg.AddOnManagedNodeGroups.Role = &Role{}
- }
- vv, err = parseEnvs(AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_PREFIX+"ROLE_", cfg.AddOnManagedNodeGroups.Role)
- if err != nil {
- return err
- }
- if av, ok := vv.(*Role); ok {
- cfg.AddOnManagedNodeGroups.Role = av
- } else {
- return fmt.Errorf("expected *AddOnManagedNodeGroups.Role, got %T", vv)
- }
-
- if cfg.AddOnCWAgent == nil {
- cfg.AddOnCWAgent = &AddOnCWAgent{}
- }
- vv, err = parseEnvs(AWS_K8S_TESTER_EKS_ADD_ON_CW_AGENT_PREFIX, cfg.AddOnCWAgent)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnCWAgent); ok {
- cfg.AddOnCWAgent = av
- } else {
- return fmt.Errorf("expected *AddOnCWAgent, got %T", vv)
- }
-
- if cfg.AddOnFluentd == nil {
- cfg.AddOnFluentd = &AddOnFluentd{}
- }
- vv, err = parseEnvs(AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_PREFIX, cfg.AddOnFluentd)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnFluentd); ok {
- cfg.AddOnFluentd = av
- } else {
- return fmt.Errorf("expected *AddOnFluentd, got %T", vv)
- }
-
- if cfg.AddOnMetricsServer == nil {
- cfg.AddOnMetricsServer = &AddOnMetricsServer{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnMetricsServer, cfg.AddOnMetricsServer)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnMetricsServer); ok {
- cfg.AddOnMetricsServer = av
- } else {
- return fmt.Errorf("expected *AddOnMetricsServer, got %T", vv)
- }
-
- if cfg.AddOnConformance == nil {
- cfg.AddOnConformance = &AddOnConformance{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnConformance, cfg.AddOnConformance)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnConformance); ok {
- cfg.AddOnConformance = av
- } else {
- return fmt.Errorf("expected *AddOnConformance, got %T", vv)
- }
-
- if cfg.AddOnAppMesh == nil {
- cfg.AddOnAppMesh = &AddOnAppMesh{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnAppMesh, cfg.AddOnAppMesh)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnAppMesh); ok {
- cfg.AddOnAppMesh = av
- } else {
- return fmt.Errorf("expected *AddOnAppMesh, got %T", vv)
- }
-
- if cfg.AddOnCSIEBS == nil {
- cfg.AddOnCSIEBS = &AddOnCSIEBS{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnCSIEBS, cfg.AddOnCSIEBS)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnCSIEBS); ok {
- cfg.AddOnCSIEBS = av
- } else {
- return fmt.Errorf("expected *AddOnCSIEBS, got %T", vv)
- }
-
- if cfg.AddOnKubernetesDashboard == nil {
- cfg.AddOnKubernetesDashboard = &AddOnKubernetesDashboard{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnKubernetesDashboard, cfg.AddOnKubernetesDashboard)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnKubernetesDashboard); ok {
- cfg.AddOnKubernetesDashboard = av
- } else {
- return fmt.Errorf("expected *AddOnKubernetesDashboard, got %T", vv)
- }
-
- if cfg.AddOnPrometheusGrafana == nil {
- cfg.AddOnPrometheusGrafana = &AddOnPrometheusGrafana{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnPrometheusGrafana, cfg.AddOnPrometheusGrafana)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnPrometheusGrafana); ok {
- cfg.AddOnPrometheusGrafana = av
- } else {
- return fmt.Errorf("expected *AddOnPrometheusGrafana, got %T", vv)
- }
-
- if cfg.AddOnPHPApache == nil {
- cfg.AddOnPHPApache = &AddOnPHPApache{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnPHPApache, cfg.AddOnPHPApache)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnPHPApache); ok {
- cfg.AddOnPHPApache = av
- } else {
- return fmt.Errorf("expected *AddOnPHPApache, got %T", vv)
- }
-
- if cfg.AddOnNLBHelloWorld == nil {
- cfg.AddOnNLBHelloWorld = &AddOnNLBHelloWorld{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnNLBHelloWorld, cfg.AddOnNLBHelloWorld)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnNLBHelloWorld); ok {
- cfg.AddOnNLBHelloWorld = av
- } else {
- return fmt.Errorf("expected *AddOnNLBHelloWorld, got %T", vv)
- }
-
- if cfg.AddOnNLBGuestbook == nil {
- cfg.AddOnNLBGuestbook = &AddOnNLBGuestbook{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnNLBGuestbook, cfg.AddOnNLBGuestbook)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnNLBGuestbook); ok {
- cfg.AddOnNLBGuestbook = av
- } else {
- return fmt.Errorf("expected *AddOnNLBGuestbook, got %T", vv)
- }
-
- if cfg.AddOnALB2048 == nil {
- cfg.AddOnALB2048 = &AddOnALB2048{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnALB2048, cfg.AddOnALB2048)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnALB2048); ok {
- cfg.AddOnALB2048 = av
- } else {
- return fmt.Errorf("expected *AddOnALB2048, got %T", vv)
- }
-
- if cfg.AddOnJobsPi == nil {
- cfg.AddOnJobsPi = &AddOnJobsPi{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnJobsPi, cfg.AddOnJobsPi)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnJobsPi); ok {
- cfg.AddOnJobsPi = av
- } else {
- return fmt.Errorf("expected *AddOnJobsPi, got %T", vv)
- }
-
- if cfg.AddOnJobsEcho == nil {
- cfg.AddOnJobsEcho = &AddOnJobsEcho{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnJobsEcho, cfg.AddOnJobsEcho)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnJobsEcho); ok {
- cfg.AddOnJobsEcho = av
- } else {
- return fmt.Errorf("expected *AddOnJobsEcho, got %T", vv)
- }
-
- if cfg.AddOnCronJobs == nil {
- cfg.AddOnCronJobs = &AddOnCronJobs{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnCronJobs, cfg.AddOnCronJobs)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnCronJobs); ok {
- cfg.AddOnCronJobs = av
- } else {
- return fmt.Errorf("expected *AddOnCronJobs, got %T", vv)
- }
-
- if cfg.AddOnCSRsLocal == nil {
- cfg.AddOnCSRsLocal = &AddOnCSRsLocal{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnCSRsLocal, cfg.AddOnCSRsLocal)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnCSRsLocal); ok {
- cfg.AddOnCSRsLocal = av
- } else {
- return fmt.Errorf("expected *AddOnCSRsLocal, got %T", vv)
- }
-
- if cfg.AddOnCSRsRemote == nil {
- cfg.AddOnCSRsRemote = &AddOnCSRsRemote{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnCSRsRemote, cfg.AddOnCSRsRemote)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnCSRsRemote); ok {
- cfg.AddOnCSRsRemote = av
- } else {
- return fmt.Errorf("expected *AddOnCSRsRemote, got %T", vv)
- }
-
- if cfg.AddOnConfigmapsLocal == nil {
- cfg.AddOnConfigmapsLocal = &AddOnConfigmapsLocal{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnConfigmapsLocal, cfg.AddOnConfigmapsLocal)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnConfigmapsLocal); ok {
- cfg.AddOnConfigmapsLocal = av
- } else {
- return fmt.Errorf("expected *AddOnConfigmapsLocal, got %T", vv)
- }
-
- if cfg.AddOnConfigmapsRemote == nil {
- cfg.AddOnConfigmapsRemote = &AddOnConfigmapsRemote{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnConfigmapsRemote, cfg.AddOnConfigmapsRemote)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnConfigmapsRemote); ok {
- cfg.AddOnConfigmapsRemote = av
- } else {
- return fmt.Errorf("expected *AddOnConfigmapsRemote, got %T", vv)
- }
-
- if cfg.AddOnSecretsLocal == nil {
- cfg.AddOnSecretsLocal = &AddOnSecretsLocal{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnSecretsLocal, cfg.AddOnSecretsLocal)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnSecretsLocal); ok {
- cfg.AddOnSecretsLocal = av
- } else {
- return fmt.Errorf("expected *AddOnSecretsLocal, got %T", vv)
- }
-
- if cfg.AddOnSecretsRemote == nil {
- cfg.AddOnSecretsRemote = &AddOnSecretsRemote{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnSecretsRemote, cfg.AddOnSecretsRemote)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnSecretsRemote); ok {
- cfg.AddOnSecretsRemote = av
- } else {
- return fmt.Errorf("expected *AddOnSecretsRemote, got %T", vv)
- }
-
- if cfg.AddOnFargate == nil {
- cfg.AddOnFargate = &AddOnFargate{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnFargate, cfg.AddOnFargate)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnFargate); ok {
- cfg.AddOnFargate = av
- } else {
- return fmt.Errorf("expected *AddOnFargate, got %T", vv)
- }
-
- if cfg.AddOnIRSA == nil {
- cfg.AddOnIRSA = &AddOnIRSA{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnIRSA, cfg.AddOnIRSA)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnIRSA); ok {
- cfg.AddOnIRSA = av
- } else {
- return fmt.Errorf("expected *AddOnIRSA, got %T", vv)
- }
-
- if cfg.AddOnIRSAFargate == nil {
- cfg.AddOnIRSAFargate = &AddOnIRSAFargate{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnIRSAFargate, cfg.AddOnIRSAFargate)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnIRSAFargate); ok {
- cfg.AddOnIRSAFargate = av
- } else {
- return fmt.Errorf("expected *AddOnIRSAFargate, got %T", vv)
- }
-
- if cfg.AddOnWordpress == nil {
- cfg.AddOnWordpress = &AddOnWordpress{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnWordpress, cfg.AddOnWordpress)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnWordpress); ok {
- cfg.AddOnWordpress = av
- } else {
- return fmt.Errorf("expected *AddOnWordpress, got %T", vv)
- }
-
- if cfg.AddOnJupyterHub == nil {
- cfg.AddOnJupyterHub = &AddOnJupyterHub{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnJupyterHub, cfg.AddOnJupyterHub)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnJupyterHub); ok {
- cfg.AddOnJupyterHub = av
- } else {
- return fmt.Errorf("expected *AddOnJupyterHub, got %T", vv)
- }
-
- if cfg.AddOnKubeflow == nil {
- cfg.AddOnKubeflow = &AddOnKubeflow{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnKubeflow, cfg.AddOnKubeflow)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnKubeflow); ok {
- cfg.AddOnKubeflow = av
- } else {
- return fmt.Errorf("expected *AddOnKubeflow, got %T", vv)
- }
-
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnCUDAVectorAdd, cfg.AddOnCUDAVectorAdd)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnCUDAVectorAdd); ok {
- cfg.AddOnCUDAVectorAdd = av
- } else {
- return fmt.Errorf("expected *AddOnCUDAVectorAdd, got %T", vv)
- }
-
- if cfg.AddOnClusterLoaderLocal == nil {
- cfg.AddOnClusterLoaderLocal = &AddOnClusterLoaderLocal{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnClusterLoaderLocal, cfg.AddOnClusterLoaderLocal)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnClusterLoaderLocal); ok {
- cfg.AddOnClusterLoaderLocal = av
- } else {
- return fmt.Errorf("expected *AddOnClusterLoaderLocal, got %T", vv)
- }
-
- if cfg.AddOnClusterLoaderRemote == nil {
- cfg.AddOnClusterLoaderRemote = &AddOnClusterLoaderRemote{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnClusterLoaderRemote, cfg.AddOnClusterLoaderRemote)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnClusterLoaderRemote); ok {
- cfg.AddOnClusterLoaderRemote = av
- } else {
- return fmt.Errorf("expected *AddOnClusterLoaderRemote, got %T", vv)
- }
-
- if cfg.AddOnStresserLocal == nil {
- cfg.AddOnStresserLocal = &AddOnStresserLocal{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnStresserLocal, cfg.AddOnStresserLocal)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnStresserLocal); ok {
- cfg.AddOnStresserLocal = av
- } else {
- return fmt.Errorf("expected *AddOnStresserLocal, got %T", vv)
- }
-
- if cfg.AddOnStresserRemote == nil {
- cfg.AddOnStresserRemote = &AddOnStresserRemote{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnStresserRemote, cfg.AddOnStresserRemote)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnStresserRemote); ok {
- cfg.AddOnStresserRemote = av
- } else {
- return fmt.Errorf("expected *AddOnStresserRemote, got %T", vv)
- }
-
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnStresserRemoteV2, cfg.AddOnStresserRemoteV2)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnStresserRemoteV2); ok {
- cfg.AddOnStresserRemoteV2 = av
- } else {
- return fmt.Errorf("expected *AddOnStresserRemoteV2, got %T", vv)
- }
-
- if cfg.AddOnClusterVersionUpgrade == nil {
- cfg.AddOnClusterVersionUpgrade = &AddOnClusterVersionUpgrade{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnClusterVersionUpgrade, cfg.AddOnClusterVersionUpgrade)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnClusterVersionUpgrade); ok {
- cfg.AddOnClusterVersionUpgrade = av
- } else {
- return fmt.Errorf("expected *AddOnClusterVersionUpgrade, got %T", vv)
- }
-
- if cfg.AddOnAmiSoftLockupIssue454 == nil {
- cfg.AddOnAmiSoftLockupIssue454 = &AddOnAmiSoftLockupIssue454{}
- }
- vv, err = parseEnvs(EnvironmentVariablePrefixAddOnAmiSoftLockupIssue454, cfg.AddOnAmiSoftLockupIssue454)
- if err != nil {
- return err
- }
- if av, ok := vv.(*AddOnAmiSoftLockupIssue454); ok {
- cfg.AddOnAmiSoftLockupIssue454 = av
- } else {
- return fmt.Errorf("expected *AddOnAmiSoftLockupIssue454, got %T", vv)
- }
-
- return nil
-}
-
-func parseEnvs(pfx string, addOn interface{}) (interface{}, error) {
- tp, vv := reflect.TypeOf(addOn).Elem(), reflect.ValueOf(addOn).Elem()
- for i := 0; i < tp.NumField(); i++ {
- jv := tp.Field(i).Tag.Get("json")
- if jv == "" {
- continue
- }
- jv = strings.Replace(jv, ",omitempty", "", -1)
- jv = strings.ToUpper(strings.Replace(jv, "-", "_", -1))
- env := pfx + jv
- sv := os.Getenv(env)
- if sv == "" {
- continue
- }
- if tp.Field(i).Tag.Get("read-only") == "true" { // error when read-only field is set for update
- return nil, fmt.Errorf("'%s=%s' is 'read-only' field; should not be set", env, sv)
- }
- fieldName := tp.Field(i).Name
-
- switch vv.Field(i).Type().Kind() {
- case reflect.String:
- vv.Field(i).SetString(sv)
-
- case reflect.Bool:
- bb, err := strconv.ParseBool(sv)
- if err != nil {
- return nil, fmt.Errorf("failed to parse %q (field name %q, environmental variable key %q, error %v)", sv, fieldName, env, err)
- }
- vv.Field(i).SetBool(bb)
-
- case reflect.Int, reflect.Int32, reflect.Int64:
- if vv.Field(i).Type().Name() == "Duration" {
- iv, err := time.ParseDuration(sv)
- if err != nil {
- return nil, fmt.Errorf("failed to parse %q (field name %q, environmental variable key %q, error %v)", sv, fieldName, env, err)
- }
- vv.Field(i).SetInt(int64(iv))
- } else {
- iv, err := strconv.ParseInt(sv, 10, 64)
- if err != nil {
- return nil, fmt.Errorf("failed to parse %q (field name %q, environmental variable key %q, error %v)", sv, fieldName, env, err)
- }
- vv.Field(i).SetInt(iv)
- }
-
- case reflect.Uint, reflect.Uint32, reflect.Uint64:
- iv, err := strconv.ParseUint(sv, 10, 64)
- if err != nil {
- return nil, fmt.Errorf("failed to parse %q (field name %q, environmental variable key %q, error %v)", sv, fieldName, env, err)
- }
- vv.Field(i).SetUint(iv)
-
- case reflect.Float32, reflect.Float64:
- fv, err := strconv.ParseFloat(sv, 64)
- if err != nil {
- return nil, fmt.Errorf("failed to parse %q (field name %q, environmental variable key %q, error %v)", sv, fieldName, env, err)
- }
- vv.Field(i).SetFloat(fv)
-
- case reflect.Slice: // only supports "[]string" for now
- ss := strings.Split(sv, ",")
- if len(ss) < 1 {
- continue
- }
- slice := reflect.MakeSlice(reflect.TypeOf([]string{}), len(ss), len(ss))
- for j := range ss {
- slice.Index(j).SetString(ss[j])
- }
- vv.Field(i).Set(slice)
-
- case reflect.Map:
- switch fieldName {
- case "Tags",
- "NodeSelector",
- "DeploymentNodeSelector",
- "DeploymentNodeSelector2048":
- vv.Field(i).Set(reflect.ValueOf(make(map[string]string)))
- mm := make(map[string]string)
- if err := json.Unmarshal([]byte(sv), &mm); err != nil {
- return nil, fmt.Errorf("failed to parse %q (field name %q, environmental variable key %q, error %v)", sv, fieldName, env, err)
- }
- vv.Field(i).Set(reflect.ValueOf(mm))
-
- case "ASGs":
- asgs := make(map[string]ASG)
- if err := json.Unmarshal([]byte(sv), &asgs); err != nil {
- return nil, fmt.Errorf("failed to parse %q (field name %q, environmental variable key %q, error %v)", sv, fieldName, env, err)
- }
- for k, v := range asgs {
- tp2, vv2 := reflect.TypeOf(&v).Elem(), reflect.ValueOf(&v).Elem()
- for j := 0; j < tp2.NumField(); j++ {
- jv := tp2.Field(j).Tag.Get("json")
- if jv == "" {
- continue
- }
- if tp2.Field(j).Tag.Get("read-only") != "true" {
- continue
- }
- if vv2.Field(j).Type().Kind() != reflect.String {
- continue
- }
- // skip updating read-only field
- vv2.Field(j).SetString("")
- }
- asgs[k] = v
- }
- vv.Field(i).Set(reflect.ValueOf(asgs))
-
- case "MNGs":
- mngs := make(map[string]MNG)
- if err := json.Unmarshal([]byte(sv), &mngs); err != nil {
- return nil, fmt.Errorf("failed to parse %q (field name %q, environmental variable key %q, error %v)", sv, fieldName, env, err)
- }
- for k, v := range mngs {
- tp2, vv2 := reflect.TypeOf(&v).Elem(), reflect.ValueOf(&v).Elem()
- for j := 0; j < tp2.NumField(); j++ {
- jv := tp2.Field(j).Tag.Get("json")
- if jv == "" {
- continue
- }
- if tp2.Field(j).Tag.Get("read-only") != "true" {
- continue
- }
- if vv2.Field(j).Type().Kind() != reflect.String {
- continue
- }
- // skip updating read-only field
- vv2.Field(j).SetString("")
- }
- mngs[k] = v
- }
- vv.Field(i).Set(reflect.ValueOf(mngs))
-
- default:
- return nil, fmt.Errorf("field %q not supported for reflect.Map", fieldName)
- }
-
- default:
- return nil, fmt.Errorf("%q (type %v) is not supported as an env", env, vv.Field(i).Type())
- }
- }
- return addOn, nil
-}
diff --git a/eksconfig/env_test.go b/eksconfig/env_test.go
deleted file mode 100644
index 8f06ed77e..000000000
--- a/eksconfig/env_test.go
+++ /dev/null
@@ -1,2086 +0,0 @@
-package eksconfig
-
-import (
- "fmt"
- "io/ioutil"
- "os"
- "reflect"
- "strings"
- "testing"
- "time"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-k8s-tester/pkg/randutil"
- "github.com/aws/aws-sdk-go/service/eks"
- "github.com/google/go-cmp/cmp"
- "github.com/stretchr/testify/assert"
-)
-
-func TestEnv(t *testing.T) {
- cfg := NewDefault()
- defer func() {
- os.RemoveAll(cfg.ConfigPath)
- os.RemoveAll(cfg.KubectlCommandsOutputPath)
- os.RemoveAll(cfg.RemoteAccessCommandsOutputPath)
- }()
-
- os.Setenv("AWS_K8S_TESTER_EKS_CONFIG", `
-spec:
- clusterAutoscaler:
- cloudProvider: kubemark
- image: 767520670908.dkr.ecr.us-west-2.amazonaws.com/cluster-autoscaler-kubemark:custom-build-20200727
- overprovisioning:
- kubemarkEnabled: true
-`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_CONFIG")
-
- os.Setenv("AWS_K8S_TESTER_EKS_LOG_COLOR", "false")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_LOG_COLOR")
- os.Setenv("AWS_K8S_TESTER_EKS_LOG_COLOR_OVERRIDE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_LOG_COLOR_OVERRIDE")
- os.Setenv("AWS_K8S_TESTER_EKS_KUBECTL_COMMANDS_OUTPUT_PATH", "hello-kubectl")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_KUBECTL_COMMANDS_OUTPUT_PATH")
- os.Setenv("AWS_K8S_TESTER_EKS_REMOTE_ACCESS_COMMANDS_OUTPUT_PATH", "hello-ssh")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_REMOTE_ACCESS_COMMANDS_OUTPUT_PATH")
- os.Setenv("AWS_K8S_TESTER_EKS_REGION", "us-east-1")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_REGION")
- os.Setenv("AWS_K8S_TESTER_EKS_LOG_LEVEL", "debug")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_LOG_LEVEL")
- os.Setenv("AWS_K8S_TESTER_EKS_KUBECTL_DOWNLOAD_URL", "https://amazon-eks.s3-us-west-2.amazonaws.com/1.11.5/2018-12-06/bin/linux/amd64/kubectl")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_KUBECTL_DOWNLOAD_URL")
- os.Setenv("AWS_K8S_TESTER_EKS_KUBECTL_PATH", "/tmp/aws-k8s-tester-test/kubectl")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_KUBECTL_PATH")
- os.Setenv("AWS_K8S_TESTER_EKS_KUBECONFIG_PATH", "/tmp/aws-k8s-tester/kubeconfig2")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_KUBECONFIG_PATH")
- os.Setenv("AWS_K8S_TESTER_EKS_ON_FAILURE_DELETE", "false")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ON_FAILURE_DELETE")
- os.Setenv("AWS_K8S_TESTER_EKS_ON_FAILURE_DELETE_WAIT_SECONDS", "780")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ON_FAILURE_DELETE_WAIT_SECONDS")
- os.Setenv("AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_CLUSTER", "echo hello1")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_CLUSTER")
- os.Setenv("AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_CLUSTER_TIMEOUT", "7m")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_CLUSTER_TIMEOUT")
- os.Setenv("AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_ADD_ONS", "echo hello2")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_ADD_ONS")
- os.Setenv("AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_ADD_ONS_TIMEOUT", "17m")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_COMMAND_AFTER_CREATE_ADD_ONS_TIMEOUT")
- os.Setenv("AWS_K8S_TESTER_EKS_S3_BUCKET_CREATE", `true`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_S3_BUCKET_CREATE")
- os.Setenv("AWS_K8S_TESTER_EKS_S3_BUCKET_CREATE_KEEP", `true`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_S3_BUCKET_CREATE_KEEP")
- os.Setenv("AWS_K8S_TESTER_EKS_S3_BUCKET_NAME", `my-bucket`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_S3_BUCKET_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_S3_BUCKET_LIFECYCLE_EXPIRATION_DAYS", `10`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_S3_BUCKET_LIFECYCLE_EXPIRATION_DAYS")
- os.Setenv("AWS_K8S_TESTER_EKS_CLIENTS", `333`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_CLIENTS")
- os.Setenv("AWS_K8S_TESTER_EKS_CLIENT_TIMEOUT", `10m`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_CLIENT_TIMEOUT")
- os.Setenv("AWS_K8S_TESTER_EKS_CLIENT_QPS", `99555.77`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_CLIENT_QPS")
- os.Setenv("AWS_K8S_TESTER_EKS_CLIENT_BURST", `177`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_CLIENT_BURST")
-
- os.Setenv("AWS_K8S_TESTER_EKS_SKIP_DELETE_CLUSTER_AND_NODES", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_SKIP_DELETE_CLUSTER_AND_NODES")
- os.Setenv("AWS_K8S_TESTER_EKS_VPC_CREATE", "false")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_VPC_CREATE")
- os.Setenv("AWS_K8S_TESTER_EKS_VPC_ID", "vpc-id")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_VPC_ID")
- os.Setenv("AWS_K8S_TESTER_EKS_VPC_CIDRS", "public-cidr1,public-cidr2,public-cidr3")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_VPC_CIDRS")
- os.Setenv("AWS_K8S_TESTER_EKS_VPC_PUBLIC_SUBNET_CIDRS", "public-cidr1,public-cidr2,public-cidr3")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_VPC_PUBLIC_SUBNET_CIDRS")
- os.Setenv("AWS_K8S_TESTER_EKS_VPC_PRIVATE_SUBNET_CIDRS", "private-cidr1,private-cidr2")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_VPC_PRIVATE_SUBNET_CIDRS")
- os.Setenv("AWS_K8S_TESTER_EKS_VPC_DHCP_OPTIONS_DOMAIN_NAME", `hello.com`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_VPC_DHCP_OPTIONS_DOMAIN_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_VPC_DHCP_OPTIONS_DOMAIN_NAME_SERVERS", `1.2.3.0,4.5.6.7`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_VPC_DHCP_OPTIONS_DOMAIN_NAME_SERVERS")
- os.Setenv("AWS_K8S_TESTER_EKS_TAGS", `{"to-delete":"2020","hello-world": "test"}`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_TAGS")
- os.Setenv("AWS_K8S_TESTER_EKS_REQUEST_HEADER_KEY", "eks-options")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_REQUEST_HEADER_KEY")
- os.Setenv("AWS_K8S_TESTER_EKS_REQUEST_HEADER_VALUE", "kubernetesVersion=1.11")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_REQUEST_HEADER_VALUE")
- os.Setenv("AWS_K8S_TESTER_EKS_RESOLVER_URL", "amazon.com")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_RESOLVER_URL")
- os.Setenv("AWS_K8S_TESTER_EKS_SIGNING_NAME", "a")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_SIGNING_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ROLE_CREATE", "false")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ROLE_CREATE")
- os.Setenv("AWS_K8S_TESTER_EKS_ROLE_ARN", "cluster-role-arn")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ROLE_ARN")
- os.Setenv("AWS_K8S_TESTER_EKS_ROLE_SERVICE_PRINCIPALS", "eks.amazonaws.com,eks-beta-pdx.aws.internal,eks-dev.aws.internal")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ROLE_SERVICE_PRINCIPALS")
- os.Setenv("AWS_K8S_TESTER_EKS_ROLE_MANAGED_POLICY_ARNS", "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy,arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ROLE_MANAGED_POLICY_ARNS")
- os.Setenv("AWS_K8S_TESTER_EKS_VERSION", "1.18")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_VERSION")
- os.Setenv("AWS_K8S_TESTER_EKS_ENCRYPTION_CMK_CREATE", "false")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ENCRYPTION_CMK_CREATE")
- os.Setenv("AWS_K8S_TESTER_EKS_ENCRYPTION_CMK_ARN", "key-arn")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ENCRYPTION_CMK_ARN")
- os.Setenv("AWS_K8S_TESTER_EKS_KUBE_APISERVER_MAX_REQUESTS_INFLIGHT", "3000")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_KUBE_APISERVER_MAX_REQUESTS_INFLIGHT")
- os.Setenv("AWS_K8S_TESTER_EKS_KUBE_CONTROLLER_MANAGER_QPS", "500")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_KUBE_CONTROLLER_MANAGER_QPS")
- os.Setenv("AWS_K8S_TESTER_EKS_KUBE_CONTROLLER_MANAGER_BURST", "500")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_KUBE_CONTROLLER_MANAGER_BURST")
- os.Setenv("AWS_K8S_TESTER_EKS_KUBE_SCHEDULER_QPS", "500")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_KUBE_SCHEDULER_QPS")
- os.Setenv("AWS_K8S_TESTER_EKS_KUBE_SCHEDULER_BURST", "500")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_KUBE_SCHEDULER_BURST")
- os.Setenv("AWS_K8S_TESTER_EKS_FE_UPDATE_MASTER_FLAGS_URL", "uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_FE_UPDATE_MASTER_FLAGS_URL")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_VERSION_UPGRADE_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_VERSION_UPGRADE_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_VERSION_UPGRADE_WAIT_BEFORE_UPGRADE", "1h27m")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_WAIT_BEFORE_UPGRADE_UPGRADE_VERSION")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_VERSION_UPGRADE_VERSION", "1.19")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_VERSION_UPGRADE_VERSION")
-
- os.Setenv("AWS_K8S_TESTER_EKS_REMOTE_ACCESS_KEY_CREATE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_REMOTE_ACCESS_KEY_CREATE")
- os.Setenv("AWS_K8S_TESTER_EKS_REMOTE_ACCESS_KEY_NAME", "a")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_REMOTE_ACCESS_KEY_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_REMOTE_ACCESS_PRIVATE_KEY_PATH", "a")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_REMOTE_ACCESS_PRIVATE_KEY_PATH")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_FETCH_LOGS", "false")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_FETCH_LOGS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS", `{"ng-test-name-cpu":{"name":"ng-test-name-cpu","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64","image-id-ssm-parameter":"/aws/service/eks/optimized-ami/1.30/amazon-linux-2/recommended/image_id","asg-min-size":17,"kubelet-extra-args":"bbb qq","bootstrap-args":"--pause-container-account 012345678901", "cluster-autoscaler" : {"enable" : false}, "asg-max-size":99,"asg-desired-capacity":77,"instance-type":"type-cpu-2","volume-size":40,"volume-type":"gp2"},"ng-test-name-gpu":{"name":"ng-test-name-gpu","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64_GPU","asg-min-size":30,"asg-max-size":35,"asg-desired-capacity":34,"instance-type":"type-gpu-2","image-id":"my-gpu-ami","volume-size":500,"volume-type":"gp3","cluster-autoscaler": {"enable":false},"kubelet-extra-args":"aaa aa"}}`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_NAME", "a")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_LOGS_DIR", "a")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_LOGS_DIR")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_FETCH_LOGS", "false")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_FETCH_LOGS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_CREATE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_CREATE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_NAME", "mng-role-name")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_ARN", "mng-role-arn")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_ARN")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_SERVICE_PRINCIPALS", "ec2.amazonaws.com,eks.amazonaws.com,hello.amazonaws.com")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_SERVICE_PRINCIPALS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_MANAGED_POLICY_ARNS", "a,b,c")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_MANAGED_POLICY_ARNS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_REQUEST_HEADER_KEY", "a")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_REQUEST_HEADER_KEY")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_REQUEST_HEADER_VALUE", "b")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_REQUEST_HEADER_VALUE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_RESOLVER_URL", "a")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_RESOLVER_URL")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_SIGNING_NAME", "a")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_SIGNING_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS", `{"mng-test-name-cpu":{"name":"mng-test-name-cpu","tags":{"cpu":"hello-world"},"remote-access-user-name":"ec2-user","release-version":"","ami-type":"AL2_x86_64","asg-min-size":17,"scale-updates":[{"enable":true,"asg-min-size":500,"asg-max-size":700, "asg-desired-capacity":600, "initial-wait-string":"1h33m"}, {"enable":true,"asg-min-size":555,"asg-max-size":755, "asg-desired-capacity":655, "initial-wait-string":"2h33m", "id":"hello1"}],"version-upgrade":{"enable":true,"initial-wait-string":"13m","version":"1.19"},"asg-max-size":99,"asg-desired-capacity":77,"instance-types":["type-cpu-1","type-cpu-2"],"volume-size":40},"mng-test-name-gpu":{"name":"mng-test-name-gpu","remote-access-user-name":"ec2-user","tags":{"gpu":"hello-world"},"release-version":"1.18.8-20200609","ami-type":"AL2_x86_64_GPU","version-upgrade":{"enable":true,"initial-wait-string":"100ms", "version":"1.19"},"scale-updates":[{"enable":true,"asg-min-size":500,"asg-max-size":700, "asg-desired-capacity":600, "initial-wait-string":"1h33m"}, {"enable":true,"asg-min-size":555,"asg-max-size":755, "asg-desired-capacity":655, "initial-wait-string":"2h33m", "id":"hello2"}],"asg-min-size":30,"asg-max-size":35,"asg-desired-capacity":34,"instance-types":["type-gpu-1","type-gpu-2"],"volume-size":500}}`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_LOGS_DIR", "a")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_LOGS_DIR")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_INIT_ACCOUNT_ID", "uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_INIT_ACCOUNT_ID")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_INIT_REGION", "eu-north-1")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_INIT_REGION")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_INIT_NAME", "aws-k8s-cni-init")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_INIT_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_INIT_IMAGE_TAG", "latest2")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_INIT_IMAGE_TAG")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_ACCOUNT_ID", "uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_ACCOUNT_ID")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_REGION", "eu-north-1")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_REGION")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_NAME", "aws-k8s-cni")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_IMAGE_TAG", "latest2")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_REPOSITORY_IMAGE_TAG")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_NODE_SELECTOR", `{"a":"b","c":"d"}`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_NODE_SELECTOR")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_MINIMUM_IP_TARGET", "10")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_MINIMUM_IP_TARGET")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_WARM_IP_TARGET", "10")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_WARM_IP_TARGET")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CW_AGENT_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CW_AGENT_ENABLE")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_REPOSITORY_BUSYBOX_ACCOUNT_ID", "uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_REPOSITORY_BUSYBOX_ACCOUNT_ID")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_REPOSITORY_BUSYBOX_REGION", "eu-north-1")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_REPOSITORY_BUSYBOX_REGION")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_REPOSITORY_BUSYBOX_NAME", "busybox")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_REPOSITORY_BUSYBOX_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_REPOSITORY_BUSYBOX_IMAGE_TAG", "latest2")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_REPOSITORY_BUSYBOX_IMAGE_TAG")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_METADATA_WATCH", "false")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_METADATA_WATCH")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_DEPLOYMENT_REPLICAS", "333")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_DEPLOYMENT_REPLICAS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_NAMESPACE", "test-namespace")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_DEPLOYMENT_NODE_SELECTOR", `{"a":"b","c":"d"}`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_PHP_APACHE_DEPLOYMENT_NODE_SELECTOR")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_DEPLOYMENT_REPLICAS", "333")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_DEPLOYMENT_REPLICAS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_NAMESPACE", "test-namespace")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_DEPLOYMENT_NODE_SELECTOR", `{"a":"b","c":"d"}`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_DEPLOYMENT_NODE_SELECTOR")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_DEPLOYMENT_REPLICAS", "333")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_DEPLOYMENT_REPLICAS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_NAMESPACE", "test-namespace")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_DEPLOYMENT_NODE_SELECTOR", `{"a":"b","c":"d"}`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_GUESTBOOK_DEPLOYMENT_NODE_SELECTOR")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_DEPLOYMENT_REPLICAS_ALB", "333")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_DEPLOYMENT_REPLICAS_ALB")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_DEPLOYMENT_REPLICAS_2048", "555")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_DEPLOYMENT_REPLICAS_2048")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_NAMESPACE", "test-namespace")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_DEPLOYMENT_NODE_SELECTOR_2048", `{"1":"2","3":"4", "5":"6"}`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_DEPLOYMENT_NODE_SELECTOR_2048")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_NAMESPACE", "hello1")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_COMPLETES", "100")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_COMPLETES")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_PARALLELS", "10")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_PI_PARALLELS")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_NAMESPACE", "hello2")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_COMPLETES", "1000")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_COMPLETES")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_PARALLELS", "100")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_PARALLELS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_ECHO_SIZE", "10000")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_ECHO_SIZE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_REPOSITORY_BUSYBOX_ACCOUNT_ID", "uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_REPOSITORY_BUSYBOX_ACCOUNT_ID")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_REPOSITORY_BUSYBOX_REGION", "eu-north-1")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_REPOSITORY_BUSYBOX_REGION")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_REPOSITORY_BUSYBOX_NAME", "busybox")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_REPOSITORY_BUSYBOX_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_REPOSITORY_BUSYBOX_IMAGE_TAG", "latest2")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JOBS_ECHO_REPOSITORY_BUSYBOX_IMAGE_TAG")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_NAMESPACE", "hello3")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_SCHEDULE", "*/1 * * * *")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_SCHEDULE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_COMPLETES", "100")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_COMPLETES")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_PARALLELS", "10")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_PARALLELS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_SUCCESSFUL_JOBS_HISTORY_LIMIT", "100")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_SUCCESSFUL_JOBS_HISTORY_LIMIT")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_FAILED_JOBS_HISTORY_LIMIT", "1000")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_FAILED_JOBS_HISTORY_LIMIT")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_ECHO_SIZE", "10000")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_ECHO_SIZE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_REPOSITORY_BUSYBOX_ACCOUNT_ID", "uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_REPOSITORY_BUSYBOX_ACCOUNT_ID")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_REPOSITORY_BUSYBOX_REGION", "eu-north-1")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_REPOSITORY_BUSYBOX_REGION")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_REPOSITORY_BUSYBOX_NAME", "busybox")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_REPOSITORY_BUSYBOX_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_REPOSITORY_BUSYBOX_IMAGE_TAG", "latest2")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CRON_JOBS_REPOSITORY_BUSYBOX_IMAGE_TAG")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_INITIAL_REQUEST_CONDITION_TYPE", "Random")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_INITIAL_REQUEST_CONDITION_TYPE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_OBJECTS", "10000")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_OBJECTS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_S3_DIR", "a/b/c")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_LOCAL_REQUESTS_RAW_WRITES_COMPARE_S3_DIR")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_NAMESPACE", "csr-namespace")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_INITIAL_REQUEST_CONDITION_TYPE", "Random")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_INITIAL_REQUEST_CONDITION_TYPE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_OBJECTS", "10000")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_OBJECTS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REPOSITORY_ACCOUNT_ID", "uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REPOSITORY_ACCOUNT_ID")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REPOSITORY_REGION", "eu-north-1")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REPOSITORY_REGION")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REPOSITORY_NAME", "csrs-repo-name")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REPOSITORY_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REPOSITORY_IMAGE_TAG", "csrs-repo-image-tag")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSRS_REMOTE_REPOSITORY_IMAGE_TAG")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_NAMESPACE", "configmap-namespace")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_OBJECTS", "10000")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_OBJECTS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_OBJECT_SIZE", "555")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_LOCAL_OBJECT_SIZE")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_NAMESPACE", "configmap-namespace")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_OBJECTS", "10000")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_OBJECTS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_OBJECT_SIZE", "555")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_OBJECT_SIZE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REPOSITORY_ACCOUNT_ID", "uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REPOSITORY_ACCOUNT_ID")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REPOSITORY_NAME", "configmaps-repo-name")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REPOSITORY_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REPOSITORY_IMAGE_TAG", "configmaps-repo-image-tag")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFIGMAPS_REMOTE_REPOSITORY_IMAGE_TAG")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_NAMESPACE", "hello")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_OBJECTS", "5")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_OBJECTS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_OBJECT_SIZE", "10")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_LOCAL_OBJECT_SIZE")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_NAMESPACE", "hello")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_S3_DIR", "hello-s3")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_S3_DIR")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_OBJECTS", "5")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_OBJECTS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_OBJECT_SIZE", "10")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_OBJECT_SIZE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REPOSITORY_ACCOUNT_ID", "uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REPOSITORY_ACCOUNT_ID")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REPOSITORY_NAME", "secrets-repo-name")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REPOSITORY_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REPOSITORY_IMAGE_TAG", "secrets-repo-image-tag")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_SECRETS_REMOTE_REPOSITORY_IMAGE_TAG")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_NAMESPACE", "hello")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ROLE_NAME", "hello")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ROLE_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ROLE_SERVICE_PRINCIPALS", "a,b,c")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ROLE_SERVICE_PRINCIPALS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ROLE_MANAGED_POLICY_ARNS", "a,b,c")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_ROLE_MANAGED_POLICY_ARNS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_PROFILE_NAME", "hello")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_PROFILE_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_REPOSITORY_ACCOUNT_ID", "uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_REPOSITORY_ACCOUNT_ID")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_REPOSITORY_NAME", "fargate-repo-name")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_REPOSITORY_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_REPOSITORY_IMAGE_TAG", "fargate-repo-image-tag")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_REPOSITORY_IMAGE_TAG")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_SECRET_NAME", "HELLO-SECRET")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_FARGATE_SECRET_NAME")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_NAMESPACE", "hello")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_ROLE_NAME", "hello")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_ROLE_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_S3_KEY", "hello")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_S3_KEY")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_REPOSITORY_ACCOUNT_ID", "uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_REPOSITORY_ACCOUNT_ID")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_REPOSITORY_NAME", "irsa-repo-name")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_REPOSITORY_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_REPOSITORY_IMAGE_TAG", "irsa-repo-image-tag")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_REPOSITORY_IMAGE_TAG")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_DEPLOYMENT_RESULT_PATH", "hello-deployment.log")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_DEPLOYMENT_RESULT_PATH")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_NAMESPACE", "hello")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ROLE_NAME", "hello")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ROLE_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ROLE_SERVICE_PRINCIPALS", "a,b,c")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ROLE_SERVICE_PRINCIPALS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ROLE_MANAGED_POLICY_ARNS", "a,b,c")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_ROLE_MANAGED_POLICY_ARNS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_S3_KEY", "hello")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_S3_KEY")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_PROFILE_NAME", "hello")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_PROFILE_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_REPOSITORY_ACCOUNT_ID", "uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_REPOSITORY_ACCOUNT_ID")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_REPOSITORY_NAME", "irsa-fargate-repo-name")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_REPOSITORY_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_REPOSITORY_IMAGE_TAG", "irsa-fargate-repo-image-tag")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_FARGATE_REPOSITORY_IMAGE_TAG")
-
- proxySecretToken := randutil.Hex(32)
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_NAMESPACE", "jhhub")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_PROXY_SECRET_TOKEN", proxySecretToken)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_JUPYTER_HUB_PROXY_SECRET_TOKEN")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CUDA_VECTOR_ADD_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CUDA_VECTOR_ADD_ENABLE")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_TEST_CONFIG_PATH", "artifacts/clusterloader2-testing-load-config.yaml")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_TEST_CONFIG_PATH")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_RUNS", "11")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_RUNS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_TIMEOUT", "3m30s")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_TIMEOUT")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_NODES", "11")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_NODES")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_NODES_PER_NAMESPACE", "11")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_NODES_PER_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_PODS_PER_NODE", "21")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_PODS_PER_NODE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_BIG_GROUP_SIZE", "26")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_BIG_GROUP_SIZE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_MEDIUM_GROUP_SIZE", "11")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_MEDIUM_GROUP_SIZE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_SMALL_GROUP_SIZE", "6")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_SMALL_GROUP_SIZE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_SMALL_STATEFUL_SETS_PER_NAMESPACE", "1")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_SMALL_STATEFUL_SETS_PER_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_MEDIUM_STATEFUL_SETS_PER_NAMESPACE", "1")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_MEDIUM_STATEFUL_SETS_PER_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_CL2_SCHEDULER_THROUGHPUT_THRESHOLD", "20")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_CL2_SCHEDULER_THROUGHPUT_THRESHOLD")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_NAMESPACE", "hello")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_REPOSITORY_ACCOUNT_ID", "uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_REPOSITORY_ACCOUNT_ID")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_REPOSITORY_NAME", "hollow-nodes-repo-name")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_REPOSITORY_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_REPOSITORY_URI", "hollow-nodes-repo-uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_REPOSITORY_URI")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_REPOSITORY_IMAGE_TAG", "hollow-nodes-repo-image-tag")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_REPOSITORY_IMAGE_TAG")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_RUNS", "21")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_RUNS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_TIMEOUT", "5m30s")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_TIMEOUT")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_NODES", "11")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_NODES")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_NODES_PER_NAMESPACE", "11")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_NODES_PER_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_PODS_PER_NODE", "21")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_PODS_PER_NODE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_BIG_GROUP_SIZE", "26")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_BIG_GROUP_SIZE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_MEDIUM_GROUP_SIZE", "11")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_MEDIUM_GROUP_SIZE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_SMALL_GROUP_SIZE", "6")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_SMALL_GROUP_SIZE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_SMALL_STATEFUL_SETS_PER_NAMESPACE", "1")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_SMALL_STATEFUL_SETS_PER_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_MEDIUM_STATEFUL_SETS_PER_NAMESPACE", "1")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_MEDIUM_STATEFUL_SETS_PER_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_CL2_SCHEDULER_THROUGHPUT_THRESHOLD", "30")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_CL2_SCHEDULER_THROUGHPUT_THRESHOLD")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_DURATION", "7m30s")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_DURATION")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_LIST_LIMIT", "133")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_LOCAL_LIST_LIMIT")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_NAMESPACE", "hello")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_DURATION", "7m30s")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_DURATION")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REPOSITORY_ACCOUNT_ID", "uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REPOSITORY_ACCOUNT_ID")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REPOSITORY_NAME", "stresser-repo-name")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REPOSITORY_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REPOSITORY_IMAGE_TAG", "stresser-repo-image-tag")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REPOSITORY_IMAGE_TAG")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_COMPLETES", "500")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_COMPLETES")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_OBJECT_SIZE", "512")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_OBJECT_SIZE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_LIST_LIMIT", "177")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_LIST_LIMIT")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_WRITES_OUTPUT_NAME_PREFIX", "stresser-out-pfx")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_WRITES_OUTPUT_NAME_PREFIX")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_READS_OUTPUT_NAME_PREFIX", "stresser-out-pfx")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_REQUESTS_SUMMARY_READS_OUTPUT_NAME_PREFIX")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_NAMESPACE", "hello")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_REPOSITORY_ACCOUNT_ID", "uri")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_REPOSITORY_ACCOUNT_ID")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_REPOSITORY_NAME", "stresser-repo-name")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_REPOSITORY_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_REPOSITORY_IMAGE_TAG", "stresser-repo-image-tag")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_REPOSITORY_IMAGE_TAG")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_REPOSITORY_BUSYBOX_NAME", "stresser-busybox-name")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_REPOSITORY_BUSYBOX_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_REPOSITORY_BUSYBOX_IMAGE_TAG", "stresser-busybox-image-tag")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_REPOSITORY_BUSYBOX_IMAGE_TAG")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_SCHEDULE", "*/1 * * * *")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_SCHEDULE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_COMPLETES", "500")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_COMPLETES")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_PARALLELS", "500")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_PARALLELS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_SUCCESSFUL_JOBS_HISTORY_LIMIT", "500")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_SUCCESSFUL_JOBS_HISTORY_LIMIT")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_FAILED_JOBS_HISTORY_LIMIT", "500")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_FAILED_JOBS_HISTORY_LIMIT")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_OBJECT_SIZE", "512")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_OBJECT_SIZE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_DURATION", "7m30s")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_DURATION")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_COROUTINES", "10")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_COROUTINES")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_SECRETS", "10")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_STRESSER_REMOTE_V2_SECRETS")
- if err := cfg.UpdateFromEnvs(); err != nil {
- t.Fatal(err)
- }
-
- expectedSpec := Spec{
- ClusterAutoscaler: &ClusterAutoscalerSpec{
- Image: "767520670908.dkr.ecr.us-west-2.amazonaws.com/cluster-autoscaler-kubemark:custom-build-20200727",
- CloudProvider: CloudProviderKubemark,
- },
- Overprovisioning: &OverprovisioningSpec{
- KubemarkEnabled: true,
- },
- }
- if !reflect.DeepEqual(expectedSpec, cfg.Spec) {
- t.Fatalf("expected %+v, got %+v", expectedSpec, cfg.Spec)
- }
-
- if cfg.LogColor {
- t.Fatalf("unexpected LogColor %v", cfg.LogColor)
- }
- if cfg.LogColorOverride != "true" {
- t.Fatalf("unexpected LogColorOverride %q", cfg.LogColorOverride)
- }
- if cfg.KubectlCommandsOutputPath != "hello-kubectl" {
- t.Fatalf("unexpected %q", cfg.KubectlCommandsOutputPath)
- }
- if cfg.RemoteAccessCommandsOutputPath != "hello-ssh" {
- t.Fatalf("unexpected %q", cfg.RemoteAccessCommandsOutputPath)
- }
- if cfg.Region != "us-east-1" {
- t.Fatalf("unexpected %q", cfg.Region)
- }
- if cfg.LogLevel != "debug" {
- t.Fatalf("unexpected %q", cfg.LogLevel)
- }
- if cfg.KubectlDownloadURL != "https://amazon-eks.s3-us-west-2.amazonaws.com/1.11.5/2018-12-06/bin/linux/amd64/kubectl" {
- t.Fatalf("unexpected KubectlDownloadURL %q", cfg.KubectlDownloadURL)
- }
- if cfg.KubectlPath != "/tmp/aws-k8s-tester-test/kubectl" {
- t.Fatalf("unexpected KubectlPath %q", cfg.KubectlPath)
- }
- if cfg.KubeConfigPath != "/tmp/aws-k8s-tester/kubeconfig2" {
- t.Fatalf("unexpected KubeConfigPath %q", cfg.KubeConfigPath)
- }
- if cfg.OnFailureDelete {
- t.Fatalf("unexpected OnFailureDelete %v", cfg.OnFailureDelete)
- }
- if cfg.OnFailureDeleteWaitSeconds != 780 {
- t.Fatalf("unexpected OnFailureDeleteWaitSeconds %d", cfg.OnFailureDeleteWaitSeconds)
- }
- if cfg.CommandAfterCreateCluster != "echo hello1" {
- t.Fatalf("unexpected CommandAfterCreateCluster %q", cfg.CommandAfterCreateCluster)
- }
- if cfg.CommandAfterCreateClusterTimeout != 7*time.Minute {
- t.Fatalf("unexpected CommandAfterCreateClusterTimeout %v", cfg.CommandAfterCreateClusterTimeout)
- }
- if cfg.CommandAfterCreateAddOns != "echo hello2" {
- t.Fatalf("unexpected CommandAfterCreateAddOns %q", cfg.CommandAfterCreateAddOns)
- }
- if cfg.CommandAfterCreateAddOnsTimeout != 17*time.Minute {
- t.Fatalf("unexpected CommandAfterCreateAddOnsTimeout %v", cfg.CommandAfterCreateAddOnsTimeout)
- }
- if !cfg.S3.BucketCreate {
- t.Fatalf("unexpected cfg.S3.BucketCreate %v", cfg.S3.BucketCreate)
- }
- if !cfg.S3.BucketCreateKeep {
- t.Fatalf("unexpected cfg.S3.BucketCreateKeep %v", cfg.S3.BucketCreateKeep)
- }
- if cfg.S3.BucketName != "my-bucket" {
- t.Fatalf("unexpected cfg.S3.BucketName %q", cfg.S3.BucketName)
- }
- if cfg.S3.BucketLifecycleExpirationDays != 10 {
- t.Fatalf("unexpected cfg.S3.BucketLifecycleExpirationDays %d", cfg.S3.BucketLifecycleExpirationDays)
- }
- if cfg.Clients != 333 {
- t.Fatalf("unexpected cfg.Clients %d", cfg.Clients)
- }
- if cfg.ClientTimeout != 10*time.Minute {
- t.Fatalf("unexpected cfg.ClientTimeout %v", cfg.ClientTimeout)
- }
- if cfg.ClientQPS != 99555.77 {
- t.Fatalf("unexpected cfg.ClientQPS %f", cfg.ClientQPS)
- }
- if cfg.ClientBurst != 177 {
- t.Fatalf("unexpected cfg.ClientBurst %d", cfg.ClientBurst)
- }
-
- if !cfg.SkipDeleteClusterAndNodes {
- t.Fatalf("unexpected SkipDeleteClusterAndNodes %v", cfg.SkipDeleteClusterAndNodes)
- }
-
- if cfg.VPC.Create {
- t.Fatalf("unexpected VPC.Create %v", cfg.VPC.Create)
- }
- if cfg.VPC.ID != "vpc-id" {
- t.Fatalf("unexpected VPC.ID %q", cfg.VPC.ID)
- }
- if !reflect.DeepEqual(cfg.VPC.CIDRs, []string{"public-cidr1", "public-cidr2", "public-cidr3"}) {
- t.Fatalf("unexpected VPC.CIDRs %q", cfg.VPC.CIDRs)
- }
- if !reflect.DeepEqual(cfg.VPC.PublicSubnetCIDRs, []string{"public-cidr1", "public-cidr2", "public-cidr3"}) {
- t.Fatalf("unexpected VPC.PublicSubnetCIDRs %q", cfg.VPC.PublicSubnetCIDRs)
- }
- if !reflect.DeepEqual(cfg.VPC.PrivateSubnetCIDRs, []string{"private-cidr1", "private-cidr2"}) {
- t.Fatalf("unexpected VPC.PrivateSubnetCIDRs %q", cfg.VPC.PrivateSubnetCIDRs)
- }
-
- if cfg.VPC.DHCPOptionsDomainName != "hello.com" {
- t.Fatalf("unexpected cfg.VPC.DHCPOptionsDomainName %q", cfg.VPC.DHCPOptionsDomainName)
- }
- if !reflect.DeepEqual(cfg.VPC.DHCPOptionsDomainNameServers, []string{"1.2.3.0", "4.5.6.7"}) {
- t.Fatalf("unexpected cfg.VPC.DHCPOptionsDomainNameServers %q", cfg.VPC.DHCPOptionsDomainNameServers)
- }
- expectedTags := map[string]string{"to-delete": "2020", "hello-world": "test"}
- if !reflect.DeepEqual(cfg.Tags, expectedTags) {
- t.Fatalf("Tags expected %v, got %v", expectedTags, cfg.Tags)
- }
- if cfg.RequestHeaderKey != "eks-options" {
- t.Fatalf("unexpected RequestHeaderKey %q", cfg.RequestHeaderKey)
- }
- if cfg.RequestHeaderValue != "kubernetesVersion=1.11" {
- t.Fatalf("unexpected RequestHeaderValue %q", cfg.RequestHeaderValue)
- }
- if cfg.ResolverURL != "amazon.com" {
- t.Fatalf("unexpected ResolverURL %q", cfg.ResolverURL)
- }
- if cfg.SigningName != "a" {
- t.Fatalf("unexpected SigningName %q", cfg.SigningName)
- }
- if cfg.Role.Create {
- t.Fatalf("unexpected Role.Create %v", cfg.Role.Create)
- }
- if cfg.Role.ARN != "cluster-role-arn" {
- t.Fatalf("unexpected Role.ARN %q", cfg.Role.ARN)
- }
- expectedRoleServicePrincipals := []string{
- "eks.amazonaws.com",
- "eks-beta-pdx.aws.internal",
- "eks-dev.aws.internal",
- }
- if !reflect.DeepEqual(expectedRoleServicePrincipals, cfg.Role.ServicePrincipals) {
- t.Fatalf("unexpected RoleServicePrincipals %+v", cfg.Role.ServicePrincipals)
- }
- expectedRoleManagedPolicyARNs := []string{
- "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy",
- "arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM",
- }
- if !reflect.DeepEqual(expectedRoleManagedPolicyARNs, cfg.Role.ManagedPolicyARNs) {
- t.Fatalf("unexpected RoleManagedPolicyARNs %+v", cfg.Role.ManagedPolicyARNs)
- }
- if cfg.Version != "1.18" {
- t.Fatalf("unexpected Version %q", cfg.Version)
- }
- if cfg.Encryption.CMKCreate {
- t.Fatalf("unexpected Encryption.CMKCreate %v", cfg.Encryption.CMKCreate)
- }
- if cfg.Encryption.CMKARN != "key-arn" {
- t.Fatalf("unexpected Encryption.CMKARN %q", cfg.Encryption.CMKARN)
- }
- if cfg.KubeAPIServerMaxRequestsInflight != "3000" {
- t.Fatalf("unexpected KubeAPIServerMaxRequestsInflight %s", cfg.KubeAPIServerMaxRequestsInflight)
- }
- if cfg.KubeControllerManagerQPS != "500" {
- t.Fatalf("unexpected KubeControllerManagerQPS %s", cfg.KubeControllerManagerQPS)
- }
- if cfg.KubeControllerManagerBurst != "500" {
- t.Fatalf("unexpected KubeControllerManagerBurst %s", cfg.KubeControllerManagerBurst)
- }
- if cfg.KubeSchedulerQPS != "500" {
- t.Fatalf("unexpected KubeSchedulerQPS %s", cfg.KubeSchedulerQPS)
- }
- if cfg.KubeSchedulerBurst != "500" {
- t.Fatalf("unexpected KubeSchedulerBurst %s", cfg.KubeSchedulerBurst)
- }
- if cfg.FEUpdateMasterFlagsURL != "uri" {
- t.Fatalf("unexpected FEUpdateMasterFlagsURL %s", cfg.FEUpdateMasterFlagsURL)
- }
-
- if !cfg.AddOnClusterVersionUpgrade.Enable {
- t.Fatalf("unexpected AddOnClusterVersionUpgrade.Enable %v", cfg.AddOnClusterVersionUpgrade.Enable)
- }
- if cfg.AddOnClusterVersionUpgrade.WaitBeforeUpgrade != time.Hour+27*time.Minute {
- t.Fatalf("unexpected AddOnClusterVersionUpgrade.WaitBeforeUpgrade %v", cfg.AddOnClusterVersionUpgrade.WaitBeforeUpgrade)
- }
- if cfg.AddOnClusterVersionUpgrade.Version != "1.19" {
- t.Fatalf("unexpected AddOnClusterVersionUpgrade.Version %q", cfg.AddOnClusterVersionUpgrade.Version)
- }
-
- if !cfg.RemoteAccessKeyCreate {
- t.Fatalf("unexpected cfg.RemoteAccessKeyCreate %v", cfg.RemoteAccessKeyCreate)
- }
- if cfg.RemoteAccessKeyName != "a" {
- t.Fatalf("unexpected cfg.RemoteAccessKeyName %q", cfg.RemoteAccessKeyName)
- }
- if cfg.RemoteAccessPrivateKeyPath != "a" {
- t.Fatalf("unexpected cfg.RemoteAccessPrivateKeyPath %q", cfg.RemoteAccessPrivateKeyPath)
- }
-
- if !cfg.AddOnNodeGroups.Enable {
- t.Fatalf("unexpected cfg.AddOnNodeGroups.Enable %v", cfg.AddOnNodeGroups.Enable)
- }
- if cfg.AddOnNodeGroups.Role.Name != "a" {
- t.Fatalf("unexpected cfg.AddOnNodeGroups.Role.Name %v", cfg.AddOnNodeGroups.Role.Name)
- }
- if cfg.AddOnNodeGroups.FetchLogs {
- t.Fatalf("unexpected cfg.AddOnNodeGroups.FetchLogs %v", cfg.AddOnNodeGroups.FetchLogs)
- }
-
- cpuName, gpuName := "ng-test-name-cpu", "ng-test-name-gpu"
- expectedASGs := map[string]ASG{
- cpuName: {
- ASG: ec2config.ASG{
- Name: cpuName,
- RemoteAccessUserName: "ec2-user",
- AMIType: "AL2_x86_64",
- ImageIDSSMParameter: "/aws/service/eks/optimized-ami/1.30/amazon-linux-2/recommended/image_id",
- ASGMinSize: 17,
- ASGMaxSize: 99,
- ASGDesiredCapacity: 77,
- InstanceType: "type-cpu-2",
- VolumeSize: 40,
- VolumeType: "gp2",
- },
- BootstrapArgs: "--pause-container-account 012345678901",
- KubeletExtraArgs: "bbb qq",
- ClusterAutoscaler: &NGClusterAutoscaler{Enable: false},
- },
- gpuName: {
- ASG: ec2config.ASG{
- Name: gpuName,
- RemoteAccessUserName: "ec2-user",
- AMIType: eks.AMITypesAl2X8664Gpu,
- ImageID: "my-gpu-ami",
- ASGMinSize: 30,
- ASGMaxSize: 35,
- ASGDesiredCapacity: 34,
- InstanceType: "type-gpu-2",
- VolumeSize: 500,
- VolumeType: "gp3",
- },
- KubeletExtraArgs: "aaa aa",
- ClusterAutoscaler: &NGClusterAutoscaler{Enable: false},
- },
- }
- if !reflect.DeepEqual(cfg.AddOnNodeGroups.ASGs, expectedASGs) {
- t.Fatalf("expected cfg.AddOnNodeGroups.ASGs\n%+v\n\ngot\n%+v\n", expectedASGs, cfg.AddOnNodeGroups.ASGs)
- }
- if cfg.AddOnNodeGroups.LogsDir != "a" {
- t.Fatalf("unexpected cfg.AddOnNodeGroups.LogsDir %q", cfg.AddOnNodeGroups.LogsDir)
- }
-
- if !cfg.AddOnManagedNodeGroups.Enable {
- t.Fatalf("unexpected cfg.AddOnManagedNodeGroups.Enable %v", cfg.AddOnManagedNodeGroups.Enable)
- }
- if cfg.AddOnManagedNodeGroups.FetchLogs {
- t.Fatalf("unexpected cfg.AddOnManagedNodeGroups.FetchLogs %v", cfg.AddOnManagedNodeGroups.FetchLogs)
- }
- if !cfg.AddOnManagedNodeGroups.Role.Create {
- t.Fatalf("unexpected AddOnManagedNodeGroups.RoleCreate %v", cfg.AddOnManagedNodeGroups.Role.Create)
- }
- if cfg.AddOnManagedNodeGroups.Role.Name != "mng-role-name" {
- t.Fatalf("unexpected cfg.AddOnManagedNodeGroups.RoleName %q", cfg.AddOnManagedNodeGroups.Role.Name)
- }
- if cfg.AddOnManagedNodeGroups.Role.ARN != "mng-role-arn" {
- t.Fatalf("unexpected cfg.AddOnManagedNodeGroups.RoleARN %q", cfg.AddOnManagedNodeGroups.Role.ARN)
- }
- expectedMNGRoleServicePrincipals := []string{
- "ec2.amazonaws.com",
- "eks.amazonaws.com",
- "hello.amazonaws.com",
- }
- if !reflect.DeepEqual(expectedMNGRoleServicePrincipals, cfg.AddOnManagedNodeGroups.Role.ServicePrincipals) {
- t.Fatalf("unexpected cfg.AddOnManagedNodeGroups.RoleServicePrincipals %+v", cfg.AddOnManagedNodeGroups.Role.ServicePrincipals)
- }
- expectedMNGRoleManagedPolicyARNs := []string{
- "a",
- "b",
- "c",
- }
- if !reflect.DeepEqual(expectedMNGRoleManagedPolicyARNs, cfg.AddOnManagedNodeGroups.Role.ManagedPolicyARNs) {
- t.Fatalf("unexpected cfg.AddOnManagedNodeGroups.RoleManagedPolicyARNs %+v", cfg.AddOnManagedNodeGroups.Role.ManagedPolicyARNs)
- }
- if cfg.AddOnManagedNodeGroups.RequestHeaderKey != "a" {
- t.Fatalf("unexpected cfg.AddOnManagedNodeGroups.RequestHeaderKey %q", cfg.AddOnManagedNodeGroups.RequestHeaderKey)
- }
- if cfg.AddOnManagedNodeGroups.RequestHeaderValue != "b" {
- t.Fatalf("unexpected cfg.AddOnManagedNodeGroups.RequestHeaderValue %q", cfg.AddOnManagedNodeGroups.RequestHeaderValue)
- }
- if cfg.AddOnManagedNodeGroups.ResolverURL != "a" {
- t.Fatalf("unexpected cfg.AddOnManagedNodeGroups.ResolverURL %q", cfg.AddOnManagedNodeGroups.ResolverURL)
- }
- if cfg.AddOnManagedNodeGroups.SigningName != "a" {
- t.Fatalf("unexpected cfg.AddOnManagedNodeGroups.SigningName %q", cfg.AddOnManagedNodeGroups.SigningName)
- }
- // ,"scale-updates":[{"enable":true,"asg-min-size":500,"asg-max-size":700, "asg-desired-capacity":600, "initial-wait-string":"1h33m"}, {"enable":true,"asg-min-size":555,"asg-max-size":755, "asg-desired-capacity":655, "initial-wait-string":"2h33m"}],
- cpuName, gpuName = "mng-test-name-cpu", "mng-test-name-gpu"
- expectedMNGs := map[string]MNG{
- cpuName: {
- Name: cpuName,
- RemoteAccessUserName: "ec2-user",
- Tags: map[string]string{"cpu": "hello-world"},
- ReleaseVersion: "",
- AMIType: "AL2_x86_64",
- ASGMinSize: 17,
- ASGMaxSize: 99,
- ASGDesiredCapacity: 77,
- InstanceTypes: []string{"type-cpu-1", "type-cpu-2"},
- VolumeSize: 40,
- ScaleUpdates: []MNGScaleUpdate{
- {
- Enable: true,
- ASGMinSize: 500,
- ASGMaxSize: 700,
- ASGDesiredCapacity: 600,
- InitialWaitString: "1h33m",
- },
- {
- ID: "hello1",
- Enable: true,
- ASGMinSize: 555,
- ASGMaxSize: 755,
- ASGDesiredCapacity: 655,
- InitialWaitString: "2h33m",
- },
- },
- VersionUpgrade: &MNGVersionUpgrade{
- Enable: true,
- InitialWaitString: "13m",
- Version: "1.19",
- },
- },
- gpuName: {
- Name: gpuName,
- RemoteAccessUserName: "ec2-user",
- Tags: map[string]string{"gpu": "hello-world"},
- ReleaseVersion: "1.18.8-20200609",
- AMIType: eks.AMITypesAl2X8664Gpu,
- ASGMinSize: 30,
- ASGMaxSize: 35,
- ASGDesiredCapacity: 34,
- InstanceTypes: []string{"type-gpu-1", "type-gpu-2"},
- VolumeSize: 500,
- ScaleUpdates: []MNGScaleUpdate{
- {
- Enable: true,
- ASGMinSize: 500,
- ASGMaxSize: 700,
- ASGDesiredCapacity: 600,
- // InitialWait: 2*time.Hour + 33*time.Minute,
- InitialWaitString: "1h33m",
- },
- {
- ID: "hello2",
- Enable: true,
- ASGMinSize: 555,
- ASGMaxSize: 755,
- ASGDesiredCapacity: 655,
- // InitialWait: time.Hour + 33*time.Minute,
- InitialWaitString: "2h33m",
- },
- },
- VersionUpgrade: &MNGVersionUpgrade{
- Enable: true,
- InitialWaitString: "100ms",
- Version: "1.19",
- },
- },
- }
- if diff := cmp.Diff(cfg.AddOnManagedNodeGroups.MNGs, expectedMNGs); diff != "" {
- t.Fatalf("AddOnManagedNodeGroups.MNGs mismatch (-want +got):\n%s", diff)
- }
- if cfg.AddOnManagedNodeGroups.LogsDir != "a" {
- t.Fatalf("unexpected cfg.AddOnManagedNodeGroups.LogsDir %q", cfg.AddOnManagedNodeGroups.LogsDir)
- }
-
- if !cfg.AddOnCNIVPC.Enable {
- t.Fatalf("unexpected cfg.AddOnCNIVPC.Enable %v", cfg.AddOnCNIVPC.Enable)
- }
- if cfg.AddOnCNIVPC.RepositoryInitAccountID != "uri" {
- t.Fatalf("unexpected cfg.AddOnCNIVPC.RepositoryInitAccountID %v", cfg.AddOnCNIVPC.RepositoryInitAccountID)
- }
- if cfg.AddOnCNIVPC.RepositoryInitRegion != "eu-north-1" {
- t.Fatalf("unexpected cfg.AddOnCNIVPC.RepositoryInitRegion %v", cfg.AddOnCNIVPC.RepositoryInitRegion)
- }
- if cfg.AddOnCNIVPC.RepositoryInitName != "aws-k8s-cni-init" {
- t.Fatalf("unexpected cfg.AddOnCNIVPC.RepositoryInitName %v", cfg.AddOnCNIVPC.RepositoryInitName)
- }
- if cfg.AddOnCNIVPC.RepositoryInitImageTag != "latest2" {
- t.Fatalf("unexpected cfg.AddOnCNIVPC.RepositoryInitImageTag %v", cfg.AddOnCNIVPC.RepositoryInitImageTag)
- }
- if cfg.AddOnCNIVPC.RepositoryAccountID != "uri" {
- t.Fatalf("unexpected cfg.AddOnCNIVPC.RepositoryAccountID %v", cfg.AddOnCNIVPC.RepositoryAccountID)
- }
- if cfg.AddOnCNIVPC.RepositoryRegion != "eu-north-1" {
- t.Fatalf("unexpected cfg.AddOnCNIVPC.RepositoryRegion %v", cfg.AddOnCNIVPC.RepositoryRegion)
- }
- if cfg.AddOnCNIVPC.RepositoryName != "aws-k8s-cni" {
- t.Fatalf("unexpected cfg.AddOnCNIVPC.RepositoryName %v", cfg.AddOnCNIVPC.RepositoryName)
- }
- if cfg.AddOnCNIVPC.RepositoryImageTag != "latest2" {
- t.Fatalf("unexpected cfg.AddOnCNIVPC.RepositoryImageTag %v", cfg.AddOnCNIVPC.RepositoryImageTag)
- }
- expectedNodeSelectorCNIVPC := map[string]string{"a": "b", "c": "d"}
- if !reflect.DeepEqual(cfg.AddOnCNIVPC.NodeSelector, expectedNodeSelectorCNIVPC) {
- t.Fatalf("unexpected cfg.AddOnCNIVPC.NodeSelector %v", cfg.AddOnCNIVPC.NodeSelector)
- }
- if cfg.AddOnCNIVPC.MinimumIPTarget != 10 {
- t.Fatalf("unexpected cfg.AddOnCNIVPC.MinimumIPTarget %d", cfg.AddOnCNIVPC.MinimumIPTarget)
- }
- if cfg.AddOnCNIVPC.WarmIPTarget != 10 {
- t.Fatalf("unexpected cfg.AddOnCNIVPC.WarmIPTarget %d", cfg.AddOnCNIVPC.WarmIPTarget)
- }
-
- if !cfg.AddOnCWAgent.Enable {
- t.Fatalf("unexpected cfg.AddOnCWAgent.Enable %v", cfg.AddOnCWAgent.Enable)
- }
-
- if !cfg.AddOnFluentd.Enable {
- t.Fatalf("unexpected cfg.AddOnFluentd.Enable %v", cfg.AddOnFluentd.Enable)
- }
- if cfg.AddOnFluentd.RepositoryBusyboxAccountID != "uri" {
- t.Fatalf("unexpected cfg.AddOnFluentd.RepositoryBusyboxAccountID %v", cfg.AddOnFluentd.RepositoryBusyboxAccountID)
- }
- if cfg.AddOnFluentd.RepositoryBusyboxRegion != "eu-north-1" {
- t.Fatalf("unexpected cfg.AddOnFluentd.RepositoryBusyboxRegion %v", cfg.AddOnFluentd.RepositoryBusyboxRegion)
- }
- if cfg.AddOnFluentd.RepositoryBusyboxName != "busybox" {
- t.Fatalf("unexpected cfg.AddOnFluentd.RepositoryBusyboxName %v", cfg.AddOnFluentd.RepositoryBusyboxName)
- }
- if cfg.AddOnFluentd.RepositoryBusyboxImageTag != "latest2" {
- t.Fatalf("unexpected cfg.AddOnFluentd.RepositoryBusyboxImageTag %v", cfg.AddOnFluentd.RepositoryBusyboxImageTag)
- }
- if cfg.AddOnFluentd.MetadataWatch {
- t.Fatalf("unexpected cfg.AddOnFluentd.MetadataWatch %v", cfg.AddOnFluentd.MetadataWatch)
- }
-
- if !cfg.AddOnPHPApache.Enable {
- t.Fatalf("unexpected cfg.AddOnPHPApache.Enable %v", cfg.AddOnPHPApache.Enable)
- }
- if cfg.AddOnPHPApache.DeploymentReplicas != 333 {
- t.Fatalf("unexpected cfg.AddOnPHPApache.DeploymentReplicas %d", cfg.AddOnPHPApache.DeploymentReplicas)
- }
- if cfg.AddOnPHPApache.Namespace != "test-namespace" {
- t.Fatalf("unexpected cfg.AddOnPHPApache.Namespace %q", cfg.AddOnPHPApache.Namespace)
- }
- expectedNodeSelectorPHPApache := map[string]string{"a": "b", "c": "d"}
- if !reflect.DeepEqual(cfg.AddOnPHPApache.DeploymentNodeSelector, expectedNodeSelectorPHPApache) {
- t.Fatalf("unexpected cfg.AddOnPHPApache.DeploymentNodeSelector %v", cfg.AddOnPHPApache.DeploymentNodeSelector)
- }
-
- if !cfg.AddOnNLBHelloWorld.Enable {
- t.Fatalf("unexpected cfg.AddOnNLBHelloWorld.Enable %v", cfg.AddOnNLBHelloWorld.Enable)
- }
- if cfg.AddOnNLBHelloWorld.DeploymentReplicas != 333 {
- t.Fatalf("unexpected cfg.AddOnNLBHelloWorld.DeploymentReplicas %d", cfg.AddOnNLBHelloWorld.DeploymentReplicas)
- }
- if cfg.AddOnNLBHelloWorld.Namespace != "test-namespace" {
- t.Fatalf("unexpected cfg.AddOnNLBHelloWorld.Namespace %q", cfg.AddOnNLBHelloWorld.Namespace)
- }
- expectedNodeSelectorNLB := map[string]string{"a": "b", "c": "d"}
- if !reflect.DeepEqual(cfg.AddOnNLBHelloWorld.DeploymentNodeSelector, expectedNodeSelectorNLB) {
- t.Fatalf("unexpected cfg.AddOnNLBHelloWorld.DeploymentNodeSelector %v", cfg.AddOnNLBHelloWorld.DeploymentNodeSelector)
- }
-
- if !cfg.AddOnNLBGuestbook.Enable {
- t.Fatalf("unexpected cfg.AddOnNLBGuestbook.Enable %v", cfg.AddOnNLBGuestbook.Enable)
- }
- if cfg.AddOnNLBGuestbook.DeploymentReplicas != 333 {
- t.Fatalf("unexpected cfg.AddOnNLBGuestbook.DeploymentReplicas %d", cfg.AddOnNLBGuestbook.DeploymentReplicas)
- }
- if cfg.AddOnNLBGuestbook.Namespace != "test-namespace" {
- t.Fatalf("unexpected cfg.AddOnNLBGuestbook.Namespace %q", cfg.AddOnNLBGuestbook.Namespace)
- }
- if !reflect.DeepEqual(cfg.AddOnNLBGuestbook.DeploymentNodeSelector, expectedNodeSelectorNLB) {
- t.Fatalf("unexpected cfg.AddOnNLBGuestbook.DeploymentNodeSelector %v", cfg.AddOnNLBGuestbook.DeploymentNodeSelector)
- }
-
- if !cfg.AddOnALB2048.Enable {
- t.Fatalf("unexpected cfg.AddOnALB2048.Enable %v", cfg.AddOnALB2048.Enable)
- }
- if cfg.AddOnALB2048.DeploymentReplicasALB != 333 {
- t.Fatalf("unexpected cfg.AddOnALB2048.DeploymentReplicasALB %d", cfg.AddOnALB2048.DeploymentReplicasALB)
- }
- if cfg.AddOnALB2048.DeploymentReplicas2048 != 555 {
- t.Fatalf("unexpected cfg.AddOnALB2048.DeploymentReplicas2048 %d", cfg.AddOnALB2048.DeploymentReplicas2048)
- }
- if cfg.AddOnALB2048.Namespace != "test-namespace" {
- t.Fatalf("unexpected cfg.AddOnALB2048.Namespace %q", cfg.AddOnALB2048.Namespace)
- }
- expectedNodeSelectorALB := map[string]string{"1": "2", "3": "4", "5": "6"}
- if !reflect.DeepEqual(cfg.AddOnALB2048.DeploymentNodeSelector2048, expectedNodeSelectorALB) {
- t.Fatalf("unexpected cfg.AddOnALB2048.DeploymentNodeSelector2048 %v", cfg.AddOnALB2048.DeploymentNodeSelector2048)
- }
-
- if !cfg.AddOnJobsPi.Enable {
- t.Fatalf("unexpected cfg.AddOnJobsPi.Enable %v", cfg.AddOnJobsPi.Enable)
- }
- if cfg.AddOnJobsPi.Namespace != "hello1" {
- t.Fatalf("unexpected cfg.AddOnJobsPi.Namespace %q", cfg.AddOnJobsPi.Namespace)
- }
- if cfg.AddOnJobsPi.Completes != 100 {
- t.Fatalf("unexpected cfg.AddOnJobsPi.Completes %v", cfg.AddOnJobsPi.Completes)
- }
- if cfg.AddOnJobsPi.Parallels != 10 {
- t.Fatalf("unexpected cfg.AddOnJobsPi.Parallels %v", cfg.AddOnJobsPi.Parallels)
- }
-
- if !cfg.AddOnJobsEcho.Enable {
- t.Fatalf("unexpected cfg.AddOnJobsEcho.Enable %v", cfg.AddOnJobsEcho.Enable)
- }
- if cfg.AddOnJobsEcho.Namespace != "hello2" {
- t.Fatalf("unexpected cfg.AddOnJobsEcho.Namespace %q", cfg.AddOnJobsEcho.Namespace)
- }
- if cfg.AddOnJobsEcho.Completes != 1000 {
- t.Fatalf("unexpected cfg.AddOnJobsEcho.Completes %v", cfg.AddOnJobsEcho.Completes)
- }
- if cfg.AddOnJobsEcho.Parallels != 100 {
- t.Fatalf("unexpected cfg.AddOnJobsEcho.Parallels %v", cfg.AddOnJobsEcho.Parallels)
- }
- if cfg.AddOnJobsEcho.EchoSize != 10000 {
- t.Fatalf("unexpected cfg.AddOnJobsEcho.EchoSize %v", cfg.AddOnJobsEcho.EchoSize)
- }
- if cfg.AddOnJobsEcho.RepositoryBusyboxAccountID != "uri" {
- t.Fatalf("unexpected cfg.AddOnJobsEcho.RepositoryBusyboxAccountID %v", cfg.AddOnJobsEcho.RepositoryBusyboxAccountID)
- }
- if cfg.AddOnJobsEcho.RepositoryBusyboxRegion != "eu-north-1" {
- t.Fatalf("unexpected cfg.AddOnJobsEcho.RepositoryBusyboxRegion %v", cfg.AddOnJobsEcho.RepositoryBusyboxRegion)
- }
- if cfg.AddOnJobsEcho.RepositoryBusyboxName != "busybox" {
- t.Fatalf("unexpected cfg.AddOnJobsEcho.RepositoryBusyboxName %v", cfg.AddOnJobsEcho.RepositoryBusyboxName)
- }
- if cfg.AddOnJobsEcho.RepositoryBusyboxImageTag != "latest2" {
- t.Fatalf("unexpected cfg.AddOnJobsEcho.RepositoryBusyboxImageTag %v", cfg.AddOnJobsEcho.RepositoryBusyboxImageTag)
- }
-
- if !cfg.AddOnCronJobs.Enable {
- t.Fatalf("unexpected cfg.AddOnCronJobs.Enable %v", cfg.AddOnCronJobs.Enable)
- }
- if cfg.AddOnCronJobs.Namespace != "hello3" {
- t.Fatalf("unexpected cfg.AddOnCronJobs.Namespace %q", cfg.AddOnCronJobs.Namespace)
- }
- if cfg.AddOnCronJobs.Schedule != "*/1 * * * *" {
- t.Fatalf("unexpected cfg.AddOnCronJobs.Schedule %q", cfg.AddOnCronJobs.Schedule)
- }
- if cfg.AddOnCronJobs.Completes != 100 {
- t.Fatalf("unexpected cfg.AddOnCronJobs.Completes %v", cfg.AddOnCronJobs.Completes)
- }
- if cfg.AddOnCronJobs.Parallels != 10 {
- t.Fatalf("unexpected cfg.AddOnCronJobs.Parallels %v", cfg.AddOnCronJobs.Parallels)
- }
- if cfg.AddOnCronJobs.SuccessfulJobsHistoryLimit != 100 {
- t.Fatalf("unexpected cfg.AddOnCronJobs.SuccessfulJobsHistoryLimit %d", cfg.AddOnCronJobs.SuccessfulJobsHistoryLimit)
- }
- if cfg.AddOnCronJobs.FailedJobsHistoryLimit != 1000 {
- t.Fatalf("unexpected cfg.AddOnCronJobs.FailedJobsHistoryLimit %d", cfg.AddOnCronJobs.FailedJobsHistoryLimit)
- }
- if cfg.AddOnCronJobs.EchoSize != 10000 {
- t.Fatalf("unexpected cfg.AddOnCronJobs.EchoSize %d", cfg.AddOnCronJobs.EchoSize)
- }
- if cfg.AddOnCronJobs.RepositoryBusyboxAccountID != "uri" {
- t.Fatalf("unexpected cfg.AddOnCronJobs.RepositoryBusyboxAccountID %v", cfg.AddOnCronJobs.RepositoryBusyboxAccountID)
- }
- if cfg.AddOnCronJobs.RepositoryBusyboxRegion != "eu-north-1" {
- t.Fatalf("unexpected cfg.AddOnCronJobs.RepositoryBusyboxRegion %v", cfg.AddOnCronJobs.RepositoryBusyboxRegion)
- }
- if cfg.AddOnCronJobs.RepositoryBusyboxName != "busybox" {
- t.Fatalf("unexpected cfg.AddOnCronJobs.RepositoryBusyboxName %v", cfg.AddOnCronJobs.RepositoryBusyboxName)
- }
- if cfg.AddOnCronJobs.RepositoryBusyboxImageTag != "latest2" {
- t.Fatalf("unexpected cfg.AddOnCronJobs.RepositoryBusyboxImageTag %v", cfg.AddOnCronJobs.RepositoryBusyboxImageTag)
- }
-
- if !cfg.AddOnCSRsLocal.Enable {
- t.Fatalf("unexpected cfg.AddOnCSRsLocal.Enable %v", cfg.AddOnCSRsLocal.Enable)
- }
- if cfg.AddOnCSRsLocal.InitialRequestConditionType != "Random" {
- t.Fatalf("unexpected cfg.AddOnCSRsLocal.InitialRequestConditionType %q", cfg.AddOnCSRsLocal.InitialRequestConditionType)
- }
- if cfg.AddOnCSRsLocal.Objects != 10000 {
- t.Fatalf("unexpected cfg.AddOnCSRsLocal.Objects %d", cfg.AddOnCSRsLocal.Objects)
- }
- if cfg.AddOnCSRsLocal.RequestsRawWritesCompareS3Dir != "a/b/c" {
- t.Fatalf("unexpected cfg.AddOnCSRsLocal.RequestsRawWritesCompareS3Dir %q", cfg.AddOnCSRsLocal.RequestsRawWritesCompareS3Dir)
- }
-
- if !cfg.AddOnCSRsRemote.Enable {
- t.Fatalf("unexpected cfg.AddOnCSRsRemote.Enable %v", cfg.AddOnCSRsRemote.Enable)
- }
- if cfg.AddOnCSRsRemote.Namespace != "csr-namespace" {
- t.Fatalf("unexpected cfg.AddOnCSRsRemote.Namespace %q", cfg.AddOnCSRsRemote.Namespace)
- }
- if cfg.AddOnCSRsRemote.InitialRequestConditionType != "Random" {
- t.Fatalf("unexpected cfg.AddOnCSRsRemote.InitialRequestConditionType %q", cfg.AddOnCSRsRemote.InitialRequestConditionType)
- }
- if cfg.AddOnCSRsRemote.Objects != 10000 {
- t.Fatalf("unexpected cfg.AddOnCSRsRemote.Objects %d", cfg.AddOnCSRsRemote.Objects)
- }
- if cfg.AddOnCSRsRemote.RepositoryAccountID != "uri" {
- t.Fatalf("unexpected cfg.AddOnCSRsRemote.RepositoryAccountID %v", cfg.AddOnCSRsRemote.RepositoryAccountID)
- }
- if cfg.AddOnCSRsRemote.RepositoryRegion != "eu-north-1" {
- t.Fatalf("unexpected cfg.AddOnCSRsRemote.RepositoryRegion %v", cfg.AddOnCSRsRemote.RepositoryRegion)
- }
- if cfg.AddOnCSRsRemote.RepositoryName != "csrs-repo-name" {
- t.Fatalf("unexpected cfg.AddOnCSRsRemote.RepositoryName %v", cfg.AddOnCSRsRemote.RepositoryName)
- }
- if cfg.AddOnCSRsRemote.RepositoryImageTag != "csrs-repo-image-tag" {
- t.Fatalf("unexpected cfg.AddOnCSRsRemote.RepositoryImageTag %v", cfg.AddOnCSRsRemote.RepositoryImageTag)
- }
-
- if !cfg.AddOnConfigmapsLocal.Enable {
- t.Fatalf("unexpected cfg.AddOnConfigmapsLocal.Enable %v", cfg.AddOnConfigmapsLocal.Enable)
- }
- if cfg.AddOnConfigmapsLocal.Namespace != "configmap-namespace" {
- t.Fatalf("unexpected cfg.AddOnConfigmapsLocal.Namespace %q", cfg.AddOnConfigmapsLocal.Namespace)
- }
- if cfg.AddOnConfigmapsLocal.Objects != 10000 {
- t.Fatalf("unexpected cfg.AddOnConfigmapsLocal.Objects %d", cfg.AddOnConfigmapsLocal.Objects)
- }
- if cfg.AddOnConfigmapsLocal.ObjectSize != 555 {
- t.Fatalf("unexpected cfg.AddOnConfigmapsLocal.ObjectSize %d", cfg.AddOnConfigmapsLocal.ObjectSize)
- }
- if !cfg.AddOnConfigmapsRemote.Enable {
- t.Fatalf("unexpected cfg.AddOnConfigmapsRemote.Enable %v", cfg.AddOnConfigmapsRemote.Enable)
- }
- if cfg.AddOnConfigmapsRemote.Namespace != "configmap-namespace" {
- t.Fatalf("unexpected cfg.AddOnConfigmapsRemote.Namespace %q", cfg.AddOnConfigmapsRemote.Namespace)
- }
- if cfg.AddOnConfigmapsRemote.Objects != 10000 {
- t.Fatalf("unexpected cfg.AddOnConfigmapsRemote.Objects %d", cfg.AddOnConfigmapsRemote.Objects)
- }
- if cfg.AddOnConfigmapsRemote.ObjectSize != 555 {
- t.Fatalf("unexpected cfg.AddOnConfigmapsRemote.ObjectSize %d", cfg.AddOnConfigmapsRemote.ObjectSize)
- }
- if cfg.AddOnConfigmapsRemote.RepositoryAccountID != "uri" {
- t.Fatalf("unexpected cfg.AddOnConfigmapsRemote.RepositoryAccountID %v", cfg.AddOnConfigmapsRemote.RepositoryAccountID)
- }
- if cfg.AddOnConfigmapsRemote.RepositoryName != "configmaps-repo-name" {
- t.Fatalf("unexpected cfg.AddOnConfigmapsRemote.RepositoryName %v", cfg.AddOnConfigmapsRemote.RepositoryName)
- }
- if cfg.AddOnConfigmapsRemote.RepositoryImageTag != "configmaps-repo-image-tag" {
- t.Fatalf("unexpected cfg.AddOnConfigmapsRemote.RepositoryImageTag %v", cfg.AddOnConfigmapsRemote.RepositoryImageTag)
- }
-
- if !cfg.AddOnSecretsLocal.Enable {
- t.Fatalf("unexpected cfg.AddOnSecretsLocal.Enable %v", cfg.AddOnSecretsLocal.Enable)
- }
- if cfg.AddOnSecretsLocal.Namespace != "hello" {
- t.Fatalf("unexpected cfg.AddOnSecretsLocal.Namespace %q", cfg.AddOnSecretsLocal.Namespace)
- }
- if cfg.AddOnSecretsLocal.Objects != 5 {
- t.Fatalf("unexpected cfg.AddOnSecretsLocal.Objects %v", cfg.AddOnSecretsLocal.Objects)
- }
- if cfg.AddOnSecretsLocal.ObjectSize != 10 {
- t.Fatalf("unexpected cfg.AddOnSecretsLocal.ObjectSize %v", cfg.AddOnSecretsLocal.ObjectSize)
- }
-
- if !cfg.AddOnSecretsRemote.Enable {
- t.Fatalf("unexpected cfg.AddOnSecretsRemote.Enable %v", cfg.AddOnSecretsRemote.Enable)
- }
- if cfg.AddOnSecretsRemote.Namespace != "hello" {
- t.Fatalf("unexpected cfg.AddOnSecretsRemote.Namespace %q", cfg.AddOnSecretsRemote.Namespace)
- }
- if cfg.AddOnSecretsRemote.S3Dir != "hello-s3" {
- t.Fatalf("unexpected cfg.AddOnSecretsRemote.S3Dir %v", cfg.AddOnSecretsRemote.S3Dir)
- }
- if cfg.AddOnSecretsRemote.Objects != 5 {
- t.Fatalf("unexpected cfg.AddOnSecretsRemote.Objects %v", cfg.AddOnSecretsRemote.Objects)
- }
- if cfg.AddOnSecretsRemote.ObjectSize != 10 {
- t.Fatalf("unexpected cfg.AddOnSecretsRemote.ObjectSize %v", cfg.AddOnSecretsRemote.ObjectSize)
- }
- if cfg.AddOnSecretsRemote.RepositoryAccountID != "uri" {
- t.Fatalf("unexpected cfg.AddOnSecretsRemote.RepositoryAccountID %v", cfg.AddOnSecretsRemote.RepositoryAccountID)
- }
- if cfg.AddOnSecretsRemote.RepositoryName != "secrets-repo-name" {
- t.Fatalf("unexpected cfg.AddOnSecretsRemote.RepositoryName %v", cfg.AddOnSecretsRemote.RepositoryName)
- }
- if cfg.AddOnSecretsRemote.RepositoryImageTag != "secrets-repo-image-tag" {
- t.Fatalf("unexpected cfg.AddOnSecretsRemote.RepositoryImageTag %v", cfg.AddOnSecretsRemote.RepositoryImageTag)
- }
-
- if !cfg.AddOnFargate.Enable {
- t.Fatalf("unexpected cfg.AddOnFargate.Enable %v", cfg.AddOnFargate.Enable)
- }
- if cfg.AddOnFargate.Namespace != "hello" {
- t.Fatalf("unexpected cfg.AddOnFargate.Namespace %q", cfg.AddOnFargate.Namespace)
- }
- if cfg.AddOnFargate.RoleName != "hello" {
- t.Fatalf("unexpected cfg.AddOnFargate.RoleName %q", cfg.AddOnFargate.RoleName)
- }
- expectedAddOnFargateRoleServicePrincipals := []string{"a", "b", "c"}
- if !reflect.DeepEqual(cfg.AddOnFargate.RoleServicePrincipals, expectedAddOnFargateRoleServicePrincipals) {
- t.Fatalf("unexpected cfg.AddOnFargate.RoleServicePrincipals %q", cfg.AddOnFargate.RoleServicePrincipals)
- }
- expectedAddOnFargateRoleManagedPolicyARNs := []string{"a", "b", "c"}
- if !reflect.DeepEqual(cfg.AddOnFargate.RoleManagedPolicyARNs, expectedAddOnFargateRoleManagedPolicyARNs) {
- t.Fatalf("unexpected cfg.AddOnFargate.RoleManagedPolicyARNs %q", cfg.AddOnFargate.RoleManagedPolicyARNs)
- }
- if cfg.AddOnFargate.ProfileName != "hello" {
- t.Fatalf("unexpected cfg.AddOnFargate.ProfileName %q", cfg.AddOnFargate.ProfileName)
- }
- if cfg.AddOnFargate.RepositoryAccountID != "uri" {
- t.Fatalf("unexpected cfg.AddOnFargate.RepositoryAccountID %v", cfg.AddOnFargate.RepositoryAccountID)
- }
- if cfg.AddOnFargate.RepositoryName != "fargate-repo-name" {
- t.Fatalf("unexpected cfg.AddOnFargate.RepositoryName %v", cfg.AddOnFargate.RepositoryName)
- }
- if cfg.AddOnFargate.RepositoryImageTag != "fargate-repo-image-tag" {
- t.Fatalf("unexpected cfg.AddOnFargate.RepositoryImageTag %v", cfg.AddOnFargate.RepositoryImageTag)
- }
- if cfg.AddOnFargate.SecretName != "HELLO-SECRET" {
- t.Fatalf("unexpected cfg.AddOnFargate.SecretName %q", cfg.AddOnFargate.SecretName)
- }
-
- if !cfg.AddOnIRSA.Enable {
- t.Fatalf("unexpected cfg.AddOnIRSA.Enable %v", cfg.AddOnIRSA.Enable)
- }
- if cfg.AddOnIRSA.Namespace != "hello" {
- t.Fatalf("unexpected cfg.AddOnIRSA.Namespace %q", cfg.AddOnIRSA.Namespace)
- }
- if cfg.AddOnIRSA.RoleName != "hello" {
- t.Fatalf("unexpected cfg.AddOnIRSA.RoleName %q", cfg.AddOnIRSA.RoleName)
- }
- if cfg.AddOnIRSA.S3Key != "hello" {
- t.Fatalf("unexpected cfg.AddOnIRSA.S3Key %q", cfg.AddOnIRSA.S3Key)
- }
- if cfg.AddOnIRSA.RepositoryAccountID != "uri" {
- t.Fatalf("unexpected cfg.AddOnIRSA.RepositoryAccountID %v", cfg.AddOnIRSA.RepositoryAccountID)
- }
- if cfg.AddOnIRSA.RepositoryName != "irsa-repo-name" {
- t.Fatalf("unexpected cfg.AddOnIRSA.RepositoryName %v", cfg.AddOnIRSA.RepositoryName)
- }
- if cfg.AddOnIRSA.RepositoryImageTag != "irsa-repo-image-tag" {
- t.Fatalf("unexpected cfg.AddOnIRSA.RepositoryImageTag %v", cfg.AddOnIRSA.RepositoryImageTag)
- }
- if cfg.AddOnIRSA.DeploymentResultPath != "hello-deployment.log" {
- t.Fatalf("unexpected cfg.AddOnIRSA.DeploymentResultPath %q", cfg.AddOnIRSA.DeploymentResultPath)
- }
-
- if !cfg.AddOnIRSAFargate.Enable {
- t.Fatalf("unexpected cfg.AddOnIRSAFargate.Enable %v", cfg.AddOnIRSAFargate.Enable)
- }
- if cfg.AddOnIRSAFargate.Namespace != "hello" {
- t.Fatalf("unexpected cfg.AddOnIRSAFargate.Namespace %q", cfg.AddOnIRSAFargate.Namespace)
- }
- if cfg.AddOnIRSAFargate.RoleName != "hello" {
- t.Fatalf("unexpected cfg.AddOnIRSAFargate.RoleName %q", cfg.AddOnIRSAFargate.RoleName)
- }
- expectedAddOnIRSAFargateRoleServicePrincipals := []string{"a", "b", "c"}
- if !reflect.DeepEqual(cfg.AddOnIRSAFargate.RoleServicePrincipals, expectedAddOnIRSAFargateRoleServicePrincipals) {
- t.Fatalf("unexpected cfg.AddOnIRSAFargate.RoleServicePrincipals %q", cfg.AddOnIRSAFargate.RoleServicePrincipals)
- }
- expectedAddOnIRSAFargateRoleManagedPolicyARNs := []string{"a", "b", "c"}
- if !reflect.DeepEqual(cfg.AddOnIRSAFargate.RoleManagedPolicyARNs, expectedAddOnIRSAFargateRoleManagedPolicyARNs) {
- t.Fatalf("unexpected cfg.AddOnIRSAFargate.RoleManagedPolicyARNs %q", cfg.AddOnIRSAFargate.RoleManagedPolicyARNs)
- }
- if cfg.AddOnIRSAFargate.S3Key != "hello" {
- t.Fatalf("unexpected cfg.AddOnIRSAFargate.S3Key %q", cfg.AddOnIRSAFargate.S3Key)
- }
- if cfg.AddOnIRSAFargate.ProfileName != "hello" {
- t.Fatalf("unexpected cfg.AddOnIRSAFargate.ProfileName %q", cfg.AddOnIRSAFargate.ProfileName)
- }
- if cfg.AddOnIRSAFargate.RepositoryAccountID != "uri" {
- t.Fatalf("unexpected cfg.AddOnIRSAFargate.RepositoryAccountID %v", cfg.AddOnIRSAFargate.RepositoryAccountID)
- }
- if cfg.AddOnIRSAFargate.RepositoryName != "irsa-fargate-repo-name" {
- t.Fatalf("unexpected cfg.AddOnIRSAFargate.RepositoryName %v", cfg.AddOnIRSAFargate.RepositoryName)
- }
- if cfg.AddOnIRSAFargate.RepositoryImageTag != "irsa-fargate-repo-image-tag" {
- t.Fatalf("unexpected cfg.AddOnIRSAFargate.RepositoryImageTag %v", cfg.AddOnIRSAFargate.RepositoryImageTag)
- }
-
- if !cfg.AddOnJupyterHub.Enable {
- t.Fatalf("unexpected cfg.AddOnJupyterHub.Enable %v", cfg.AddOnJupyterHub.Enable)
- }
- if cfg.AddOnJupyterHub.Namespace != "jhhub" {
- t.Fatalf("unexpected cfg.AddOnJupyterHub.Namespace %q", cfg.AddOnJupyterHub.Namespace)
- }
- if cfg.AddOnJupyterHub.ProxySecretToken != proxySecretToken {
- t.Fatalf("unexpected cfg.AddOnJupyterHub.ProxySecretToken %q", cfg.AddOnJupyterHub.ProxySecretToken)
- }
-
- if !cfg.AddOnCUDAVectorAdd.Enable {
- t.Fatalf("Unexpected cfg.AddOnCUDAVectorAdd.Enable #{cfg.AddOnCUDAVectorAdd.Enable}")
- }
-
- if !cfg.AddOnClusterLoaderLocal.Enable {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderLocal.Enable %v", cfg.AddOnClusterLoaderLocal.Enable)
- }
- if cfg.AddOnClusterLoaderLocal.TestConfigPath != "artifacts/clusterloader2-testing-load-config.yaml" {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderLocal.TestConfigPath %v", cfg.AddOnClusterLoaderLocal.TestConfigPath)
- }
- if cfg.AddOnClusterLoaderLocal.Runs != 11 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderLocal.Runs %v", cfg.AddOnClusterLoaderLocal.Runs)
- }
- if cfg.AddOnClusterLoaderLocal.Timeout != 3*time.Minute+30*time.Second {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderLocal.Timeout %v", cfg.AddOnClusterLoaderLocal.Timeout)
- }
- if cfg.AddOnClusterLoaderLocal.Nodes != 11 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderLocal.Nodes %v", cfg.AddOnClusterLoaderLocal.Nodes)
- }
- if cfg.AddOnClusterLoaderLocal.NodesPerNamespace != 11 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderLocal.NodesPerNamespace %v", cfg.AddOnClusterLoaderLocal.NodesPerNamespace)
- }
- if cfg.AddOnClusterLoaderLocal.PodsPerNode != 21 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderLocal.PodsPerNode %v", cfg.AddOnClusterLoaderLocal.PodsPerNode)
- }
- if cfg.AddOnClusterLoaderLocal.BigGroupSize != 26 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderLocal.BigGroupSize %v", cfg.AddOnClusterLoaderLocal.BigGroupSize)
- }
- if cfg.AddOnClusterLoaderLocal.BigGroupSize != 26 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderLocal.BigGroupSize %v", cfg.AddOnClusterLoaderLocal.BigGroupSize)
- }
- if cfg.AddOnClusterLoaderLocal.MediumGroupSize != 11 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderLocal.MediumGroupSize %v", cfg.AddOnClusterLoaderLocal.MediumGroupSize)
- }
- if cfg.AddOnClusterLoaderLocal.SmallGroupSize != 6 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderLocal.SmallGroupSize %v", cfg.AddOnClusterLoaderLocal.SmallGroupSize)
- }
- if cfg.AddOnClusterLoaderLocal.SmallStatefulSetsPerNamespace != 1 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderLocal.SmallStatefulSetsPerNamespace %v", cfg.AddOnClusterLoaderLocal.SmallStatefulSetsPerNamespace)
- }
- if cfg.AddOnClusterLoaderLocal.MediumStatefulSetsPerNamespace != 1 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderLocal.MediumStatefulSetsPerNamespace %v", cfg.AddOnClusterLoaderLocal.MediumStatefulSetsPerNamespace)
- }
- if cfg.AddOnClusterLoaderLocal.CL2SchedulerThroughputThreshold != 20 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderLocal.CL2SchedulerThroughputThreshold %v", cfg.AddOnClusterLoaderLocal.CL2SchedulerThroughputThreshold)
- }
-
- if !cfg.AddOnClusterLoaderRemote.Enable {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderRemote.Enable %v", cfg.AddOnClusterLoaderRemote.Enable)
- }
- if cfg.AddOnClusterLoaderRemote.Namespace != "hello" {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderRemote.Namespace %q", cfg.AddOnClusterLoaderRemote.Namespace)
- }
- if cfg.AddOnClusterLoaderRemote.Runs != 21 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderRemote.Runs %v", cfg.AddOnClusterLoaderRemote.Runs)
- }
- if cfg.AddOnClusterLoaderRemote.Timeout != 5*time.Minute+30*time.Second {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderRemote.Timeout %v", cfg.AddOnClusterLoaderRemote.Timeout)
- }
- if cfg.AddOnClusterLoaderRemote.Nodes != 11 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderRemote.Nodes %v", cfg.AddOnClusterLoaderRemote.Nodes)
- }
- if cfg.AddOnClusterLoaderRemote.NodesPerNamespace != 11 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderRemote.NodesPerNamespace %v", cfg.AddOnClusterLoaderRemote.NodesPerNamespace)
- }
- if cfg.AddOnClusterLoaderRemote.PodsPerNode != 21 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderRemote.PodsPerNode %v", cfg.AddOnClusterLoaderRemote.PodsPerNode)
- }
- if cfg.AddOnClusterLoaderRemote.BigGroupSize != 26 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderRemote.BigGroupSize %v", cfg.AddOnClusterLoaderRemote.BigGroupSize)
- }
- if cfg.AddOnClusterLoaderRemote.BigGroupSize != 26 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderRemote.BigGroupSize %v", cfg.AddOnClusterLoaderRemote.BigGroupSize)
- }
- if cfg.AddOnClusterLoaderRemote.MediumGroupSize != 11 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderRemote.MediumGroupSize %v", cfg.AddOnClusterLoaderRemote.MediumGroupSize)
- }
- if cfg.AddOnClusterLoaderRemote.SmallGroupSize != 6 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderRemote.SmallGroupSize %v", cfg.AddOnClusterLoaderRemote.SmallGroupSize)
- }
- if cfg.AddOnClusterLoaderRemote.SmallStatefulSetsPerNamespace != 1 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderRemote.SmallStatefulSetsPerNamespace %v", cfg.AddOnClusterLoaderRemote.SmallStatefulSetsPerNamespace)
- }
- if cfg.AddOnClusterLoaderRemote.MediumStatefulSetsPerNamespace != 1 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderRemote.MediumStatefulSetsPerNamespace %v", cfg.AddOnClusterLoaderRemote.MediumStatefulSetsPerNamespace)
- }
- if cfg.AddOnClusterLoaderRemote.CL2SchedulerThroughputThreshold != 30 {
- t.Fatalf("unexpected cfg.AddOnClusterLoaderRemote.CL2SchedulerThroughputThreshold %v", cfg.AddOnClusterLoaderRemote.CL2SchedulerThroughputThreshold)
- }
-
- if !cfg.AddOnStresserLocal.Enable {
- t.Fatalf("unexpected cfg.AddOnStresserLocal.Enable %v", cfg.AddOnStresserLocal.Enable)
- }
- if cfg.AddOnStresserLocal.Duration != 7*time.Minute+30*time.Second {
- t.Fatalf("unexpected cfg.AddOnStresserLocal.Duration %v", cfg.AddOnStresserLocal.Duration)
- }
- if cfg.AddOnStresserLocal.ListLimit != 133 {
- t.Fatalf("unexpected cfg.AddOnStresserLocal.ListLimit %v", cfg.AddOnStresserLocal.ListLimit)
- }
-
- if !cfg.AddOnStresserRemote.Enable {
- t.Fatalf("unexpected cfg.AddOnStresserRemote.Enable %v", cfg.AddOnStresserRemote.Enable)
- }
- if cfg.AddOnStresserRemote.Namespace != "hello" {
- t.Fatalf("unexpected cfg.AddOnStresserRemote.Namespace %q", cfg.AddOnStresserRemote.Namespace)
- }
- if cfg.AddOnStresserRemote.Duration != 7*time.Minute+30*time.Second {
- t.Fatalf("unexpected cfg.AddOnStresserRemote.Duration %v", cfg.AddOnStresserRemote.Duration)
- }
- if cfg.AddOnStresserRemote.RepositoryAccountID != "uri" {
- t.Fatalf("unexpected cfg.AddOnStresserRemote.RepositoryAccountID %v", cfg.AddOnStresserRemote.RepositoryAccountID)
- }
- if cfg.AddOnStresserRemote.RepositoryName != "stresser-repo-name" {
- t.Fatalf("unexpected cfg.AddOnStresserRemote.RepositoryName %v", cfg.AddOnStresserRemote.RepositoryName)
- }
- if cfg.AddOnStresserRemote.RepositoryImageTag != "stresser-repo-image-tag" {
- t.Fatalf("unexpected cfg.AddOnStresserRemote.RepositoryImageTag %v", cfg.AddOnStresserRemote.RepositoryImageTag)
- }
- if cfg.AddOnStresserRemote.Completes != 500 {
- t.Fatalf("unexpected cfg.AddOnStresserRemote.Completes %v", cfg.AddOnStresserRemote.Completes)
- }
- if cfg.AddOnStresserRemote.ObjectSize != 512 {
- t.Fatalf("unexpected cfg.AddOnStresserRemote.ObjectSize %v", cfg.AddOnStresserRemote.ObjectSize)
- }
- if cfg.AddOnStresserRemote.ListLimit != 177 {
- t.Fatalf("unexpected cfg.AddOnStresserRemote.ListLimit %v", cfg.AddOnStresserRemote.ListLimit)
- }
- if cfg.AddOnStresserRemote.RequestsSummaryWritesOutputNamePrefix != "stresser-out-pfx" {
- t.Fatalf("unexpected cfg.AddOnStresserRemote.RequestsSummaryWritesOutputNamePrefix %v", cfg.AddOnStresserRemote.RequestsSummaryWritesOutputNamePrefix)
- }
- if cfg.AddOnStresserRemote.RequestsSummaryReadsOutputNamePrefix != "stresser-out-pfx" {
- t.Fatalf("unexpected cfg.AddOnStresserRemote.RequestsSummaryReadsOutputNamePrefix %v", cfg.AddOnStresserRemote.RequestsSummaryReadsOutputNamePrefix)
- }
-
- if !cfg.AddOnStresserRemoteV2.Enable {
- t.Fatalf("unexpected AddOnStresserRemoteV2.Enable %v", cfg.AddOnStresserRemoteV2.Enable)
- }
- if cfg.AddOnStresserRemoteV2.Namespace != "hello" {
- t.Fatalf("unexpected cfg.AddOnStresserRemoteV2.Namespace %q", cfg.AddOnStresserRemoteV2.Namespace)
- }
- if cfg.AddOnStresserRemoteV2.RepositoryAccountID != "uri" {
- t.Fatalf("unexpected cfg.AddOnStresserRemoteV2.RepositoryAccountID %s", cfg.AddOnStresserRemoteV2.RepositoryAccountID)
- }
- if cfg.AddOnStresserRemoteV2.RepositoryName != "stresser-repo-name" {
- t.Fatalf("unexpected cfg.AddOnStresserRemoteV2.RepositoryName %s", cfg.AddOnStresserRemoteV2.RepositoryName)
- }
- if cfg.AddOnStresserRemoteV2.RepositoryImageTag != "stresser-repo-image-tag" {
- t.Fatalf("unexpected cfg.AddOnStresserRemoteV2.RepositoryImageTag %s", cfg.AddOnStresserRemoteV2.RepositoryImageTag)
- }
- if cfg.AddOnStresserRemoteV2.RepositoryBusyBoxName != "stresser-busybox-name" {
- t.Fatalf("unexpected cfg.AddOnStresserRemoteV2.RepositoryBusyBoxName %s", cfg.AddOnStresserRemoteV2.RepositoryBusyBoxName)
- }
- if cfg.AddOnStresserRemoteV2.RepositoryBusyBoxImageTag != "stresser-busybox-image-tag" {
- t.Fatalf("unexpected cfg.AddOnStresserRemoteV2.RepositoryBusyBoxImageTag %s", cfg.AddOnStresserRemoteV2.RepositoryBusyBoxImageTag)
- }
- if cfg.AddOnStresserRemoteV2.Schedule != "*/1 * * * *" {
- t.Fatalf("unexpected cfg.AddOnStresserRemoteV2.Schedule %s", cfg.AddOnStresserRemoteV2.Schedule)
- }
- if cfg.AddOnStresserRemoteV2.Completes != 500 {
- t.Fatalf("unexpected cfg.AddOnStresserRemoteV2.Completes %d", cfg.AddOnStresserRemoteV2.Completes)
- }
- if cfg.AddOnStresserRemoteV2.Parallels != 500 {
- t.Fatalf("unexpected cfg.AddOnStresserRemoteV2.Parallels %d", cfg.AddOnStresserRemoteV2.Parallels)
- }
- if cfg.AddOnStresserRemoteV2.SuccessfulJobsHistoryLimit != 500 {
- t.Fatalf("unexpected cfg.AddOnStresserRemoteV2.SuccessfulJobsHistoryLimit %d", cfg.AddOnStresserRemoteV2.SuccessfulJobsHistoryLimit)
- }
- if cfg.AddOnStresserRemoteV2.FailedJobsHistoryLimit != 500 {
- t.Fatalf("unexpected cfg.AddOnStresserRemoteV2.FailedJobsHistoryLimit %d", cfg.AddOnStresserRemoteV2.FailedJobsHistoryLimit)
- }
- if cfg.AddOnStresserRemoteV2.ObjectSize != 512 {
- t.Fatalf("unexpected cfg.AddOnStresserRemoteV2.ObjectSize %d", cfg.AddOnStresserRemoteV2.ObjectSize)
- }
- if cfg.AddOnStresserRemoteV2.Duration != 7*time.Minute+30*time.Second {
- t.Fatalf("unexpected cfg.AddOnStresserRemoteV2.Duration %v", cfg.AddOnStresserRemoteV2.Duration)
- }
- if cfg.AddOnStresserRemoteV2.Coroutines != 10 {
- t.Fatalf("unexpected cfg.AddOnStresserRemoteV2.Coroutines %d", cfg.AddOnStresserRemoteV2.Coroutines)
- }
- if cfg.AddOnStresserRemoteV2.Secrets != 10 {
- t.Fatalf("unexpected cfg.AddOnStresserRemoteV2.Secrets %d", cfg.AddOnStresserRemoteV2.Secrets)
- }
-
- if err := cfg.ValidateAndSetDefaults(); err != nil {
- t.Fatal(err)
- }
-
- cfg.AddOnNLBHelloWorld.Enable = false
- cfg.AddOnALB2048.Enable = false
- cfg.AddOnJobsEcho.Enable = false
- cfg.AddOnJobsPi.Enable = false
- if err := cfg.ValidateAndSetDefaults(); err != nil {
- t.Fatal(err)
- }
-
- if cfg.ClientTimeoutString != "10m0s" {
- t.Fatalf("unexpected ClientTimeoutString %q", cfg.ClientTimeoutString)
- }
-
- if cfg.AddOnFargate.SecretName != "hellosecret" {
- t.Fatalf("unexpected cfg.AddOnFargate.SecretName %q", cfg.AddOnFargate.SecretName)
- }
-
- if cfg.TotalNodes != 222 {
- t.Fatalf("unexpected cfg.TotalNodes %d", cfg.TotalNodes)
- }
-
- d, err := ioutil.ReadFile(cfg.ConfigPath)
- if err != nil {
- t.Fatal(err)
- }
- fmt.Println(string(d))
-}
-
-func TestEnvAddOnManagedNodeGroups(t *testing.T) {
- cfg := NewDefault()
- defer func() {
- os.RemoveAll(cfg.ConfigPath)
- os.RemoveAll(cfg.KubectlCommandsOutputPath)
- os.RemoveAll(cfg.RemoteAccessCommandsOutputPath)
- }()
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ENABLE", "false")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE", "false")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE")
-
- if err := cfg.UpdateFromEnvs(); err != nil {
- t.Fatal(err)
- }
-
- if cfg.AddOnNodeGroups.Enable {
- t.Fatal("AddOnNodeGroups.Enable expected false, got true")
- }
- if cfg.AddOnManagedNodeGroups.Enable {
- t.Fatal("AddOnManagedNodeGroups.Enable expected false, got true")
- }
-
- cfg.AddOnNLBHelloWorld.Enable = true
- if err := cfg.ValidateAndSetDefaults(); !strings.Contains(err.Error(), "AddOnNLBHelloWorld.Enable true") {
- t.Fatalf("expected add-on error, got %v", err)
- }
-}
-
-func TestEnvAddOnNodeGroupsGetRef(t *testing.T) {
- cfg := NewDefault()
- defer func() {
- os.RemoveAll(cfg.ConfigPath)
- os.RemoveAll(cfg.KubectlCommandsOutputPath)
- os.RemoveAll(cfg.RemoteAccessCommandsOutputPath)
- }()
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ENABLE", `true`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS", `{"GetRef.Name-ng-for-cni":{"name":"GetRef.Name-ng-for-cni","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64","asg-min-size":30,"asg-max-size":35,"asg-desired-capacity":34, "instance-type":"type-2", "image-id":"my-ami", "ssm":{"document-create":true, "document-name":"GetRef.Name-document", "document-commands":"echo 1"}, "kubelet-extra-args":"aaa aa", "cluster-autoscaler": {"enable" : true}, "volume-size":500, "volume-type":"gp3"}}`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE", `true`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS", `{"GetRef.Name-mng-for-cni":{"name":"GetRef.Name-mng-for-cni","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["c5.xlarge"]}}`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS")
-
- if err := cfg.UpdateFromEnvs(); err != nil {
- t.Fatal(err)
- }
- if err := cfg.ValidateAndSetDefaults(); err != nil {
- t.Fatal(err)
- }
-
- cur1 := cfg.AddOnNodeGroups.ASGs[cfg.Name+"-ng-for-cni"]
- cfg.AddOnNodeGroups.ASGs[cfg.Name+"-ng-for-cni"] = cur1
- expectedNGs := map[string]ASG{
- cfg.Name + "-ng-for-cni": {
- ASG: ec2config.ASG{
- Name: cfg.Name + "-ng-for-cni",
- RemoteAccessUserName: "ec2-user",
- SSM: &ec2config.SSM{
- DocumentName: regex.ReplaceAllString(cfg.Name+"-document", ""),
- DocumentExecutionTimeoutSeconds: 3600,
- DocumentCreate: true,
- DocumentCommands: "echo 1",
- },
- ImageID: "my-ami",
- AMIType: eks.AMITypesAl2X8664,
- InstanceType: "type-2",
- VolumeSize: 500,
- VolumeType: "gp3",
- ASGMinSize: 30,
- ASGMaxSize: 35,
- ASGDesiredCapacity: 34,
- LaunchTemplateName: cfg.Name + "-ng-for-cni-launch-template",
- },
- KubeletExtraArgs: "aaa aa",
- ClusterAutoscaler: &NGClusterAutoscaler{Enable: true},
- },
- }
- if !reflect.DeepEqual(*cfg.AddOnNodeGroups.ASGs[cfg.Name+"-ng-for-cni"].SSM, *cur1.SSM) {
- t.Fatalf("expected cfg.AddOnNodeGroups.ASGs %+v, got %+v", expectedNGs, cfg.AddOnNodeGroups.ASGs)
- }
- if !reflect.DeepEqual(*cfg.AddOnNodeGroups.ASGs[cfg.Name+"-ng-for-cni"].ClusterAutoscaler, *cur1.ClusterAutoscaler) {
- t.Fatalf("expected cfg.AddOnNodeGroups.ASGs %+v, got %+v", expectedNGs, cfg.AddOnNodeGroups.ASGs)
- }
-
- v := cfg.AddOnNodeGroups.ASGs[cfg.Name+"-ng-for-cni"]
- v.ClusterAutoscaler = nil
- cfg.AddOnNodeGroups.ASGs[cfg.Name+"-ng-for-cni"] = v
-
- v2 := expectedNGs[cfg.Name+"-ng-for-cni"]
- v2.ClusterAutoscaler = nil
- expectedNGs[cfg.Name+"-ng-for-cni"] = v2
-
- if !reflect.DeepEqual(cfg.AddOnNodeGroups.ASGs, expectedNGs) {
- t.Fatalf("expected cfg.AddOnNodeGroups.ASGs %+v, got %+v", expectedNGs, cfg.AddOnNodeGroups.ASGs)
- }
-
- cur2 := cfg.AddOnManagedNodeGroups.MNGs[cfg.Name+"-mng-for-cni"]
- cfg.AddOnManagedNodeGroups.MNGs[cfg.Name+"-mng-for-cni"] = cur2
- expectedMNGs := map[string]MNG{
- cfg.Name + "-mng-for-cni": {
- Name: cfg.Name + "-mng-for-cni",
- RemoteAccessUserName: "ec2-user",
- Tags: map[string]string{"group": "amazon-vpc-cni-k8s"},
- AMIType: "AL2_x86_64",
- ASGMinSize: 3,
- ASGMaxSize: 3,
- ASGDesiredCapacity: 3,
- InstanceTypes: []string{"c5.xlarge"},
- VolumeSize: 40,
- },
- }
- if !reflect.DeepEqual(cfg.AddOnManagedNodeGroups.MNGs, expectedMNGs) {
- t.Fatalf("expected cfg.AddOnManagedNodeGroups.MNGs %+v, got %+v", expectedMNGs, cfg.AddOnManagedNodeGroups.MNGs)
- }
-}
-
-func TestEnvAddOnConformance(t *testing.T) {
- cfg := NewDefault()
- defer func() {
- os.RemoveAll(cfg.ConfigPath)
- os.RemoveAll(cfg.KubectlCommandsOutputPath)
- os.RemoveAll(cfg.RemoteAccessCommandsOutputPath)
- }()
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ENABLE", `true`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS", `{"GetRef.Name-ng-for-cni":{"name":"GetRef.Name-ng-for-cni","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64","asg-min-size":30,"asg-max-size":35,"instance-types":["type-2"],"asg-desired-capacity":34,"image-id":"my-ami", "ssm":{"document-create":true, "document-name":"GetRef.Name-document", "document-commands":"echo 1"}, "cluster-autoscaler": {"enable" : false}, "kubelet-extra-args":"aaa aa", "volume-size":500}}`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_NAMESPACE", "conformance-test")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_PATH", "aaaaa")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_PATH")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_DOWNLOAD_URL", "sonobuoy-download-here")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_DOWNLOAD_URL")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_DELETE_TIMEOUT", "10s")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_DELETE_TIMEOUT")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_TIMEOUT", "10h")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_TIMEOUT")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_MODE", "non-disruptive-conformance")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_MODE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_KUBE_CONFORMANCE_IMAGE", "hello.com/v1")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_KUBE_CONFORMANCE_IMAGE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_IMAGE", "sonobuoy/sonobuoy:v0.18.3")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_IMAGE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SYSTEMD_LOGS_IMAGE", "sonobuoy/systemd-logs:v0.3")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SYSTEMD_LOGS_IMAGE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_E2E_REPO_CONFIG", "/path/to/config.yml")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_E2E_REPO_CONFIG")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_E2E_FOCUS", "sig-network")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_E2E_FOCUS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_E2E_SKIP", "sig-network")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CONFORMANCE_SONOBUOY_RUN_E2E_SKIP")
-
- if err := cfg.UpdateFromEnvs(); err != nil {
- t.Fatal(err)
- }
- if err := cfg.ValidateAndSetDefaults(); err != nil {
- t.Fatal(err)
- }
-
- if !cfg.AddOnConformance.Enable {
- t.Fatalf("unexpected cfg.AddOnConformance.Enable %v", cfg.AddOnConformance.Enable)
- }
- if cfg.AddOnConformance.Namespace != "conformance-test" {
- t.Fatalf("unexpected cfg.AddOnConformance.Namespace %q", cfg.AddOnConformance.Namespace)
- }
- if cfg.AddOnConformance.SonobuoyPath != "aaaaa" {
- t.Fatalf("unexpected cfg.AddOnConformance.SonobuoyPath %q", cfg.AddOnConformance.SonobuoyPath)
- }
- if cfg.AddOnConformance.SonobuoyDownloadURL != "sonobuoy-download-here" {
- t.Fatalf("unexpected cfg.AddOnConformance.SonobuoyDownloadURL %q", cfg.AddOnConformance.SonobuoyDownloadURL)
- }
- if cfg.AddOnConformance.SonobuoyDeleteTimeout != 10*time.Second {
- t.Fatalf("unexpected cfg.AddOnConformance.SonobuoyDeleteTimeout %v", cfg.AddOnConformance.SonobuoyDeleteTimeout)
- }
- if cfg.AddOnConformance.SonobuoyRunTimeout != 10*time.Hour {
- t.Fatalf("unexpected cfg.AddOnConformance.SonobuoyRunTimeout %v", cfg.AddOnConformance.SonobuoyRunTimeout)
- }
- if cfg.AddOnConformance.SonobuoyRunMode != "non-disruptive-conformance" {
- t.Fatalf("unexpected cfg.AddOnConformance.SonobuoyRunMode %q", cfg.AddOnConformance.SonobuoyRunMode)
- }
- if cfg.AddOnConformance.SonobuoyRunKubeConformanceImage != "hello.com/v1" {
- t.Fatalf("unexpected cfg.AddOnConformance.SonobuoyRunKubeConformanceImage %q", cfg.AddOnConformance.SonobuoyRunKubeConformanceImage)
- }
- if cfg.AddOnConformance.SonobuoyImage != "sonobuoy/sonobuoy:v0.18.3" {
- t.Fatalf("unexpected cfg.AddOnConformance.SonobuoyImage %q", cfg.AddOnConformance.SonobuoyImage)
- }
- if cfg.AddOnConformance.SystemdLogsImage != "sonobuoy/systemd-logs:v0.3" {
- t.Fatalf("unexpected cfg.AddOnConformance.SystemdLogsImage %q", cfg.AddOnConformance.SystemdLogsImage)
- }
- if cfg.AddOnConformance.SonobuoyE2eRepoConfig != "/path/to/config.yml" {
- t.Fatalf("unexpected cfg.AddOnConformance.SonobuoyE2eRepoConfig %q", cfg.AddOnConformance.SonobuoyE2eRepoConfig)
- }
- if cfg.AddOnConformance.SonobuoyRunE2eFocus != "sig-network" {
- t.Fatalf("unexpected cfg.AddOnConformance.SonobuoyRunE2eFocus %q", cfg.AddOnConformance.SonobuoyRunE2eFocus)
- }
- if cfg.AddOnConformance.SonobuoyRunE2eSkip != "sig-network" {
- t.Fatalf("unexpected cfg.AddOnConformance.SonobuoyRunE2eSkip %q", cfg.AddOnConformance.SonobuoyRunE2eSkip)
- }
-}
-
-// TestEnvAddOnManagedNodeGroupsCNI tests CNI integration test MNG settings.
-// https://github.com/aws/amazon-vpc-cni-k8s/blob/master/scripts/lib/cluster.sh
-func TestEnvAddOnManagedNodeGroupsCNI(t *testing.T) {
- cfg := NewDefault()
- defer func() {
- os.RemoveAll(cfg.ConfigPath)
- os.RemoveAll(cfg.KubectlCommandsOutputPath)
- os.RemoveAll(cfg.RemoteAccessCommandsOutputPath)
- }()
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE", `true`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_REMOTE_ACCESS_PRIVATE_KEY_PATH", `a`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_REMOTE_ACCESS_PRIVATE_KEY_PATH")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS", `{"test-mng-for-cni":{"name":"test-mng-for-cni","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["c5.xlarge"]}}`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS")
-
- if err := cfg.UpdateFromEnvs(); err != nil {
- t.Fatal(err)
- }
- if err := cfg.ValidateAndSetDefaults(); err != nil {
- t.Fatal(err)
- }
-
- if cfg.RemoteAccessPrivateKeyPath != "a" {
- t.Fatalf("unexpected cfg.RemoteAccessPrivateKeyPath %q", cfg.RemoteAccessPrivateKeyPath)
- }
- cur := cfg.AddOnManagedNodeGroups.MNGs["test-mng-for-cni"]
- cfg.AddOnManagedNodeGroups.MNGs["test-mng-for-cni"] = cur
- expectedMNGs := map[string]MNG{
- "test-mng-for-cni": {
- Name: "test-mng-for-cni",
- RemoteAccessUserName: "ec2-user",
- Tags: map[string]string{"group": "amazon-vpc-cni-k8s"},
- AMIType: "AL2_x86_64",
- ASGMinSize: 3,
- ASGMaxSize: 3,
- ASGDesiredCapacity: 3,
- InstanceTypes: []string{"c5.xlarge"},
- VolumeSize: 40,
- },
- }
- if !reflect.DeepEqual(cfg.AddOnManagedNodeGroups.MNGs, expectedMNGs) {
- t.Fatalf("expected cfg.AddOnManagedNodeGroups.MNGs %+v, got %+v", expectedMNGs, cfg.AddOnManagedNodeGroups.MNGs)
- }
-}
-
-// TestEnvAddOnManagedNodeGroupsInvalidInstanceType tests invalid instance types.
-func TestEnvAddOnManagedNodeGroupsInvalidInstanceType(t *testing.T) {
- cfg := NewDefault()
- defer func() {
- os.RemoveAll(cfg.ConfigPath)
- os.RemoveAll(cfg.KubectlCommandsOutputPath)
- os.RemoveAll(cfg.RemoteAccessCommandsOutputPath)
- }()
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE", `true`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_REMOTE_ACCESS_PRIVATE_KEY_PATH", `a`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_REMOTE_ACCESS_PRIVATE_KEY_PATH")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS", `{"test-mng-for-cni":{"name":"test-mng-for-cni","tags":{"group":"amazon-vpc-cni-k8s"},"ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["m3.xlarge"]}}`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE", `true`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE")
-
- if err := cfg.UpdateFromEnvs(); err != nil {
- t.Fatal(err)
- }
- err := cfg.ValidateAndSetDefaults()
- if err == nil {
- t.Fatal("expected error")
- }
- if !strings.Contains(err.Error(), "older instance type InstanceTypes") {
- t.Fatalf("unexpected error %v", err)
- }
-}
-
-func TestEnvAddOnCSIEBS(t *testing.T) {
- cfg := NewDefault()
- defer func() {
- os.RemoveAll(cfg.ConfigPath)
- os.RemoveAll(cfg.KubectlCommandsOutputPath)
- os.RemoveAll(cfg.RemoteAccessCommandsOutputPath)
- }()
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE", `true`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSI_EBS_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSI_EBS_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSI_EBS_CHART_REPO_URL", "test-chart-repo")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSI_EBS_CHART_REPO_URL")
-
- if err := cfg.UpdateFromEnvs(); err != nil {
- t.Fatal(err)
- }
- err := cfg.ValidateAndSetDefaults()
- assert.NoError(t, err)
-
- if !cfg.AddOnCSIEBS.Enable {
- t.Fatalf("unexpected cfg.AddOnCSIEBS.Enable %v", cfg.AddOnCSIEBS.Enable)
- }
- if cfg.AddOnCSIEBS.ChartRepoURL != "test-chart-repo" {
- t.Fatalf("unexpected cfg.AddOnCSIEBS.ChartRepoURL %q", cfg.AddOnCSIEBS.ChartRepoURL)
- }
-}
-
-func TestEnvAddOnAppMesh(t *testing.T) {
- cfg := NewDefault()
- defer func() {
- os.RemoveAll(cfg.ConfigPath)
- os.RemoveAll(cfg.KubectlCommandsOutputPath)
- os.RemoveAll(cfg.RemoteAccessCommandsOutputPath)
- }()
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE", `true`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_ENABLE", `false`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_IRSA_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_NAMESPACE", "custom-namespace")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_CONTROLLER_IMAGE", "repo/controller:v1.1.3")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_CONTROLLER_IMAGE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_INJECTOR_IMAGE", "repo/injector:v1.1.3")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_APP_MESH_INJECTOR_IMAGE")
-
- if err := cfg.UpdateFromEnvs(); err != nil {
- t.Fatal(err)
- }
- err := cfg.ValidateAndSetDefaults()
- assert.NoError(t, err)
-
- assert.True(t, cfg.AddOnAppMesh.Enable)
- assert.Equal(t, cfg.AddOnAppMesh.Namespace, "custom-namespace")
- assert.Equal(t, cfg.AddOnAppMesh.ControllerImage, "repo/controller:v1.1.3")
- assert.Equal(t, cfg.AddOnAppMesh.InjectorImage, "repo/injector:v1.1.3")
-
- if cfg.AddOnAppMesh.PolicyCFNStackID != "" {
- t.Fatalf("read-only AddOnAppMesh.PolicyCFNStackID is set to %q", cfg.AddOnAppMesh.PolicyCFNStackID)
- }
-}
-
-func TestEnvAddOnWordpress(t *testing.T) {
- cfg := NewDefault()
- defer func() {
- os.RemoveAll(cfg.ConfigPath)
- os.RemoveAll(cfg.KubectlCommandsOutputPath)
- os.RemoveAll(cfg.RemoteAccessCommandsOutputPath)
- }()
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE", `true`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSI_EBS_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSI_EBS_ENABLE")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_NAMESPACE", "word-press")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_USER_NAME", "my-user")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_USER_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_PASSWORD", "my-password")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_WORDPRESS_PASSWORD")
-
- if err := cfg.UpdateFromEnvs(); err != nil {
- t.Fatal(err)
- }
- err := cfg.ValidateAndSetDefaults()
- assert.NoError(t, err)
-
- if !cfg.AddOnWordpress.Enable {
- t.Fatalf("unexpected cfg.AddOnWordpress.Enable %v", cfg.AddOnWordpress.Enable)
- }
- if cfg.AddOnWordpress.Namespace != "word-press" {
- t.Fatalf("unexpected cfg.AddOnWordpress.Namespace %q", cfg.AddOnWordpress.Namespace)
- }
- if cfg.AddOnWordpress.UserName != "my-user" {
- t.Fatalf("unexpected cfg.AddOnWordpress.UserName %q", cfg.AddOnWordpress.UserName)
- }
- if cfg.AddOnWordpress.Password != "my-password" {
- t.Fatalf("unexpected cfg.AddOnWordpress.Password %q", cfg.AddOnWordpress.Password)
- }
-}
-
-func TestEnvAddOnKubernetesDashboard(t *testing.T) {
- cfg := NewDefault()
- defer func() {
- os.RemoveAll(cfg.ConfigPath)
- os.RemoveAll(cfg.KubectlCommandsOutputPath)
- os.RemoveAll(cfg.RemoteAccessCommandsOutputPath)
- }()
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE", `true`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_METRICS_SERVER_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_METRICS_SERVER_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_KUBERNETES_DASHBOARD_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_KUBERNETES_DASHBOARD_ENABLE")
-
- if err := cfg.UpdateFromEnvs(); err != nil {
- t.Fatal(err)
- }
- err := cfg.ValidateAndSetDefaults()
- assert.NoError(t, err)
-
- if !cfg.AddOnMetricsServer.Enable {
- t.Fatalf("unexpected cfg.AddOnMetricsServer.Enable %v", cfg.AddOnMetricsServer.Enable)
- }
- if !cfg.AddOnKubernetesDashboard.Enable {
- t.Fatalf("unexpected cfg.AddOnKubernetesDashboard.Enable %v", cfg.AddOnKubernetesDashboard.Enable)
- }
- fmt.Println(cfg.KubectlCommands())
-}
-
-func TestEnvAddOnPrometheusGrafana(t *testing.T) {
- cfg := NewDefault()
- defer func() {
- os.RemoveAll(cfg.ConfigPath)
- os.RemoveAll(cfg.KubectlCommandsOutputPath)
- os.RemoveAll(cfg.RemoteAccessCommandsOutputPath)
- }()
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE", `true`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_CSI_EBS_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_CSI_EBS_ENABLE")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_GRAFANA_ADMIN_USER_NAME", "MY_ADMIN_USER_NAME")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_GRAFANA_ADMIN_USER_NAME")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_GRAFANA_ADMIN_PASSWORD", "MY_ADMIN_PASSWORD")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_PROMETHEUS_GRAFANA_GRAFANA_ADMIN_PASSWORD")
-
- if err := cfg.UpdateFromEnvs(); err != nil {
- t.Fatal(err)
- }
- err := cfg.ValidateAndSetDefaults()
- assert.NoError(t, err)
-
- if !cfg.AddOnPrometheusGrafana.Enable {
- t.Fatalf("unexpected cfg.AddOnPrometheusGrafana.Enable %v", cfg.AddOnPrometheusGrafana.Enable)
- }
- if cfg.AddOnPrometheusGrafana.GrafanaAdminUserName != "MY_ADMIN_USER_NAME" {
- t.Fatalf("unexpected cfg.AddOnPrometheusGrafana.GrafanaAdminUserName %q", cfg.AddOnPrometheusGrafana.GrafanaAdminUserName)
- }
- if cfg.AddOnPrometheusGrafana.GrafanaAdminPassword != "MY_ADMIN_PASSWORD" {
- t.Fatalf("unexpected cfg.AddOnPrometheusGrafana.GrafanaAdminPassword %q", cfg.AddOnPrometheusGrafana.GrafanaAdminPassword)
- }
-}
-
-func TestEnvAddOnKubeflow(t *testing.T) {
- cfg := NewDefault()
- defer func() {
- os.RemoveAll(cfg.ConfigPath)
- os.RemoveAll(cfg.KubectlCommandsOutputPath)
- os.RemoveAll(cfg.RemoteAccessCommandsOutputPath)
- }()
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE", `true`)
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE")
-
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_ENABLE", "true")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_ENABLE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_NAMESPACE", "kubeflow")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_NAMESPACE")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_KFCTL_DOWNLOAD_URL", "kubeflow-download-here")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_KFCTL_DOWNLOAD_URL")
- os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_BASE_DIR", "kubeflow-base-dir")
- defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_KUBEFLOW_BASE_DIR")
-
- if err := cfg.UpdateFromEnvs(); err != nil {
- t.Fatal(err)
- }
- err := cfg.ValidateAndSetDefaults()
- assert.NoError(t, err)
-
- if !cfg.AddOnKubeflow.Enable {
- t.Fatalf("unexpected cfg.AddOnKubeflow.Enable %v", cfg.AddOnKubeflow.Enable)
- }
- if cfg.AddOnKubeflow.KfctlDownloadURL != "kubeflow-download-here" {
- t.Fatalf("unexpected cfg.AddOnKubeflow.KfctlDownloadURL %q", cfg.AddOnKubeflow.KfctlDownloadURL)
- }
- if cfg.AddOnKubeflow.BaseDir != "kubeflow-base-dir" {
- t.Fatalf("unexpected cfg.AddOnKubeflow.BaseDir %q", cfg.AddOnKubeflow.BaseDir)
- }
-}
diff --git a/eksconfig/gen/main.go b/eksconfig/gen/main.go
deleted file mode 100644
index aa628a938..000000000
--- a/eksconfig/gen/main.go
+++ /dev/null
@@ -1,266 +0,0 @@
-// gen generates eksconfig documentation.
-package main
-
-import (
- "bytes"
- "fmt"
- "io/ioutil"
- "reflect"
- "strings"
-
- "github.com/aws/aws-k8s-tester/eksconfig"
- "github.com/olekukonko/tablewriter"
-)
-
-func main() {
- doc := createDoc()
- if err := ioutil.WriteFile("eksconfig/README.md", []byte("\n```\n"+doc+"```\n"), 0666); err != nil {
- panic(err)
- }
- fmt.Println("generated")
-}
-
-func createDoc() string {
- es := &enableEnvVars{envs: make([]string, 0)}
- b := strings.Builder{}
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.AWS_K8S_TESTER_EKS_PREFIX, &eksconfig.Config{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.AWS_K8S_TESTER_EKS_S3_PREFIX, &eksconfig.S3{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.AWS_K8S_TESTER_EKS_ENCRYPTION_PREFIX, &eksconfig.Encryption{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.AWS_K8S_TESTER_EKS_ROLE_PREFIX, &eksconfig.Role{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.AWS_K8S_TESTER_EKS_VPC_PREFIX, &eksconfig.VPC{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.AWS_K8S_TESTER_EKS_ADD_ON_CNI_VPC_PREFIX, &eksconfig.AddOnCNIVPC{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_PREFIX, &eksconfig.AddOnNodeGroups{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_PREFIX, &eksconfig.Role{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_PREFIX, &eksconfig.AddOnManagedNodeGroups{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_PREFIX, &eksconfig.Role{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.AWS_K8S_TESTER_EKS_ADD_ON_CW_AGENT_PREFIX, &eksconfig.AddOnCWAgent{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.AWS_K8S_TESTER_EKS_ADD_ON_FLUENTD_PREFIX, &eksconfig.AddOnFluentd{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnMetricsServer, &eksconfig.AddOnMetricsServer{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnConformance, &eksconfig.AddOnConformance{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnAppMesh, &eksconfig.AddOnAppMesh{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnCSIEBS, &eksconfig.AddOnCSIEBS{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnKubernetesDashboard, &eksconfig.AddOnKubernetesDashboard{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnPrometheusGrafana, &eksconfig.AddOnPrometheusGrafana{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnPHPApache, &eksconfig.AddOnPHPApache{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnNLBHelloWorld, &eksconfig.AddOnNLBHelloWorld{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnNLBGuestbook, &eksconfig.AddOnNLBGuestbook{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnALB2048, &eksconfig.AddOnALB2048{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnJobsPi, &eksconfig.AddOnJobsPi{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnJobsEcho, &eksconfig.AddOnJobsEcho{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnCronJobs, &eksconfig.AddOnCronJobs{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnCSRsLocal, &eksconfig.AddOnCSRsLocal{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnCSRsRemote, &eksconfig.AddOnCSRsRemote{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnConfigmapsLocal, &eksconfig.AddOnConfigmapsLocal{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnConfigmapsRemote, &eksconfig.AddOnConfigmapsRemote{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnSecretsLocal, &eksconfig.AddOnSecretsLocal{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnSecretsRemote, &eksconfig.AddOnSecretsRemote{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnFargate, &eksconfig.AddOnFargate{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnIRSA, &eksconfig.AddOnIRSA{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnIRSAFargate, &eksconfig.AddOnIRSAFargate{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnWordpress, &eksconfig.AddOnWordpress{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnJupyterHub, &eksconfig.AddOnJupyterHub{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString("# NOT WORKING...")
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnKubeflow, &eksconfig.AddOnKubeflow{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnCUDAVectorAdd, &eksconfig.AddOnCUDAVectorAdd{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnClusterLoaderLocal, &eksconfig.AddOnClusterLoaderLocal{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnClusterLoaderRemote, &eksconfig.AddOnClusterLoaderRemote{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnStresserLocal, &eksconfig.AddOnStresserLocal{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnStresserRemote, &eksconfig.AddOnStresserRemote{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
- b.WriteString(es.writeDoc(eksconfig.EnvironmentVariablePrefixAddOnClusterVersionUpgrade, &eksconfig.AddOnClusterVersionUpgrade{}))
-
- b.WriteByte('\n')
- b.WriteByte('\n')
-
- txt := b.String()
-
- return fmt.Sprintf("# total %d add-ons\n", len(es.envs)) +
- "# set the following *_ENABLE env vars to enable add-ons, rest are set with default values\n" +
- strings.Join(es.envs, "\n") +
- "\n\n" +
- txt
-}
-
-type enableEnvVars struct {
- envs []string
-}
-
-var columns = []string{
- "environmental variable",
- "read only",
- "type",
- "go type",
-}
-
-func (es *enableEnvVars) writeDoc(pfx string, st interface{}) string {
- buf := bytes.NewBuffer(nil)
- tb := tablewriter.NewWriter(buf)
- tb.SetAutoWrapText(false)
- tb.SetAlignment(tablewriter.ALIGN_LEFT)
- tb.SetColWidth(1500)
- tb.SetCenterSeparator("*")
- tb.SetHeader(columns)
-
- ts := reflect.TypeOf(st)
- tp, vv := reflect.TypeOf(st).Elem(), reflect.ValueOf(st).Elem()
- for i := 0; i < tp.NumField(); i++ {
- jv := tp.Field(i).Tag.Get("json")
- if jv == "" {
- continue
- }
- if vv.Field(i).Type().Kind() == reflect.Ptr {
- continue
- }
-
- readOnly := "false"
- if tp.Field(i).Tag.Get("read-only") == "true" {
- readOnly = "true"
- }
- jv = strings.Replace(jv, ",omitempty", "", -1)
- jv = strings.ToUpper(strings.Replace(jv, "-", "_", -1))
- env := pfx + jv
-
- tb.Append([]string{
- env,
- fmt.Sprintf("read-only %q", readOnly),
- fmt.Sprintf("%s.%s", ts, tp.Field(i).Name),
- fmt.Sprintf("%s", vv.Field(i).Type()),
- })
-
- if strings.HasSuffix(env, "_ENABLE") {
- es.envs = append(es.envs, env+"=true \\")
- }
- }
-
- tb.Render()
- return buf.String()
-}
diff --git a/eksconfig/init.go b/eksconfig/init.go
deleted file mode 100644
index d41bc787b..000000000
--- a/eksconfig/init.go
+++ /dev/null
@@ -1,17 +0,0 @@
-package eksconfig
-
-import (
- "fmt"
-
- "github.com/aws/aws-k8s-tester/ec2config"
- "github.com/aws/aws-sdk-go/service/eks"
-)
-
-func init() {
- if ec2config.AMITypeAL2X8664 != eks.AMITypesAl2X8664 {
- panic(fmt.Errorf("ec2config.AMITypeAL2X8664 %q != eks.AMITypesAl2X8664 %q", ec2config.AMITypeAL2X8664, eks.AMITypesAl2X8664))
- }
- if ec2config.AMITypeAL2X8664GPU != eks.AMITypesAl2X8664Gpu {
- panic(fmt.Errorf("ec2config.AMITypeAL2X8664GPU %q != eks.AMITypesAl2X8664Gpu %q", ec2config.AMITypeAL2X8664GPU, eks.AMITypesAl2X8664Gpu))
- }
-}
diff --git a/eksconfig/metrics-server.go b/eksconfig/metrics-server.go
deleted file mode 100644
index b5f3cadba..000000000
--- a/eksconfig/metrics-server.go
+++ /dev/null
@@ -1,18 +0,0 @@
-package eksconfig
-
-// MetricsServerSpec defines the spec for the Addon
-type MetricsServerSpec struct {
-}
-
-// MetricsServerStatus defines the status for the Addon
-type MetricsServerStatus struct {
- AddonStatus `json:",inline"`
-}
-
-// Validate installs the addon
-func (spec *MetricsServerSpec) Validate(cfg *Config) error {
- return nil
-}
-
-// Default installs the addon
-func (spec *MetricsServerSpec) Default(cfg *Config) {}
diff --git a/eksconfig/overprovisioning.go b/eksconfig/overprovisioning.go
deleted file mode 100644
index 983b7c645..000000000
--- a/eksconfig/overprovisioning.go
+++ /dev/null
@@ -1,47 +0,0 @@
-package eksconfig
-
-import (
- corev1 "k8s.io/api/core/v1"
- "k8s.io/apimachinery/pkg/api/resource"
-)
-
-// OverprovisioningSpec defines the spec for the Addon
-type OverprovisioningSpec struct {
- Namespace string `json:"namespace,omitempty"`
- Image string `json:"image,omitempty"`
- Replicas int `json:"replicas,omitempty"`
- Resources corev1.ResourceRequirements `json:"resources,omitempty"`
- KubemarkEnabled bool `json:"kubemarkEnabled,omitempty"`
-}
-
-// OverprovisioningStatus defines the status for the Addon
-type OverprovisioningStatus struct {
- AddonStatus `json:",inline"`
-}
-
-// Validate installs the addon
-func (spec *OverprovisioningSpec) Validate(cfg *Config) error {
- return nil
-}
-
-// Default installs the addon
-func (spec *OverprovisioningSpec) Default(cfg *Config) {
- // if cfg.AddOnHollowNodesRemote.Enable {
- // spec.KubemarkEnabled = true
- // }
- if spec.Namespace == "" {
- spec.Namespace = "overprovisioning"
- }
- if spec.Image == "" {
- spec.Image = "k8s.gcr.io/pause"
- }
- if spec.Resources.Requests == nil {
- spec.Resources.Requests = corev1.ResourceList{}
- }
- if spec.Resources.Requests.Cpu().IsZero() {
- spec.Resources.Requests[corev1.ResourceCPU] = resource.MustParse("400m")
- }
- if spec.Resources.Requests.Memory().IsZero() {
- spec.Resources.Requests[corev1.ResourceMemory] = resource.MustParse("1Gi")
- }
-}
diff --git a/eksconfig/shared_types.go b/eksconfig/shared_types.go
deleted file mode 100644
index 8812a73bc..000000000
--- a/eksconfig/shared_types.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package eksconfig
-
-// AddonStatus contains shared status fields for all addons
-type AddonStatus struct {
- // Installed is true after the configuration has been applied
- Installed bool `json:"installed"`
- // Ready is true after the addon's health check has passed
- Ready bool `json:"ready"`
-}
diff --git a/eksconfig/spec.go b/eksconfig/spec.go
deleted file mode 100644
index a06fa7642..000000000
--- a/eksconfig/spec.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package eksconfig
-
-// Spec contains specs for addons
-type Spec struct {
- // ClusterAutoscaler defines the addon's spec
- ClusterAutoscaler *ClusterAutoscalerSpec `json:"clusterAutoscaler,omitempty"`
- // Overprovisioning defines the addon's spec
- Overprovisioning *OverprovisioningSpec `json:"overprovisioning,omitempty"`
- // MetricsServer defines the addon's spec
- MetricsServer *MetricsServerSpec `json:"metricsServer,omitempty"`
- // ClusterLoader2 defines the addon's spec
- ClusterLoader *ClusterLoaderSpec `json:"clusterLoader,omitempty"`
-}
diff --git a/eksconfig/status.go b/eksconfig/status.go
deleted file mode 100644
index 746d84557..000000000
--- a/eksconfig/status.go
+++ /dev/null
@@ -1,150 +0,0 @@
-package eksconfig
-
-import (
- "encoding/json"
- "fmt"
- "time"
-
- k8s_client "github.com/aws/aws-k8s-tester/pkg/k8s-client"
- "github.com/aws/aws-k8s-tester/pkg/timeutil"
- aws_eks "github.com/aws/aws-sdk-go/service/eks"
-)
-
-// Status represents the current status of AWS resources.
-// Read-only. Cannot be configured via environmental variables.
-type Status struct {
- // Up is true if the cluster is up.
- // If "Up" is set true, cluster creation is skipped.
- Up bool `json:"up"`
-
- TimeFrameCreate timeutil.TimeFrame `json:"time-frame-create" read-only:"true"`
- TimeFrameDelete timeutil.TimeFrame `json:"time-frame-delete" read-only:"true"`
-
- // ServerVersionInfo is the server version from EKS kube-apiserver.
- ServerVersionInfo k8s_client.ServerVersionInfo `json:"server-version-info" read-only:"true"`
-
- // AWSAccountID is the account ID of the eks tester caller session.
- AWSAccountID string `json:"aws-account-id"`
- // AWSUserID is the user ID of the eks tester caller session.
- AWSUserID string `json:"aws-user-id"`
- // AWSIAMRoleARN is the user IAM Role ARN of the eks tester caller session.
- AWSIAMRoleARN string `json:"aws-iam-role-arn"`
- // AWSCredentialPath is automatically set via AWS SDK Go.
- // And to be mounted as a volume as 'Secret' object.
- AWSCredentialPath string `json:"aws-credential-path"`
-
- ClusterARN string `json:"cluster-arn"`
-
- // ClusterAPIServerEndpoint is the cluster endpoint of the EKS cluster,
- // required for KUBECONFIG write.
- ClusterAPIServerEndpoint string `json:"cluster-api-server-endpoint"`
- // ClusterOIDCIssuerURL is the issuer URL for the OpenID Connect
- // (https://openid.net/connect/) identity provider .
- ClusterOIDCIssuerURL string `json:"cluster-oidc-issuer-url"`
- // ClusterOIDCIssuerHostPath is the issuer host path.
- ClusterOIDCIssuerHostPath string `json:"cluster-oidc-issuer-host-path"`
- // ClusterOIDCIssuerARN is the issuer ARN for the OpenID Connect
- // (https://openid.net/connect/) identity provider .
- ClusterOIDCIssuerARN string `json:"cluster-oidc-issuer-arn"`
- // ClusterOIDCIssuerCAThumbprint is the issuer CA thumbprint.
- ClusterOIDCIssuerCAThumbprint string `json:"cluster-oidc-issuer-ca-thumbprint"`
-
- // ClusterCA is the EKS cluster CA, required for KUBECONFIG write.
- ClusterCA string `json:"cluster-ca"`
- // ClusterCADecoded is the decoded EKS cluster CA, required for k8s.io/client-go.
- ClusterCADecoded string `json:"cluster-ca-decoded"`
-
- // ClusterStatusCurrent represents the current status of the cluster.
- ClusterStatusCurrent string `json:"cluster-status-current"`
- // ClusterStatus represents the status of the cluster.
- ClusterStatus []ClusterStatus `json:"cluster-status"`
-
- // ClusterAutoscaler defines the addon's status
- ClusterAutoscaler *ClusterAutoscalerStatus `json:"clusterAutoscaler,omitempty"`
- // Overprovisioning defines the addon's status
- Overprovisioning *OverprovisioningStatus `json:"overprovisioning,omitempty"`
- // MetricsServer defines the addon's status
- MetricsServer *MetricsServerStatus `json:"metricsServer,omitempty"`
- // ClusterLoader defines the addon's status
- ClusterLoader *ClusterLoaderStatus `json:"clusterLoader,omitempty"`
-
- // PrivateDNSToNodeInfo maps each worker node's private IP to its public IP,
- // public DNS, and SSH access user name.
- // Kubernetes node object name is the node's EC2 instance private DNS.
- // This is used for SSH access.
- PrivateDNSToNodeInfo map[string]NodeInfo `json:"private-dns-to-node-info"`
-
- DeletedResources map[string]string `json:"deleted-resources"`
-}
-
-// NodeInfo represents basic SSH access configuration for worker nodes.
-type NodeInfo struct {
- NodeGroupName string `json:"node-group-name"`
- AMIType string `json:"ami-type"`
- PublicIP string `json:"public-ip"`
- PublicDNSName string `json:"public-dns-name"`
- UserName string `json:"user-name"`
-}
-
-func (sc NodeInfo) ToString() string {
- b, err := json.Marshal(sc)
- if err != nil {
- return fmt.Sprintf("%+v", sc)
- }
- return string(b)
-}
-
-/*
-map all private IPs to public IP + public DNS
-map node name to internal ip, private ip
-pod node name to internal ip ->
-*/
-
-// ClusterStatus represents the cluster status.
-type ClusterStatus struct {
- Time time.Time `json:"time"`
- Status string `json:"status"`
-}
-
-// ClusterStatusDELETEDORNOTEXIST defines the cluster status when the cluster is not found.
-//
-// ref. https://docs.aws.amazon.com/eks/latest/APIReference/API_Cluster.html#AmazonEKS-Type-Cluster-status
-//
-// CREATING
-// ACTIVE
-// UPDATING
-// DELETING
-// FAILED
-//
-const ClusterStatusDELETEDORNOTEXIST = "DELETED/NOT-EXIST"
-
-// RecordStatus records cluster status.
-func (cfg *Config) RecordStatus(status string) {
- cfg.mu.Lock()
- defer cfg.mu.Unlock()
-
- if cfg.Status == nil {
- cfg.Status = &Status{}
- }
- cfg.Status.ClusterStatusCurrent = status
- switch status {
- case ClusterStatusDELETEDORNOTEXIST:
- cfg.Status.Up = false
- case aws_eks.ClusterStatusActive:
- cfg.Status.Up = true
- }
-
- sv := ClusterStatus{Time: time.Now(), Status: status}
- n := len(cfg.Status.ClusterStatus)
- if n == 0 {
- cfg.Status.ClusterStatus = []ClusterStatus{sv}
- cfg.unsafeSync()
- return
- }
-
- copied := make([]ClusterStatus, n+1)
- copy(copied[1:], cfg.Status.ClusterStatus)
- copied[0] = sv
- cfg.Status.ClusterStatus = copied
- cfg.unsafeSync()
-}
diff --git a/go.mod b/go.mod
index 7e5550afb..beed1b499 100644
--- a/go.mod
+++ b/go.mod
@@ -1,271 +1,333 @@
module github.com/aws/aws-k8s-tester
-go 1.21
+go 1.23.2
-toolchain go1.22.1
+toolchain go1.23.4
require (
- github.com/aws/aws-sdk-go v1.43.16
- github.com/aws/aws-sdk-go-v2 v1.18.0
- github.com/aws/aws-sdk-go-v2/config v1.18.23
- github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 // indirect
- github.com/aws/aws-sdk-go-v2/service/autoscaling v1.0.0
- github.com/aws/aws-sdk-go-v2/service/cloudformation v1.0.0
- github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.0.0
- github.com/aws/aws-sdk-go-v2/service/ec2 v1.11.0
- github.com/aws/aws-sdk-go-v2/service/ecr v1.17.20
- github.com/aws/aws-sdk-go-v2/service/eks v1.0.0
- github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.0.0
- github.com/aws/aws-sdk-go-v2/service/iam v1.0.0
- github.com/aws/aws-sdk-go-v2/service/kms v1.0.0
- github.com/aws/aws-sdk-go-v2/service/s3 v1.0.0
- github.com/aws/aws-sdk-go-v2/service/ssm v1.0.0
- github.com/aws/aws-sdk-go-v2/service/sts v1.18.11
- github.com/aws/smithy-go v1.13.5
- github.com/briandowns/spinner v1.11.1
- github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575
- github.com/davecgh/go-spew v1.1.1
- github.com/dustin/go-humanize v1.0.1
- github.com/go-ini/ini v1.55.0
- github.com/gofrs/flock v0.8.1
- github.com/google/go-cmp v0.6.0
- github.com/google/uuid v1.6.0
- github.com/manifoldco/promptui v0.8.0
- github.com/mholt/archiver/v3 v3.3.0
- github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db
- github.com/mitchellh/ioprogress v0.0.0-20180201004757-6a23b12fa88e
- github.com/olekukonko/tablewriter v0.0.5
- github.com/onsi/ginkgo v1.16.5
- github.com/onsi/gomega v1.30.0
- github.com/prometheus/client_golang v1.18.0
- github.com/prometheus/client_model v0.5.0
- github.com/prometheus/common v0.45.0
- github.com/spf13/cobra v1.8.0
+ github.com/aws/aws-sdk-go v1.51.2
+ github.com/aws/aws-sdk-go-v2 v1.32.7
+ github.com/aws/aws-sdk-go-v2/config v1.27.8
+ github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.4
+ github.com/aws/aws-sdk-go-v2/service/cloudformation v1.48.0
+ github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.36.3
+ github.com/aws/aws-sdk-go-v2/service/ec2 v1.151.1
+ github.com/aws/aws-sdk-go-v2/service/eks v1.53.0
+ github.com/aws/aws-sdk-go-v2/service/iam v1.38.3
+ github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1
+ github.com/aws/smithy-go v1.22.1
+ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
+ github.com/octago/sflags v0.2.0
github.com/spf13/pflag v1.0.5
- github.com/stretchr/testify v1.8.4
- // etcd v3.4.9
- go.etcd.io/etcd v3.3.27+incompatible
- go.uber.org/zap v1.27.0
- golang.org/x/crypto v0.21.0
- golang.org/x/oauth2 v0.17.0
- golang.org/x/time v0.3.0
- gopkg.in/yaml.v2 v2.4.0
- helm.sh/helm/v3 v3.14.3
- k8s.io/api v0.29.3
- k8s.io/apiextensions-apiserver v1.24.3
- k8s.io/apimachinery v0.29.3
- k8s.io/cli-runtime v0.29.3
- k8s.io/client-go v1.5.2
+ github.com/stretchr/testify v1.9.0
+ golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
+ k8s.io/api v0.31.3
+ k8s.io/apimachinery v0.31.3
+ k8s.io/client-go v0.31.3
k8s.io/klog v1.0.0
- k8s.io/perf-tests/clusterloader2 v0.0.0-20220805114947-bdcf75fa01d0
- k8s.io/utils v0.0.0-20230726121419-3b25d923346b
- sigs.k8s.io/kubetest2 v0.0.0-20220728001911-c76fb417aa01
- sigs.k8s.io/yaml v1.4.0
+ k8s.io/klog/v2 v2.130.1
+ sigs.k8s.io/controller-runtime v0.19.3
+ sigs.k8s.io/karpenter v1.1.1
+ sigs.k8s.io/kubetest2 v0.0.0-20240309080311-0d7ca9ccb41e
)
require (
- github.com/crowdstrike/falcon-operator v0.9.5
- github.com/octago/sflags v0.2.0
- go.etcd.io/etcd/client/v3 v3.5.10
+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 // indirect
+ github.com/awslabs/operatorpkg v0.0.0-20241205163410-0fff9f28d115 // indirect
+ github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+ github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+ github.com/go-logr/zapr v1.3.0 // indirect
+ github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
+ github.com/pkg/errors v0.9.1
+ github.com/robfig/cron/v3 v3.0.1 // indirect
+ github.com/samber/lo v1.47.0 // indirect
+ github.com/x448/float16 v0.8.4 // indirect
+ golang.org/x/crypto v0.28.0 // indirect
+ gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
+ k8s.io/apiextensions-apiserver v0.31.3 // indirect
)
require (
- cloud.google.com/go/compute v1.25.1 // indirect
- cloud.google.com/go/compute/metadata v0.2.3 // indirect
- github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
- github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
- github.com/BurntSushi/toml v1.3.2 // indirect
- github.com/MakeNowJust/heredoc v1.0.0 // indirect
- github.com/Masterminds/goutils v1.1.1 // indirect
- github.com/Masterminds/semver/v3 v3.2.1 // indirect
- github.com/Masterminds/sprig/v3 v3.2.3 // indirect
- github.com/Masterminds/squirrel v1.5.4 // indirect
- github.com/Microsoft/hcsshim v0.11.4 // indirect
- github.com/andybalholm/brotli v1.0.4 // indirect
+ cloud.google.com/go v0.112.1 // indirect
+ cloud.google.com/go/compute/metadata v0.3.0 // indirect
+ cloud.google.com/go/iam v1.1.7 // indirect
+ cloud.google.com/go/storage v1.39.1 // indirect
+ cuelabs.dev/go/oci/ociregistry v0.0.0-20240318100017-39d12ee67b8b // indirect
+ cuelang.org/go v0.8.0 // indirect
+ dario.cat/mergo v1.0.0 // indirect
+ filippo.io/edwards25519 v1.1.0 // indirect
+ github.com/AliyunContainerService/ack-ram-tool/pkg/credentials/alibabacloudsdkgo/helper v0.2.0 // indirect
+ github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
+ github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+ github.com/Azure/go-autorest/autorest v0.11.29 // indirect
+ github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect
+ github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect
+ github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect
+ github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+ github.com/Azure/go-autorest/logger v0.2.1 // indirect
+ github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+ github.com/MakeNowJust/heredoc/v2 v2.0.1 // indirect
+ github.com/Microsoft/go-winio v0.6.1 // indirect
+ github.com/OneOfOne/xxhash v1.2.8 // indirect
+ github.com/ProtonMail/go-crypto v1.0.0 // indirect
+ github.com/ThalesIgnite/crypto11 v1.2.5 // indirect
+ github.com/agnivade/levenshtein v1.1.1 // indirect
+ github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 // indirect
+ github.com/alibabacloud-go/cr-20160607 v1.0.1 // indirect
+ github.com/alibabacloud-go/cr-20181201 v1.0.10 // indirect
+ github.com/alibabacloud-go/darabonba-openapi v0.2.1 // indirect
+ github.com/alibabacloud-go/debug v1.0.0 // indirect
+ github.com/alibabacloud-go/endpoint-util v1.1.1 // indirect
+ github.com/alibabacloud-go/openapi-util v0.1.0 // indirect
+ github.com/alibabacloud-go/tea v1.2.2 // indirect
+ github.com/alibabacloud-go/tea-utils v1.4.5 // indirect
+ github.com/alibabacloud-go/tea-xml v1.1.3 // indirect
+ github.com/aliyun/credentials-go v1.3.2 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
- github.com/aws/aws-sdk-go-v2/credentials v1.13.22 // indirect
- github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 // indirect
- github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect
- github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 // indirect
- github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.0 // indirect
- github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 // indirect
- github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.0.0 // indirect
- github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 // indirect
- github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect
+ github.com/aws/aws-sdk-go-v2/credentials v1.17.8 // indirect
+ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.4 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+ github.com/aws/aws-sdk-go-v2/service/ecr v1.27.3 // indirect
+ github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.23.3 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 // indirect
+ github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2
+ github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect
+ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect
+ github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240318154307-a1a918375412 // indirect
github.com/beorn7/perks v1.0.1 // indirect
+ github.com/blang/semver v3.5.1+incompatible // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
- github.com/cespare/xxhash/v2 v2.2.0 // indirect
- github.com/chai2010/gettext-go v1.0.2 // indirect
- github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
- github.com/containerd/containerd v1.7.12 // indirect
- github.com/containerd/log v0.1.0 // indirect
- github.com/coreos/go-semver v0.3.1 // indirect
- github.com/coreos/go-systemd/v22 v22.5.0 // indirect
- github.com/crowdstrike/gofalcon v0.4.2 // indirect
+ github.com/buildkite/agent/v3 v3.66.0 // indirect
+ github.com/buildkite/go-pipeline v0.4.1 // indirect
+ github.com/buildkite/interpolate v0.0.0-20200526001904-07f35b4ae251 // indirect
+ github.com/buildkite/roko v1.2.0 // indirect
+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
+ github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect
+ github.com/clbanning/mxj/v2 v2.7.0 // indirect
+ github.com/cloudflare/circl v1.3.7 // indirect
+ github.com/cockroachdb/apd/v3 v3.2.1 // indirect
+ github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect
+ github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
+ github.com/coreos/go-oidc/v3 v3.9.0 // indirect
+ github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
- github.com/docker/cli v24.0.6+incompatible // indirect
- github.com/docker/distribution v2.8.2+incompatible // indirect
- github.com/docker/docker v24.0.7+incompatible // indirect
- github.com/docker/docker-credential-helpers v0.7.0 // indirect
- github.com/docker/go-connections v0.4.0 // indirect
- github.com/docker/go-metrics v0.0.1 // indirect
- github.com/docker/go-units v0.5.0 // indirect
- github.com/dsnet/compress v0.0.1 // indirect
- github.com/emicklei/go-restful/v3 v3.11.0 // indirect
- github.com/evanphx/json-patch v5.7.0+incompatible // indirect
- github.com/evanphx/json-patch/v5 v5.8.0 // indirect
- github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
- github.com/fatih/color v1.13.0 // indirect
+ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+ github.com/digitorus/pkcs7 v0.0.0-20230818184609-3a137a874352 // indirect
+ github.com/digitorus/timestamp v0.0.0-20231217203849-220c5c2851b7 // indirect
+ github.com/dimchansky/utfbom v1.1.1 // indirect
+ github.com/docker/cli v25.0.4+incompatible // indirect
+ github.com/docker/distribution v2.8.3+incompatible // indirect
+ github.com/docker/docker v27.3.1+incompatible // indirect
+ github.com/docker/docker-credential-helpers v0.8.1 // indirect
+ github.com/dustin/go-humanize v1.0.1 // indirect
+ github.com/emicklei/go-restful/v3 v3.12.0 // indirect
+ github.com/emicklei/proto v1.13.2 // indirect
+ github.com/emirpasic/gods v1.18.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
- github.com/go-errors/errors v1.4.2 // indirect
- github.com/go-gorp/gorp/v3 v3.1.0 // indirect
- github.com/go-logr/logr v1.4.1 // indirect
+ github.com/glebarez/go-sqlite v1.22.0 // indirect
+ github.com/go-chi/chi v4.1.2+incompatible // indirect
+ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+ github.com/go-git/go-billy/v5 v5.5.0 // indirect
+ github.com/go-git/go-git/v5 v5.11.0 // indirect
+ github.com/go-ini/ini v1.67.0 // indirect
+ github.com/go-jose/go-jose/v3 v3.0.3 // indirect
+ github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
- github.com/go-openapi/analysis v0.21.4 // indirect
- github.com/go-openapi/errors v0.20.4 // indirect
- github.com/go-openapi/jsonpointer v0.19.6 // indirect
- github.com/go-openapi/jsonreference v0.20.2 // indirect
- github.com/go-openapi/loads v0.21.2 // indirect
- github.com/go-openapi/runtime v0.26.0 // indirect
- github.com/go-openapi/spec v0.20.9 // indirect
- github.com/go-openapi/strfmt v0.21.7 // indirect
- github.com/go-openapi/swag v0.22.4 // indirect
- github.com/go-openapi/validate v0.22.1 // indirect
+ github.com/go-ole/go-ole v1.3.0 // indirect
+ github.com/go-openapi/analysis v0.23.0 // indirect
+ github.com/go-openapi/errors v0.22.0 // indirect
+ github.com/go-openapi/jsonpointer v0.21.0 // indirect
+ github.com/go-openapi/jsonreference v0.21.0 // indirect
+ github.com/go-openapi/loads v0.22.0 // indirect
+ github.com/go-openapi/runtime v0.28.0 // indirect
+ github.com/go-openapi/spec v0.21.0 // indirect
+ github.com/go-openapi/strfmt v0.23.0 // indirect
+ github.com/go-openapi/swag v0.23.0 // indirect
+ github.com/go-openapi/validate v0.24.0 // indirect
+ github.com/go-piv/piv-go v1.11.0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
+ github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
- github.com/google/btree v1.1.2 // indirect
- github.com/google/gnostic-models v0.6.8 // indirect
- github.com/google/gofuzz v1.2.0 // indirect
- github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
- github.com/gorilla/mux v1.8.0 // indirect
- github.com/gorilla/websocket v1.5.0 // indirect
- github.com/gosuri/uitable v0.0.4 // indirect
- github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
- github.com/hashicorp/errwrap v1.1.0 // indirect
- github.com/hashicorp/go-multierror v1.1.1 // indirect
- github.com/huandu/xstrings v1.4.0 // indirect
- github.com/imdario/mergo v0.3.15 // indirect
+ github.com/google/certificate-transparency-go v1.1.8 // indirect
+ github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
+ github.com/google/go-cmp v0.6.0 // indirect
+ github.com/google/go-containerregistry v0.19.1 // indirect
+ github.com/google/go-github/v55 v55.0.0 // indirect
+ github.com/google/go-github/v58 v58.0.0 // indirect
+ github.com/google/go-querystring v1.1.0 // indirect
+ github.com/google/gofuzz v1.2.1-0.20210504230335-f78f29fc09ea // indirect
+ github.com/google/licenseclassifier/v2 v2.0.0 // indirect
+ github.com/google/s2a-go v0.1.7 // indirect
+ github.com/google/uuid v1.6.0 // indirect
+ github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
+ github.com/googleapis/gax-go/v2 v2.12.3 // indirect
+ github.com/gorilla/mux v1.8.1 // indirect
+ github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+ github.com/hashicorp/go-retryablehttp v0.7.5 // indirect
+ github.com/hashicorp/hcl v1.0.1-vault-5 // indirect
+ github.com/imdario/mergo v0.3.16 // indirect
+ github.com/in-toto/in-toto-golang v0.9.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
+ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+ github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 // indirect
+ github.com/jellydator/ttlcache/v3 v3.2.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
- github.com/jmoiron/sqlx v1.3.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
- github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a // indirect
- github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
- github.com/klauspost/compress v1.16.3 // indirect
- github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect
- github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
- github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
- github.com/lib/pq v1.10.9 // indirect
- github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
- github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a // indirect
+ github.com/kevinburke/ssh_config v1.2.0 // indirect
+ github.com/klauspost/compress v1.17.9 // indirect
+ github.com/knqyf263/go-rpmdb v0.1.0 // indirect
+ github.com/letsencrypt/boulder v0.0.0-20240318162201-5e68cbe552b9 // indirect
+ github.com/magiconair/properties v1.8.7 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
- github.com/mattn/go-colorable v0.1.13 // indirect
- github.com/mattn/go-isatty v0.0.17 // indirect
- github.com/mattn/go-runewidth v0.0.14 // indirect
- github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
- github.com/mitchellh/copystructure v1.2.0 // indirect
+ github.com/mattn/go-isatty v0.0.20 // indirect
+ github.com/miekg/pkcs11 v1.1.1 // indirect
+ github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
- github.com/mitchellh/reflectwalk v1.0.2 // indirect
- github.com/moby/locker v1.0.1 // indirect
- github.com/moby/spdystream v0.2.0 // indirect
- github.com/moby/term v0.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
- github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
- github.com/morikuni/aec v1.0.0 // indirect
+ github.com/mozillazg/docker-credential-acr-helper v0.3.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
- github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
- github.com/nwaples/rardecode v1.0.0 // indirect
- github.com/nxadm/tail v1.4.8 // indirect
+ github.com/ncruces/go-strftime v0.1.9 // indirect
+ github.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481 // indirect
github.com/oklog/ulid v1.3.1 // indirect
+ github.com/oleiade/reflections v1.0.1 // indirect
+ github.com/open-policy-agent/opa v0.62.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
- github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
+ github.com/opencontainers/image-spec v1.1.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
- github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
- github.com/pierrec/lz4 v2.0.5+incompatible // indirect
- github.com/pkg/errors v0.9.1 // indirect
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/prometheus/procfs v0.12.0 // indirect
- github.com/rivo/uniseg v0.4.4 // indirect
- github.com/rubenv/sql-migrate v1.5.2 // indirect
- github.com/russross/blackfriday/v2 v2.1.0 // indirect
- github.com/shopspring/decimal v1.3.1 // indirect
+ github.com/package-url/packageurl-go v0.1.2 // indirect
+ github.com/pborman/uuid v1.2.1 // indirect
+ github.com/pelletier/go-toml/v2 v2.1.1 // indirect
+ github.com/pjbgf/sha1cd v0.3.0 // indirect
+ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
+ github.com/prometheus/client_golang v1.20.5 // indirect
+ github.com/prometheus/client_model v0.6.1 // indirect
+ github.com/prometheus/common v0.55.0 // indirect
+ github.com/prometheus/procfs v0.15.1 // indirect
+ github.com/protocolbuffers/txtpbfmt v0.0.0-20240116145035-ef3ab179eed6 // indirect
+ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
+ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+ github.com/rogpeppe/go-internal v1.12.0 // indirect
+ github.com/sagikazarmark/locafero v0.4.0 // indirect
+ github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+ github.com/sassoftware/relic v7.2.1+incompatible // indirect
+ github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect
+ github.com/segmentio/ksuid v1.0.4 // indirect
+ github.com/sergi/go-diff v1.3.1 // indirect
+ github.com/shibumi/go-pathspec v1.3.0 // indirect
+ github.com/shirou/gopsutil/v3 v3.24.2 // indirect
+ github.com/sigstore/cosign/v2 v2.2.3 // indirect
+ github.com/sigstore/fulcio v1.4.4 // indirect
+ github.com/sigstore/rekor v1.3.5 // indirect
+ github.com/sigstore/sigstore v1.8.2 // indirect
+ github.com/sigstore/timestamp-authority v1.2.2 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
- github.com/spf13/cast v1.5.0 // indirect
- github.com/ulikunitz/xz v0.5.11 // indirect
+ github.com/skeema/knownhosts v1.2.2 // indirect
+ github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
+ github.com/sourcegraph/conc v0.3.0 // indirect
+ github.com/spf13/afero v1.11.0 // indirect
+ github.com/spf13/cast v1.6.0 // indirect
+ github.com/spf13/cobra v1.8.1 // indirect
+ github.com/spf13/viper v1.18.2 // indirect
+ github.com/spiffe/go-spiffe/v2 v2.1.7 // indirect
+ github.com/subosito/gotenv v1.6.0 // indirect
+ github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
+ github.com/tchap/go-patricia/v2 v2.3.1 // indirect
+ github.com/thales-e-security/pool v0.0.2 // indirect
+ github.com/theupdateframework/go-tuf v0.7.0 // indirect
+ github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect
+ github.com/tjfoc/gmsm v1.4.1 // indirect
+ github.com/transparency-dev/merkle v0.0.2 // indirect
+ github.com/vbatts/tar-split v0.11.5 // indirect
+ github.com/xanzy/go-gitlab v0.100.0 // indirect
+ github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
- github.com/xeipuuv/gojsonschema v1.2.0 // indirect
- github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
- github.com/xlab/treeprint v1.2.0 // indirect
- go.etcd.io/etcd/api/v3 v3.5.10 // indirect
- go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect
- go.mongodb.org/mongo-driver v1.11.3 // indirect
- go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
- go.opentelemetry.io/otel v1.24.0 // indirect
- go.opentelemetry.io/otel/metric v1.24.0 // indirect
- go.opentelemetry.io/otel/trace v1.24.0 // indirect
- go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
+ github.com/yashtewari/glob-intersection v0.2.0 // indirect
+ github.com/yusufpapurcu/wmi v1.2.4 // indirect
+ github.com/zeebo/errs v1.3.0 // indirect
+ gitlab.alpinelinux.org/alpine/go v0.10.0 // indirect
+ go.mongodb.org/mongo-driver v1.14.0 // indirect
+ go.opencensus.io v0.24.0 // indirect
+ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect
+ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
+ go.opentelemetry.io/otel v1.28.0 // indirect
+ go.opentelemetry.io/otel/metric v1.28.0 // indirect
+ go.opentelemetry.io/otel/sdk v1.28.0 // indirect
+ go.opentelemetry.io/otel/trace v1.28.0 // indirect
+ go.step.sm/crypto v0.43.1 // indirect
go.uber.org/multierr v1.11.0 // indirect
- golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
- golang.org/x/net v0.22.0 // indirect
- golang.org/x/sync v0.6.0 // indirect
- golang.org/x/sys v0.18.0 // indirect
- golang.org/x/term v0.18.0 // indirect
- golang.org/x/text v0.14.0 // indirect
- gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
- google.golang.org/appengine v1.6.8 // indirect
- google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect
- google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
- google.golang.org/grpc v1.62.1 // indirect
- google.golang.org/protobuf v1.33.0 // indirect
+ go.uber.org/zap v1.27.0 // indirect
+ golang.org/x/mod v0.21.0 // indirect
+ golang.org/x/net v0.30.0 // indirect
+ golang.org/x/oauth2 v0.21.0 // indirect
+ golang.org/x/sync v0.9.0 // indirect
+ golang.org/x/sys v0.26.0 // indirect
+ golang.org/x/term v0.25.0 // indirect
+ golang.org/x/text v0.20.0 // indirect
+ golang.org/x/time v0.8.0 // indirect
+ golang.org/x/tools v0.26.0 // indirect
+ golang.org/x/tools/go/vcs v0.1.0-deprecated // indirect
+ golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+ google.golang.org/api v0.170.0 // indirect
+ google.golang.org/genproto v0.0.0-20240318140521-94a12d6c2237 // indirect
+ google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
+ google.golang.org/grpc v1.65.0 // indirect
+ google.golang.org/protobuf v1.35.1 // indirect
+ gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
- gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+ gopkg.in/ini.v1 v1.67.0 // indirect
+ gopkg.in/warnings.v0 v0.1.2 // indirect
+ gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
- k8s.io/apiserver v0.29.3 // indirect
- k8s.io/component-base v0.29.3 // indirect
- k8s.io/klog/v2 v2.110.1 // indirect
- k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
- k8s.io/kubectl v0.29.0 // indirect
- k8s.io/kubernetes v1.24.3 // indirect
- oras.land/oras-go v1.2.4 // indirect
- sigs.k8s.io/controller-runtime v0.17.2 // indirect
+ k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+ k8s.io/release v0.16.5 // indirect
+ k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
+ modernc.org/libc v1.45.2 // indirect
+ modernc.org/mathutil v1.6.0 // indirect
+ modernc.org/memory v1.7.2 // indirect
+ modernc.org/sqlite v1.29.5 // indirect
+ sigs.k8s.io/bom v0.6.0 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
- sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
- sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
+ sigs.k8s.io/promo-tools/v3 v3.6.0 // indirect
+ sigs.k8s.io/release-sdk v0.11.0 // indirect
+ sigs.k8s.io/release-utils v0.7.7 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+ sigs.k8s.io/yaml v1.4.0
+)
+
+require (
+ github.com/kubeflow/mpi-operator v0.4.0
+ k8s.io/cli-runtime v0.28.3
+ sigs.k8s.io/e2e-framework v0.3.0
)
-replace (
- k8s.io/api => k8s.io/api v0.29.3
- k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.29.3
- k8s.io/apimachinery => k8s.io/apimachinery v0.29.3
- k8s.io/apiserver => k8s.io/apiserver v0.29.3
- k8s.io/cli-runtime => k8s.io/cli-runtime v0.29.3
- k8s.io/client-go => k8s.io/client-go v0.29.3
- k8s.io/cloud-provider => k8s.io/cloud-provider v0.29.3
- k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.29.3
- k8s.io/code-generator => k8s.io/code-generator v0.29.3
- k8s.io/component-base => k8s.io/component-base v0.29.3
- k8s.io/component-helpers => k8s.io/component-helpers v0.29.3
- k8s.io/controller-manager => k8s.io/controller-manager v0.29.3
- k8s.io/cri-api => k8s.io/cri-api v0.29.3
- k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.29.3
- k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.29.3
- k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.29.3
- k8s.io/kube-proxy => k8s.io/kube-proxy v0.29.3
- k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.29.3
- k8s.io/kubectl => k8s.io/kubectl v0.29.3
- k8s.io/kubelet => k8s.io/kubelet v0.29.3
- k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.29.3
- k8s.io/metrics => k8s.io/metrics v0.29.3
- k8s.io/mount-utils => k8s.io/mount-utils v0.29.3
- k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.29.3
- k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.3
+require (
+ github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+ github.com/go-errors/errors v1.4.2 // indirect
+ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+ github.com/gorilla/websocket v1.5.0 // indirect
+ github.com/kubeflow/common v0.4.6 // indirect
+ github.com/moby/spdystream v0.4.0 // indirect
+ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+ github.com/xlab/treeprint v1.2.0 // indirect
+ go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
+ sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
+ sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
)
diff --git a/go.sum b/go.sum
index 252722845..610d5cceb 100644
--- a/go.sum
+++ b/go.sum
@@ -1,1488 +1,399 @@
-bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
-bitbucket.org/bertimus9/systemstat v0.0.0-20180207000608-0eeff89b0690/go.mod h1:Ulb78X89vxKYgdL24HMTiXYHlyHEvruOj1ZPlqeNEZM=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
-cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
-cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
-cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
-cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY=
-cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
-cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
-cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
-cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY=
-cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM=
-cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY=
-cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ=
-cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
-cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4=
-cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc=
-cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA=
-cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U=
-cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A=
-cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc=
-cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU=
-cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA=
-cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM=
-cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I=
-cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY=
-cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw=
-cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI=
-cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI=
-cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4=
-cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw=
-cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E=
-cloud.google.com/go/accessapproval v1.7.1/go.mod h1:JYczztsHRMK7NTXb6Xw+dwbs/WnOJxbo/2mTI+Kgg68=
-cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o=
-cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE=
-cloud.google.com/go/accesscontextmanager v1.6.0/go.mod h1:8XCvZWfYw3K/ji0iVnp+6pu7huxoQTLmxAbVjbloTtM=
-cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ=
-cloud.google.com/go/accesscontextmanager v1.8.0/go.mod h1:uI+AI/r1oyWK99NN8cQ3UK76AMelMzgZCvJfsi2c+ps=
-cloud.google.com/go/accesscontextmanager v1.8.1/go.mod h1:JFJHfvuaTC+++1iL1coPiG1eu5D24db2wXCDWDjIrxo=
-cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw=
-cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY=
-cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg=
-cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6lKv3w0+tFTgXQ=
-cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k=
-cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw=
-cloud.google.com/go/aiplatform v1.45.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA=
-cloud.google.com/go/aiplatform v1.48.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA=
-cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI=
-cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4=
-cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M=
-cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE=
-cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE=
-cloud.google.com/go/analytics v0.21.2/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo=
-cloud.google.com/go/analytics v0.21.3/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo=
-cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk=
-cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc=
-cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8=
-cloud.google.com/go/apigateway v1.6.1/go.mod h1:ufAS3wpbRjqfZrzpvLC2oh0MFlpRJm2E/ts25yyqmXA=
-cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc=
-cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04=
-cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8=
-cloud.google.com/go/apigeeconnect v1.6.1/go.mod h1:C4awq7x0JpLtrlQCr8AzVIzAaYgngRqWf9S5Uhg+wWs=
-cloud.google.com/go/apigeeregistry v0.4.0/go.mod h1:EUG4PGcsZvxOXAdyEghIdXwAEi/4MEaoqLMLDMIwKXY=
-cloud.google.com/go/apigeeregistry v0.5.0/go.mod h1:YR5+s0BVNZfVOUkMa5pAR2xGd0A473vA5M7j247o1wM=
-cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc=
-cloud.google.com/go/apigeeregistry v0.7.1/go.mod h1:1XgyjZye4Mqtw7T9TsY4NW10U7BojBvG4RMD+vRDrIw=
-cloud.google.com/go/apikeys v0.4.0/go.mod h1:XATS/yqZbaBK0HOssf+ALHp8jAlNHUgyfprvNcBIszU=
-cloud.google.com/go/apikeys v0.5.0/go.mod h1:5aQfwY4D+ewMMWScd3hm2en3hCj+BROlyrt3ytS7KLI=
-cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8=
-cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno=
-cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak=
-cloud.google.com/go/appengine v1.6.0/go.mod h1:hg6i0J/BD2cKmDJbaFSYHFyZkgBEfQrDg/X0V5fJn84=
-cloud.google.com/go/appengine v1.7.0/go.mod h1:eZqpbHFCqRGa2aCdope7eC0SWLV1j0neb/QnMJVWx6A=
-cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E=
-cloud.google.com/go/appengine v1.8.1/go.mod h1:6NJXGLVhZCN9aQ/AEDvmfzKEfoYBlfB80/BHiKVputY=
-cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4=
-cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0=
-cloud.google.com/go/area120 v0.7.0/go.mod h1:a3+8EUD1SX5RUcCs3MY5YasiO1z6yLiNLRiFrykbynY=
-cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k=
-cloud.google.com/go/area120 v0.8.1/go.mod h1:BVfZpGpB7KFVNxPiQBuHkX6Ed0rS51xIgmGyjrAfzsg=
-cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ=
-cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk=
-cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0=
-cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc=
-cloud.google.com/go/artifactregistry v1.11.1/go.mod h1:lLYghw+Itq9SONbCa1YWBoWs1nOucMH0pwXN1rOBZFI=
-cloud.google.com/go/artifactregistry v1.11.2/go.mod h1:nLZns771ZGAwVLzTX/7Al6R9ehma4WUEhZGWV6CeQNQ=
-cloud.google.com/go/artifactregistry v1.12.0/go.mod h1:o6P3MIvtzTOnmvGagO9v/rOjjA0HmhJ+/6KAXrmYDCI=
-cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08=
-cloud.google.com/go/artifactregistry v1.14.1/go.mod h1:nxVdG19jTaSTu7yA7+VbWL346r3rIdkZ142BSQqhn5E=
-cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o=
-cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s=
-cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0=
-cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ=
-cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY=
-cloud.google.com/go/asset v1.11.1/go.mod h1:fSwLhbRvC9p9CXQHJ3BgFeQNM4c9x10lqlrdEUYXlJo=
-cloud.google.com/go/asset v1.12.0/go.mod h1:h9/sFOa4eDIyKmH6QMpm4eUK3pDojWnUhTgJlk762Hg=
-cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw=
-cloud.google.com/go/asset v1.14.1/go.mod h1:4bEJ3dnHCqWCDbWJ/6Vn7GVI9LerSi7Rfdi03hd+WTQ=
-cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY=
-cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw=
-cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI=
-cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo=
-cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0=
-cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E=
-cloud.google.com/go/assuredworkloads v1.11.1/go.mod h1:+F04I52Pgn5nmPG36CWFtxmav6+7Q+c5QyJoL18Lry0=
-cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0=
-cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8=
-cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8=
-cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM=
-cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU=
-cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3zK4bheQE=
-cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc=
-cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI=
-cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss=
-cloud.google.com/go/baremetalsolution v1.1.1/go.mod h1:D1AV6xwOksJMV4OSlWHtWuFNZZYujJknMAP4Qa27QIA=
-cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE=
-cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE=
-cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g=
-cloud.google.com/go/batch v1.3.1/go.mod h1:VguXeQKXIYaeeIYbuozUmBR13AfL4SJP7IltNPS+A4A=
-cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4=
-cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8=
-cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM=
-cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU=
-cloud.google.com/go/beyondcorp v0.6.1/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4=
-cloud.google.com/go/beyondcorp v1.0.0/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
-cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
-cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA=
-cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw=
-cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc=
-cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/ZurWFIxmF9I/E=
-cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac=
-cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q=
-cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU=
-cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4=
-cloud.google.com/go/bigquery v1.53.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4=
-cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o=
-cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY=
-cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s=
-cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI=
-cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y=
-cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss=
-cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc=
-cloud.google.com/go/billing v1.16.0/go.mod h1:y8vx09JSSJG02k5QxbycNRrN7FGZB6F3CAcgum7jvGA=
-cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM=
-cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI=
-cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0=
-cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk=
-cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q=
-cloud.google.com/go/binaryauthorization v1.6.1/go.mod h1:TKt4pa8xhowwffiBmbrbcxijJRZED4zrqnwZ1lKH51U=
-cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg=
-cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590=
-cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8=
-cloud.google.com/go/certificatemanager v1.7.1/go.mod h1:iW8J3nG6SaRYImIa+wXQ0g8IgoofDFRp5UMzaNk1UqI=
-cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk=
-cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk=
-cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE=
-cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU=
-cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeHCJNf/oHmc=
-cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U=
-cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA=
-cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M=
-cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg=
-cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s=
-cloud.google.com/go/cloudbuild v1.10.1/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU=
-cloud.google.com/go/cloudbuild v1.13.0/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU=
-cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM=
-cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk=
-cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA=
-cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI=
-cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY=
-cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI=
-cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4=
-cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI=
-cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y=
-cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs=
-cloud.google.com/go/cloudtasks v1.11.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM=
-cloud.google.com/go/cloudtasks v1.12.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM=
-cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow=
-cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM=
-cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M=
-cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s=
-cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU=
-cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U=
-cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU=
-cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU=
-cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU=
-cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE=
-cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo=
-cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA=
-cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs=
-cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU=
-cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE=
-cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI=
-cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
-cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
-cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
-cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU=
-cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls=
-cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU=
-cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
-cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
-cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
-cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
-cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY=
-cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck=
-cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w=
-cloud.google.com/go/contactcenterinsights v1.9.1/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM=
-cloud.google.com/go/contactcenterinsights v1.10.0/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM=
-cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg=
-cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo=
-cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4=
-cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM=
-cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA=
-cloud.google.com/go/container v1.22.1/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4=
-cloud.google.com/go/container v1.24.0/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4=
-cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I=
-cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4=
-cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI=
-cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s=
-cloud.google.com/go/containeranalysis v0.10.1/go.mod h1:Ya2jiILITMY68ZLPaogjmOMNkwsDrWBSTyBubGXO7j0=
-cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0=
-cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs=
-cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc=
-cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE=
-cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM=
-cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M=
-cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0=
-cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8=
-cloud.google.com/go/datacatalog v1.14.0/go.mod h1:h0PrGtlihoutNMp/uvwhawLQ9+c63Kz65UFqh49Yo+E=
-cloud.google.com/go/datacatalog v1.14.1/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4=
-cloud.google.com/go/datacatalog v1.16.0/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4=
-cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM=
-cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ=
-cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE=
-cloud.google.com/go/dataflow v0.9.1/go.mod h1:Wp7s32QjYuQDWqJPFFlnBKhkAtiFpMTdg00qGbnIHVw=
-cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo=
-cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE=
-cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0=
-cloud.google.com/go/dataform v0.6.0/go.mod h1:QPflImQy33e29VuapFdf19oPbE4aYTJxr31OAPV+ulA=
-cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE=
-cloud.google.com/go/dataform v0.8.1/go.mod h1:3BhPSiw8xmppbgzeBbmDvmSWlwouuJkXsXsb8UBih9M=
-cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38=
-cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w=
-cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8=
-cloud.google.com/go/datafusion v1.7.1/go.mod h1:KpoTBbFmoToDExJUso/fcCiguGDk7MEzOWXUsJo0wsI=
-cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I=
-cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ=
-cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM=
-cloud.google.com/go/datalabeling v0.8.1/go.mod h1:XS62LBSVPbYR54GfYQsPXZjTW8UxCK2fkDciSrpRFdY=
-cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA=
-cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A=
-cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ=
-cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs=
-cloud.google.com/go/dataplex v1.8.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE=
-cloud.google.com/go/dataplex v1.9.0/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE=
-cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s=
-cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI=
-cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4=
-cloud.google.com/go/dataproc/v2 v2.0.1/go.mod h1:7Ez3KRHdFGcfY7GcevBbvozX+zyWGcwLJvvAMwCaoZ4=
-cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo=
-cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA=
-cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c=
-cloud.google.com/go/dataqna v0.8.1/go.mod h1:zxZM0Bl6liMePWsHA8RMGAfmTG34vJMapbHAxQ5+WA8=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM=
-cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c=
-cloud.google.com/go/datastore v1.12.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70=
-cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70=
-cloud.google.com/go/datastore v1.13.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70=
-cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo=
-cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ=
-cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g=
-cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4=
-cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs=
-cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww=
-cloud.google.com/go/datastream v1.9.1/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q=
-cloud.google.com/go/datastream v1.10.0/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q=
-cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c=
-cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s=
-cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI=
-cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ=
-cloud.google.com/go/deploy v1.11.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g=
-cloud.google.com/go/deploy v1.13.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g=
-cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4=
-cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0=
-cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8=
-cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek=
-cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0=
-cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHihAdRFMtn2WXuM=
-cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4=
-cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE=
-cloud.google.com/go/dialogflow v1.38.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4=
-cloud.google.com/go/dialogflow v1.40.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4=
-cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM=
-cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q=
-cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4=
-cloud.google.com/go/dlp v1.10.1/go.mod h1:IM8BWz1iJd8njcNcG0+Kyd9OPnqnRNkDV8j42VT5KOI=
-cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU=
-cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU=
-cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k=
-cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4=
-cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM=
-cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs=
-cloud.google.com/go/documentai v1.20.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E=
-cloud.google.com/go/documentai v1.22.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E=
-cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y=
-cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg=
-cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE=
-cloud.google.com/go/domains v0.9.1/go.mod h1:aOp1c0MbejQQ2Pjf1iJvnVyT+z6R6s8pX66KaCSDYfE=
-cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk=
-cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w=
-cloud.google.com/go/edgecontainer v0.3.0/go.mod h1:FLDpP4nykgwwIfcLt6zInhprzw0lEi2P1fjO6Ie0qbc=
-cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY=
-cloud.google.com/go/edgecontainer v1.1.1/go.mod h1:O5bYcS//7MELQZs3+7mabRqoWQhXCzenBu0R8bz2rwk=
-cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU=
-cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI=
-cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8=
-cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M=
-cloud.google.com/go/essentialcontacts v1.6.2/go.mod h1:T2tB6tX+TRak7i88Fb2N9Ok3PvY3UNbUsMag9/BARh4=
-cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc=
-cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw=
-cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw=
-cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY=
-cloud.google.com/go/eventarc v1.12.1/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI=
-cloud.google.com/go/eventarc v1.13.0/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI=
-cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w=
-cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI=
-cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs=
-cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg=
-cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4=
-cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
-cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE=
-cloud.google.com/go/firestore v1.11.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4=
-cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk=
-cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg=
-cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY=
-cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08=
-cloud.google.com/go/functions v1.10.0/go.mod h1:0D3hEOe3DbEvCXtYOZHQZmD+SzYsi1YbI7dGvHfldXw=
-cloud.google.com/go/functions v1.12.0/go.mod h1:AXWGrF3e2C/5ehvwYo/GH6O5s09tOPksiKhz+hH8WkA=
-cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c=
-cloud.google.com/go/functions v1.15.1/go.mod h1:P5yNWUTkyU+LvW/S9O6V+V423VZooALQlqoXdoPz5AE=
-cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM=
-cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA=
-cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w=
-cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM=
-cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0=
-cloud.google.com/go/gaming v1.10.1/go.mod h1:XQQvtfP8Rb9Rxnxm5wFVpAp9zCQkJi2bLIb7iHGwB3s=
-cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60=
-cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo=
-cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg=
-cloud.google.com/go/gkebackup v1.3.0/go.mod h1:vUDOu++N0U5qs4IhG1pcOnD1Mac79xWy6GoBFlWCWBU=
-cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o=
-cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A=
-cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw=
-cloud.google.com/go/gkeconnect v0.8.1/go.mod h1:KWiK1g9sDLZqhxB2xEuPV8V9NYzrqTUmQR9shJHpOZw=
-cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0=
-cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0=
-cloud.google.com/go/gkehub v0.11.0/go.mod h1:JOWHlmN+GHyIbuWQPl47/C2RFhnFKH38jH9Ascu3n0E=
-cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw=
-cloud.google.com/go/gkehub v0.14.1/go.mod h1:VEXKIJZ2avzrbd7u+zeMtW00Y8ddk/4V9511C9CQGTY=
-cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA=
-cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI=
-cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y=
-cloud.google.com/go/gkemulticloud v0.6.1/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw=
-cloud.google.com/go/gkemulticloud v1.0.0/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw=
-cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc=
-cloud.google.com/go/grafeas v0.3.0/go.mod h1:P7hgN24EyONOTMyeJH6DxG4zD7fwiYa5Q6GUgyFSOU8=
-cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM=
-cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o=
-cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo=
-cloud.google.com/go/gsuiteaddons v1.6.1/go.mod h1:CodrdOqRZcLp5WOwejHWYBjZvfY0kOphkAKpF/3qdZY=
-cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c=
-cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY=
-cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc=
-cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc=
-cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg=
-cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE=
-cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY=
-cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY=
-cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0=
-cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBalRE8=
-cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk=
-cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU=
-cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc=
-cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A=
-cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk=
-cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo=
-cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74=
-cloud.google.com/go/iap v1.8.1/go.mod h1:sJCbeqg3mvWLqjZNsI6dfAtbbV1DL2Rl7e1mTyXYREQ=
-cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM=
-cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY=
-cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4=
-cloud.google.com/go/ids v1.4.1/go.mod h1:np41ed8YMU8zOgv53MMMoCntLTn2lF+SUzlM+O3u/jw=
-cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs=
-cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g=
-cloud.google.com/go/iot v1.5.0/go.mod h1:mpz5259PDl3XJthEmh9+ap0affn/MqNSP4My77Qql9o=
-cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE=
-cloud.google.com/go/iot v1.7.1/go.mod h1:46Mgw7ev1k9KqK1ao0ayW9h0lI+3hxeanz+L1zmbbbk=
-cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA=
-cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
-cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
-cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg=
-cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
-cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24=
-cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI=
-cloud.google.com/go/kms v1.11.0/go.mod h1:hwdiYC0xjnWsKQQCQQmIQnS9asjYVSK6jtXm+zFqXLM=
-cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM=
-cloud.google.com/go/kms v1.15.0/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM=
-cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
-cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
-cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
-cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8=
-cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY=
-cloud.google.com/go/language v1.10.1/go.mod h1:CPp94nsdVNiQEt1CNjF5WkTcisLiHPyIbMhvR8H2AW0=
-cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8=
-cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08=
-cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo=
-cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc=
-cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw=
-cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
-cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
-cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
-cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
-cloud.google.com/go/longrunning v0.4.2/go.mod h1:OHrnaYyLUV6oqwh0xiS7e5sLQhP1m0QU9R+WhGDMgIQ=
-cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc=
-cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc=
-cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE=
-cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM=
-cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA=
-cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHnypMbu4RB3yl8YcuEak=
-cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI=
-cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw=
-cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY=
-cloud.google.com/go/maps v1.4.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s=
-cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4=
-cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w=
-cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I=
-cloud.google.com/go/mediatranslation v0.8.1/go.mod h1:L/7hBdEYbYHQJhX2sldtTO5SZZ1C1vkapubj0T2aGig=
-cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE=
-cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM=
-cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA=
-cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY=
-cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM=
-cloud.google.com/go/memcache v1.10.1/go.mod h1:47YRQIarv4I3QS5+hoETgKO40InqzLP6kpNLvyXuyaA=
-cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY=
-cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s=
-cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8=
-cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI=
-cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo=
-cloud.google.com/go/metastore v1.11.1/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA=
-cloud.google.com/go/metastore v1.12.0/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA=
-cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk=
-cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4=
-cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w=
-cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw=
-cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM=
-cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA=
-cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o=
-cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM=
-cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8=
-cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E=
-cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM=
-cloud.google.com/go/networkconnectivity v1.12.1/go.mod h1:PelxSWYM7Sh9/guf8CFhi6vIqf19Ir/sbfZRUwXh92E=
-cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8=
-cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4=
-cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY=
-cloud.google.com/go/networkmanagement v1.8.0/go.mod h1:Ho/BUGmtyEqrttTgWEe7m+8vDdK74ibQc+Be0q7Fof0=
-cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ=
-cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU=
-cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k=
-cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU=
-cloud.google.com/go/networksecurity v0.9.1/go.mod h1:MCMdxOKQ30wsBI1eI659f9kEp4wuuAueoC9AJKSPWZQ=
-cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY=
-cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34=
-cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA=
-cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0=
-cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE=
-cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ=
-cloud.google.com/go/notebooks v1.9.1/go.mod h1:zqG9/gk05JrzgBt4ghLzEepPHNwE5jgPcHZRKhlC1A8=
-cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4=
-cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs=
-cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI=
-cloud.google.com/go/optimization v1.4.1/go.mod h1:j64vZQP7h9bO49m2rVaTVoNM0vEBEN5eKPUPbZyXOrk=
-cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA=
-cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk=
-cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ=
-cloud.google.com/go/orchestration v1.8.1/go.mod h1:4sluRF3wgbYVRqz7zJ1/EUNc90TTprliq9477fGobD8=
-cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE=
-cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc=
-cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc=
-cloud.google.com/go/orgpolicy v1.11.0/go.mod h1:2RK748+FtVvnfuynxBzdnyu7sygtoZa1za/0ZfpOs1M=
-cloud.google.com/go/orgpolicy v1.11.1/go.mod h1:8+E3jQcpZJQliP+zaFfayC2Pg5bmhuLK755wKhIIUCE=
-cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs=
-cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg=
-cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo=
-cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw=
-cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw=
-cloud.google.com/go/osconfig v1.12.0/go.mod h1:8f/PaYzoS3JMVfdfTubkowZYGmAhUCjjwnjqWI7NVBc=
-cloud.google.com/go/osconfig v1.12.1/go.mod h1:4CjBxND0gswz2gfYRCUoUzCm9zCABp91EeTtWXyz0tE=
-cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E=
-cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU=
-cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70=
-cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo=
-cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs=
-cloud.google.com/go/oslogin v1.10.1/go.mod h1:x692z7yAue5nE7CsSnoG0aaMbNoRJRXO4sn73R+ZqAs=
-cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0=
-cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA=
-cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk=
-cloud.google.com/go/phishingprotection v0.8.1/go.mod h1:AxonW7GovcA8qdEk13NfHq9hNx5KPtfxXNeUxTDxB6I=
-cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg=
-cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE=
-cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw=
-cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc=
-cloud.google.com/go/policytroubleshooter v1.7.1/go.mod h1:0NaT5v3Ag1M7U5r0GfDCpUFkWd9YqpubBWsQlhanRv0=
-cloud.google.com/go/policytroubleshooter v1.8.0/go.mod h1:tmn5Ir5EToWe384EuboTcVQT7nTag2+DuH3uHmKd1HU=
-cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0=
-cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI=
-cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg=
-cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs=
-cloud.google.com/go/privatecatalog v0.9.1/go.mod h1:0XlDXW2unJXdf9zFz968Hp35gl/bhF4twwpXZAW50JA=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
-cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI=
-cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0=
-cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8=
-cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4=
-cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc=
-cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc=
-cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg=
-cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k=
-cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM=
-cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0=
-cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4=
-cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o=
-cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk=
-cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo=
-cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE=
-cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U=
-cloud.google.com/go/recaptchaenterprise/v2 v2.6.0/go.mod h1:RPauz9jeLtB3JVzg6nCbe12qNoaa8pXc4d/YukAmcnA=
-cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c=
-cloud.google.com/go/recaptchaenterprise/v2 v2.7.2/go.mod h1:kR0KjsJS7Jt1YSyWFkseQ756D45kaYNTlDPPaRAvDBU=
-cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg=
-cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4=
-cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac=
-cloud.google.com/go/recommendationengine v0.8.1/go.mod h1:MrZihWwtFYWDzE6Hz5nKcNz3gLizXVIDI/o3G1DLcrE=
-cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg=
-cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c=
-cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs=
-cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70=
-cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ=
-cloud.google.com/go/recommender v1.10.1/go.mod h1:XFvrE4Suqn5Cq0Lf+mCP6oBHD/yRMA8XxP5sb7Q7gpA=
-cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y=
-cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A=
-cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA=
-cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM=
-cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ=
-cloud.google.com/go/redis v1.13.1/go.mod h1:VP7DGLpE91M6bcsDdMuyCm2hIpB6Vp2hI090Mfd1tcg=
-cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA=
-cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0=
-cloud.google.com/go/resourcemanager v1.5.0/go.mod h1:eQoXNAiAvCf5PXxWxXjhKQoTMaUSNrEfg+6qdf/wots=
-cloud.google.com/go/resourcemanager v1.6.0/go.mod h1:YcpXGRs8fDzcUl1Xw8uOVmI8JEadvhRIkoXXUNVYcVo=
-cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI=
-cloud.google.com/go/resourcemanager v1.9.1/go.mod h1:dVCuosgrh1tINZ/RwBufr8lULmWGOkPS8gL5gqyjdT8=
-cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU=
-cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg=
-cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA=
-cloud.google.com/go/resourcesettings v1.6.1/go.mod h1:M7mk9PIZrC5Fgsu1kZJci6mpgN8o0IUzVx3eJU3y4Jw=
-cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4=
-cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY=
-cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc=
-cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y=
-cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14=
-cloud.google.com/go/retail v1.14.1/go.mod h1:y3Wv3Vr2k54dLNIrCzenyKG8g8dhvhncT2NcNjb/6gE=
-cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do=
-cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo=
-cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM=
-cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg=
-cloud.google.com/go/run v1.2.0/go.mod h1:36V1IlDzQ0XxbQjUx6IYbw8H3TJnWvhii963WW3B/bo=
-cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s=
-cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI=
-cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk=
-cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44=
-cloud.google.com/go/scheduler v1.8.0/go.mod h1:TCET+Y5Gp1YgHT8py4nlg2Sew8nUHMqcpousDgXJVQc=
-cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc=
-cloud.google.com/go/scheduler v1.10.1/go.mod h1:R63Ldltd47Bs4gnhQkmNDse5w8gBRrhObZ54PxgR2Oo=
-cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA=
-cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4=
-cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4=
-cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU=
-cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw=
-cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4=
-cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0=
-cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU=
-cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q=
-cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA=
-cloud.google.com/go/security v1.12.0/go.mod h1:rV6EhrpbNHrrxqlvW0BWAIawFWq3X90SduMJdFwtLB8=
-cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0=
-cloud.google.com/go/security v1.15.1/go.mod h1:MvTnnbsWnehoizHi09zoiZob0iCHVcL4AUBj76h9fXA=
-cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU=
-cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc=
-cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk=
-cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk=
-cloud.google.com/go/securitycenter v1.18.1/go.mod h1:0/25gAzCM/9OL9vVx4ChPeM/+DlfGQJDwBy/UC8AKK0=
-cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag=
-cloud.google.com/go/securitycenter v1.23.0/go.mod h1:8pwQ4n+Y9WCWM278R8W3nF65QtY172h4S8aXyI9/hsQ=
-cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU=
-cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s=
-cloud.google.com/go/servicecontrol v1.10.0/go.mod h1:pQvyvSRh7YzUF2efw7H87V92mxU8FnFDawMClGCNuAA=
-cloud.google.com/go/servicecontrol v1.11.0/go.mod h1:kFmTzYzTUIuZs0ycVqRHNaNhgR+UMUpw9n02l/pY+mc=
-cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk=
-cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs=
-cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg=
-cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4=
-cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U=
-cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY=
-cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s=
-cloud.google.com/go/servicedirectory v1.10.1/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ=
-cloud.google.com/go/servicedirectory v1.11.0/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ=
-cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco=
-cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo=
-cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc=
-cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4=
-cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E=
-cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU=
-cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec=
-cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA=
-cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4=
-cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw=
-cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A=
-cloud.google.com/go/shell v1.7.1/go.mod h1:u1RaM+huXFaTojTbW4g9P5emOrrmLE69KrxqQahKn4g=
-cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos=
-cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk=
-cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M=
-cloud.google.com/go/spanner v1.47.0/go.mod h1:IXsJwVW2j4UKs0eYDqodab6HgGuA1bViSqW4uH9lfUI=
-cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM=
-cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ=
-cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0=
-cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco=
-cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0=
-cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI=
-cloud.google.com/go/speech v1.17.1/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo=
-cloud.google.com/go/speech v1.19.0/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
-cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y=
-cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc=
-cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s=
-cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y=
-cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4=
-cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E=
-cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w=
-cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I=
-cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4=
-cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw=
-cloud.google.com/go/storagetransfer v1.10.0/go.mod h1:DM4sTlSmGiNczmV6iZyceIh2dbs+7z2Ayg6YAiQlYfA=
-cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw=
-cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g=
-cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM=
-cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA=
-cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c=
-cloud.google.com/go/talent v1.6.2/go.mod h1:CbGvmKCG61mkdjcqTcLOkb2ZN1SrQI8MDyma2l7VD24=
-cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8=
-cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4=
-cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc=
-cloud.google.com/go/texttospeech v1.7.1/go.mod h1:m7QfG5IXxeneGqTapXNxv2ItxP/FS0hCZBwXYqucgSk=
-cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ=
-cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg=
-cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM=
-cloud.google.com/go/tpu v1.6.1/go.mod h1:sOdcHVIgDEEOKuqUoi6Fq53MKHJAtOwtz0GuKsWSH3E=
-cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28=
-cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y=
-cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA=
-cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk=
-cloud.google.com/go/trace v1.10.1/go.mod h1:gbtL94KE5AJLH3y+WVpfWILmqgc6dXcqgNXdOPAQTYk=
-cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs=
-cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg=
-cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0=
-cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos=
-cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos=
-cloud.google.com/go/translate v1.8.1/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs=
-cloud.google.com/go/translate v1.8.2/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs=
-cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk=
-cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw=
-cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg=
-cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1tR5lGthk=
-cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ=
-cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ=
-cloud.google.com/go/video v1.17.1/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU=
-cloud.google.com/go/video v1.19.0/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU=
-cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU=
-cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4=
-cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M=
-cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU=
-cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU=
-cloud.google.com/go/videointelligence v1.11.1/go.mod h1:76xn/8InyQHarjTWsBR058SmlPCwQjgcvoW0aZykOvo=
-cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0=
-cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo=
-cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo=
-cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY=
-cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E=
-cloud.google.com/go/vision/v2 v2.6.0/go.mod h1:158Hes0MvOS9Z/bDMSFpjwsUrZ5fPrdwuyyvKSGAGMY=
-cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0=
-cloud.google.com/go/vision/v2 v2.7.2/go.mod h1:jKa8oSYBWhYiXarHPvP4USxYANYUEdEsQrloLjrSwJU=
-cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE=
-cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g=
-cloud.google.com/go/vmmigration v1.5.0/go.mod h1:E4YQ8q7/4W9gobHjQg4JJSgXXSgY21nA5r8swQV+Xxc=
-cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY=
-cloud.google.com/go/vmmigration v1.7.1/go.mod h1:WD+5z7a/IpZ5bKK//YmT9E047AD+rjycCAvyMxGJbro=
-cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208=
-cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8=
-cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY=
-cloud.google.com/go/vmwareengine v0.4.1/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0=
-cloud.google.com/go/vmwareengine v1.0.0/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0=
-cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w=
-cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8=
-cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes=
-cloud.google.com/go/vpcaccess v1.7.1/go.mod h1:FogoD46/ZU+JUBX9D606X21EnxiszYi2tArQwLY4SXs=
-cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE=
-cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg=
-cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc=
-cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A=
-cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg=
-cloud.google.com/go/webrisk v1.9.1/go.mod h1:4GCmXKcOa2BZcZPn6DCEvE7HypmEJcJkr4mtM+sqYPc=
-cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo=
-cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ=
-cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng=
-cloud.google.com/go/websecurityscanner v1.6.1/go.mod h1:Njgaw3rttgRHXzwCB8kgCYqv5/rGpFCsBOvPbYgszpg=
-cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0=
-cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M=
-cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M=
-cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA=
-cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw=
-cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvofaL6FqlET6g=
-collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8=
-git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc=
-github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=
-github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
-github.com/Azure/azure-sdk-for-go v43.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/Azure/azure-sdk-for-go v52.5.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/Azure/azure-sdk-for-go v55.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM=
+cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4=
+cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
+cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
+cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM=
+cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA=
+cloud.google.com/go/kms v1.15.8 h1:szIeDCowID8th2i8XE4uRev5PMxQFqW+JjwYxL9h6xs=
+cloud.google.com/go/kms v1.15.8/go.mod h1:WoUHcDjD9pluCg7pNds131awnH429QGvRM3N/4MyoVs=
+cloud.google.com/go/storage v1.39.1 h1:MvraqHKhogCOTXTlct/9C3K3+Uy2jBmFYb3/Sp6dVtY=
+cloud.google.com/go/storage v1.39.1/go.mod h1:xK6xZmxZmo+fyP7+DEF6FhNc24/JAe95OLyOHCXFH1o=
+cuelabs.dev/go/oci/ociregistry v0.0.0-20240318100017-39d12ee67b8b h1:U9PBmYNLgTOoSgsqBD1K8MlqjqYpxf5cqZ1mp6JIaF4=
+cuelabs.dev/go/oci/ociregistry v0.0.0-20240318100017-39d12ee67b8b/go.mod h1:pK23AUVXuNzzTpfMCA06sxZGeVQ/75FdVtW249de9Uo=
+cuelang.org/go v0.8.0 h1:fO1XPe/SUGtc7dhnGnTPbpIDoQm/XxhDtoSF7jzO01c=
+cuelang.org/go v0.8.0/go.mod h1:CoDbYolfMms4BhWUlhD+t5ORnihR7wvjcfgyO9lL5FI=
+dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
+dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230919221257-8b5d3ce2d11d h1:zjqpY4C7H15HjRPEenkS4SAn3Jy2eRRjkjZbGR30TOg=
+github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230919221257-8b5d3ce2d11d/go.mod h1:XNqJ7hv2kY++g8XEHREpi+JqZo3+0l+CH2egBVN4yqM=
+github.com/AliyunContainerService/ack-ram-tool/pkg/credentials/alibabacloudsdkgo/helper v0.2.0 h1:8+4G8JaejP8Xa6W46PzJEwisNgBXMvFcz78N6zG/ARw=
+github.com/AliyunContainerService/ack-ram-tool/pkg/credentials/alibabacloudsdkgo/helper v0.2.0/go.mod h1:GgeIE+1be8Ivm7Sh4RgwI42aTtC9qrcj+Y9Y6CjJhJs=
+github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU=
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
-github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
-github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 h1:c4k2FIYIh4xtwqrQwV0Ct1v5+ehlNXj5NI/MWVsiTkQ=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2/go.mod h1:5FDJtLEO/GxwNgUxbwrY3LP0pEoThTQJtk2oysdXHxM=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.1 h1:MyVTgWR8qd/Jw1Le0NZebGBUCLbtak3bJ3z1OlqZBpw=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.1/go.mod h1:GpPjLhVR9dnUoJMyHWSPy71xY9/lcmpzIPZXmF0FCVY=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0 h1:D3occbWoio4EBLkbkevetNMAVX197GkzbUMtqjGWn80=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0/go.mod h1:bTSOgj05NGRuHHhQwAdPnYr9TOdNmKlZTgGLL6nyAdI=
+github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
-github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
+github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc=
+github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw=
github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs=
-github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
-github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
-github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
+github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk=
+github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8=
github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c=
+github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk=
+github.com/Azure/go-autorest/autorest/azure/auth v0.5.12/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg=
+github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg=
+github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc=
+github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0=
+github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw=
github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU=
-github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc=
-github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE=
-github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E=
-github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=
github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
-github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
-github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
-github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
-github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20200415212048-7901bc822317/go.mod h1:DF8FZRxMHMGv/vP2lQP6h+dYzzjpuRn24VeRiYn3qjQ=
-github.com/GoogleCloudPlatform/k8s-cloud-provider v1.16.1-0.20210702024009-ea6160c1d0e3/go.mod h1:8XasY4ymP2V/tn2OOV9ZadmiTE1FIB/h3W+yNlPttKw=
-github.com/GoogleCloudPlatform/k8s-cloud-provider v1.18.1-0.20220218231025-f11817397a1b/go.mod h1:FNj4KYEAAHfYu68kRYolGoxkaJn+6mdEsaM12VTwuI0=
-github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM=
-github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA=
-github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
-github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
-github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
-github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
-github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
-github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
-github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
-github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
-github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
-github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
-github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
-github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM=
-github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
-github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
-github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
-github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
-github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
+github.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZYIR/J6A=
+github.com/MakeNowJust/heredoc/v2 v2.0.1/go.mod h1:6/2Abh5s+hc3g9nbWLe9ObDIOhaRrqsyY9MWy+4JdRM=
+github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
-github.com/Microsoft/hcsshim v0.8.10-0.20200715222032-5eafd1556990/go.mod h1:ay/0dTb7NsG8QMDfsRfLHgZo/6xAJShLe1+ePPflihk=
-github.com/Microsoft/hcsshim v0.8.22/go.mod h1:91uVCVzvX2QD16sMCenoxxXo6L1wJnLMX2PSufFMtF0=
-github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8=
-github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
-github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
-github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs=
-github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
-github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
-github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
-github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
-github.com/a8m/tree v0.0.0-20210115125333-10a5fd5b637d/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg=
-github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
-github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
-github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
-github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
-github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
-github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM=
-github.com/alecthomas/kingpin/v2 v2.3.1/go.mod h1:oYL5vtsvEHZGHxU7DMp32Dvx+qL+ptGn6lWaot2vCNE=
-github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
-github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
-github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
-github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6/go.mod h1:+lx6/Aqd1kLJ1GQfkvOnaZ1WGmLpMpbprPuIOOZX30U=
-github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
-github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
-github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
-github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0=
-github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
-github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI=
-github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg=
-github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
-github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
-github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
-github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
-github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
+github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
+github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg=
+github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y=
+github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78=
+github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
+github.com/ThalesIgnite/crypto11 v1.2.5 h1:1IiIIEqYmBvUYFeMnHqRft4bwf/O36jryEUpY+9ef8E=
+github.com/ThalesIgnite/crypto11 v1.2.5/go.mod h1:ILDKtnCKiQ7zRoNxcp36Y1ZR8LBPmR2E23+wTQe/MlE=
+github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
+github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
+github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
+github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
+github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.2/go.mod h1:sCavSAvdzOjul4cEqeVtvlSaSScfNsTQ+46HwlTL1hc=
+github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 h1:iC9YFYKDGEy3n/FtqJnOkZsene9olVspKmkX5A2YBEo=
+github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4/go.mod h1:sCavSAvdzOjul4cEqeVtvlSaSScfNsTQ+46HwlTL1hc=
+github.com/alibabacloud-go/cr-20160607 v1.0.1 h1:WEnP1iPFKJU74ryUKh/YDPHoxMZawqlPajOymyNAkts=
+github.com/alibabacloud-go/cr-20160607 v1.0.1/go.mod h1:QHeKZtZ3F3FOE+/uIXCBAp8POwnUYekpLwr1dtQa5r0=
+github.com/alibabacloud-go/cr-20181201 v1.0.10 h1:B60f6S1imsgn2fgC6X6FrVNrONDrbCT0NwYhsJ0C9/c=
+github.com/alibabacloud-go/cr-20181201 v1.0.10/go.mod h1:VN9orB/w5G20FjytoSpZROqu9ZqxwycASmGqYUJSoDc=
+github.com/alibabacloud-go/darabonba-openapi v0.1.12/go.mod h1:sTAjsFJmVsmcVeklL9d9uDBlFsgl43wZ6jhI6BHqHqU=
+github.com/alibabacloud-go/darabonba-openapi v0.1.14/go.mod h1:w4CosR7O/kapCtEEMBm3JsQqWBU/CnZ2o0pHorsTWDI=
+github.com/alibabacloud-go/darabonba-openapi v0.2.1 h1:WyzxxKvhdVDlwpAMOHgAiCJ+NXa6g5ZWPFEzaK/ewwY=
+github.com/alibabacloud-go/darabonba-openapi v0.2.1/go.mod h1:zXOqLbpIqq543oioL9IuuZYOQgHQ5B8/n5OPrnko8aY=
+github.com/alibabacloud-go/darabonba-string v1.0.0/go.mod h1:93cTfV3vuPhhEwGGpKKqhVW4jLe7tDpo3LUM0i0g6mA=
+github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68/go.mod h1:6pb/Qy8c+lqua8cFpEy7g39NRRqOWc3rOwAy8m5Y2BY=
+github.com/alibabacloud-go/debug v1.0.0 h1:3eIEQWfay1fB24PQIEzXAswlVJtdQok8f3EVN5VrBnA=
+github.com/alibabacloud-go/debug v1.0.0/go.mod h1:8gfgZCCAC3+SCzjWtY053FrOcd4/qlH6IHTI4QyICOc=
+github.com/alibabacloud-go/endpoint-util v1.1.0/go.mod h1:O5FuCALmCKs2Ff7JFJMudHs0I5EBgecXXxZRyswlEjE=
+github.com/alibabacloud-go/endpoint-util v1.1.1 h1:ZkBv2/jnghxtU0p+upSU0GGzW1VL9GQdZO3mcSUTUy8=
+github.com/alibabacloud-go/endpoint-util v1.1.1/go.mod h1:O5FuCALmCKs2Ff7JFJMudHs0I5EBgecXXxZRyswlEjE=
+github.com/alibabacloud-go/openapi-util v0.0.9/go.mod h1:sQuElr4ywwFRlCCberQwKRFhRzIyG4QTP/P4y1CJ6Ws=
+github.com/alibabacloud-go/openapi-util v0.0.10/go.mod h1:sQuElr4ywwFRlCCberQwKRFhRzIyG4QTP/P4y1CJ6Ws=
+github.com/alibabacloud-go/openapi-util v0.0.11/go.mod h1:sQuElr4ywwFRlCCberQwKRFhRzIyG4QTP/P4y1CJ6Ws=
+github.com/alibabacloud-go/openapi-util v0.1.0 h1:0z75cIULkDrdEhkLWgi9tnLe+KhAFE/r5Pb3312/eAY=
+github.com/alibabacloud-go/openapi-util v0.1.0/go.mod h1:sQuElr4ywwFRlCCberQwKRFhRzIyG4QTP/P4y1CJ6Ws=
+github.com/alibabacloud-go/tea v1.1.0/go.mod h1:IkGyUSX4Ba1V+k4pCtJUc6jDpZLFph9QMy2VUPTwukg=
+github.com/alibabacloud-go/tea v1.1.7/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4=
+github.com/alibabacloud-go/tea v1.1.8/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4=
+github.com/alibabacloud-go/tea v1.1.11/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4=
+github.com/alibabacloud-go/tea v1.1.17/go.mod h1:nXxjm6CIFkBhwW4FQkNrolwbfon8Svy6cujmKFUq98A=
+github.com/alibabacloud-go/tea v1.1.19/go.mod h1:nXxjm6CIFkBhwW4FQkNrolwbfon8Svy6cujmKFUq98A=
+github.com/alibabacloud-go/tea v1.2.2 h1:aTsR6Rl3ANWPfqeQugPglfurloyBJY85eFy7Gc1+8oU=
+github.com/alibabacloud-go/tea v1.2.2/go.mod h1:CF3vOzEMAG+bR4WOql8gc2G9H3EkH3ZLAQdpmpXMgwk=
+github.com/alibabacloud-go/tea-utils v1.3.1/go.mod h1:EI/o33aBfj3hETm4RLiAxF/ThQdSngxrpF8rKUDJjPE=
+github.com/alibabacloud-go/tea-utils v1.3.9/go.mod h1:EI/o33aBfj3hETm4RLiAxF/ThQdSngxrpF8rKUDJjPE=
+github.com/alibabacloud-go/tea-utils v1.4.3/go.mod h1:KNcT0oXlZZxOXINnZBs6YvgOd5aYp9U67G+E3R8fcQw=
+github.com/alibabacloud-go/tea-utils v1.4.5 h1:h0/6Xd2f3bPE4XHTvkpjwxowIwRCJAJOqY6Eq8f3zfA=
+github.com/alibabacloud-go/tea-utils v1.4.5/go.mod h1:KNcT0oXlZZxOXINnZBs6YvgOd5aYp9U67G+E3R8fcQw=
+github.com/alibabacloud-go/tea-xml v1.1.2/go.mod h1:Rq08vgCcCAjHyRi/M7xlHKUykZCEtyBy9+DPF6GgEu8=
+github.com/alibabacloud-go/tea-xml v1.1.3 h1:7LYnm+JbOq2B+T/B0fHC4Ies4/FofC4zHzYtqw7dgt0=
+github.com/alibabacloud-go/tea-xml v1.1.3/go.mod h1:Rq08vgCcCAjHyRi/M7xlHKUykZCEtyBy9+DPF6GgEu8=
+github.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6qTJya2bDq4X1Tw=
+github.com/aliyun/credentials-go v1.3.2 h1:L4WppI9rctC8PdlMgyTkF8bBsy9pyKQEzBD1bHMRl+g=
+github.com/aliyun/credentials-go v1.3.2/go.mod h1:tlpz4uys4Rn7Ik4/piGRrTbXy2uLKvePgQJJduE+Y5c=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
+github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
+github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
-github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
-github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
-github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
-github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7/go.mod h1:LWMyo4iOLWXHGdBki7NIht1kHru/0wM179h+d3g8ATM=
-github.com/auth0/go-jwt-middleware v1.0.1/go.mod h1:YSeUX3z6+TF2H+7padiEqNJ73Zy9vXW72U//IgN0BIM=
-github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
-github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
-github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
-github.com/aws/aws-sdk-go v1.38.3/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
-github.com/aws/aws-sdk-go v1.38.49/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
-github.com/aws/aws-sdk-go v1.43.16 h1:Y7wBby44f+tINqJjw5fLH3vA+gFq4uMITIKqditwM14=
-github.com/aws/aws-sdk-go v1.43.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
-github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
-github.com/aws/aws-sdk-go-v2 v1.0.0/go.mod h1:smfAbmpW+tcRVuNUjo3MOArSZmW72t62rkCzc2i0TWM=
-github.com/aws/aws-sdk-go-v2 v1.7.0/go.mod h1:tb9wi5s61kTDA5qCkcDbt3KRVV74GGslQkl/DRdX/P4=
-github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw=
-github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY=
-github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
-github.com/aws/aws-sdk-go-v2/config v1.18.23 h1:gc3lPsAnZpwfi2exupmgHfva0JiAY2BWDg5JWYlmA28=
-github.com/aws/aws-sdk-go-v2/config v1.18.23/go.mod h1:rx0ruaQ+gk3OrLFHRRx56lA//XxP8K8uPzeNiKNuWVY=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.22 h1:Hp9rwJS4giQ48xqonRV/s7QcDf/wxF6UY7osRmBabvI=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.22/go.mod h1:BfNcm6A9nSd+bzejDcMJ5RE+k6WbkCwWkQil7q4heRk=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 h1:jJPgroehGvjrde3XufFIJUZVK5A2L9a3KwSFgKy9n8w=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 h1:kG5eQilShqmJbv11XL1VpyDbaEJzWxd4zRiCG30GSn4=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 h1:vFQlirhuM8lLlpI7imKOMsjdQLuN9CPi+k44F/OFVsk=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 h1:gGLG7yKaXG02/jBlg210R7VgQIotiQntNhsCFejawx8=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc=
-github.com/aws/aws-sdk-go-v2/service/autoscaling v1.0.0 h1:qhlzq+/+r7x85qcd+dMMzUJ2WdaHSMkYBalMaIUH3c0=
-github.com/aws/aws-sdk-go-v2/service/autoscaling v1.0.0/go.mod h1:XGqFiu9uLXgwJvujnm9EGAwk6+bRnUn1omVyuNt3mks=
-github.com/aws/aws-sdk-go-v2/service/cloudformation v1.0.0 h1:kt1v8ZnGsSYusSCnnOpKcBfIHZC/JLj+Rzu47VWz6Vo=
-github.com/aws/aws-sdk-go-v2/service/cloudformation v1.0.0/go.mod h1:eWeVWUXYoJly9d7xI2346JNW4rx9UbIp5Bki6mtCBTo=
-github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.0.0 h1:SREEMUFRBIDGmo9IU4zqmGwHBdKd+Fz0RdM4m+142uw=
-github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.0.0/go.mod h1:u1GqwOV+isp7n1DZF+aCa7TkA8QwVYq6mHkPbeWnuLk=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.11.0 h1:KFzMDGBBkeo22Ty+A4xFc7LY7TmwOaJSETj/l7o90Vo=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.11.0/go.mod h1:WEDK28a3G3+BQCzP50oGA/6807+Sx/Ogn8BttfJ27zY=
-github.com/aws/aws-sdk-go-v2/service/ecr v1.17.20 h1:nJnXfQggNZdrWz/0cm2ZGyddGK+FqTiN4QJGanzKZoY=
-github.com/aws/aws-sdk-go-v2/service/ecr v1.17.20/go.mod h1:kEVGiy2tACP0cegVqx4MrjsgQMSgrtgRq1fSa+Ix6F0=
-github.com/aws/aws-sdk-go-v2/service/eks v1.0.0 h1:6W2OA2mfmr8P8taz5zCsODVPZUk/+w7I3DS1R+a1YvM=
-github.com/aws/aws-sdk-go-v2/service/eks v1.0.0/go.mod h1:/cWWNlzpw38M5ckeNr/orjoT+sZc2wTWubnW2IIV3K0=
-github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.0.0 h1:OJnzXg++TleNvDO+/Ysx+8XPiz2VxoPJ1UdiyL9fVHY=
-github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.0.0/go.mod h1:n5YmmB7VY/iK0TtXWSUkuO8dx11DXoMeNJ5HrCYJSQs=
-github.com/aws/aws-sdk-go-v2/service/iam v1.0.0 h1:hbMu6cCgLxEYyhrba9RqkxewyfxrUWiNDc12Epmk338=
-github.com/aws/aws-sdk-go-v2/service/iam v1.0.0/go.mod h1:2Q65VwdiZuvBXXmr45Velx3g5sEgqQomdwJKu2+413Q=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.0 h1:jjZzz89+Uii7XKlgWXNHiLVtJfvCG8oVoMLpiWsjnt8=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.0/go.mod h1:cZbnzYflIuoRkuKp4BB4q/R4xklYIwpLYs26vS3/Sac=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.0/go.mod h1:3jExOmpbjgPnz2FJaMOfbSk1heTkZ66aD3yNtVhnjvI=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.0/go.mod h1:a7XLWNKuVgOxjssEF019IiHPv35k8KHBaWv/wJAfi2A=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 h1:0iKliEXAcCa2qVtRs7Ot5hItA2MsufrphbRFlz1Owxo=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.0.0 h1:Cg1XFRo41piOIT8Qp9RPQxfwLac5ddwGQxTPM8lowGk=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.0.0/go.mod h1:ElU0+utGClu2dFpCf1NIFxFAG+xO4n5b5RBuIiVaCY0=
-github.com/aws/aws-sdk-go-v2/service/kms v1.0.0 h1:RWmKjqHuZ3s72FNxosKd3JpvWrozdS0x67kTvjfB0nY=
-github.com/aws/aws-sdk-go-v2/service/kms v1.0.0/go.mod h1:cCF1wkxigOtOGsQ4HWEG8mET3EmqQl1sZSvY4JVJieM=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.0.0 h1:7petFdJE3VuXZnXNVDdynznREElHSzjYI4xjkGNWPX8=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.0.0/go.mod h1:IdVR1fGqVS8Zv/oraQXdBzbGmdpc3FBOHhCTI7tpsYE=
-github.com/aws/aws-sdk-go-v2/service/ssm v1.0.0 h1:6kOQZ2+aazkPflMg+hsycxObxaRG0dSFxxSE+2E5Hgc=
-github.com/aws/aws-sdk-go-v2/service/ssm v1.0.0/go.mod h1:AEGyxPnsQBqbeGRhLN7b4au2PbLzXWR9WXhmfKEeiRc=
-github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 h1:UBQjaMTCKwyUYwiVnUt6toEJwGXsLBI6al083tpjJzY=
-github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 h1:PkHIIJs8qvq0e5QybnZoG1K/9QTrLr9OsqCIo59jOBA=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk=
-github.com/aws/aws-sdk-go-v2/service/sts v1.18.11 h1:uBE+Zj478pfxV98L6SEpvxYiADNjTlMNY714PJLE7uo=
-github.com/aws/aws-sdk-go-v2/service/sts v1.18.11/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8=
-github.com/aws/smithy-go v1.0.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw=
-github.com/aws/smithy-go v1.5.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
-github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
-github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
-github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
-github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
+github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
+github.com/aws/aws-sdk-go v1.51.2 h1:Ruwgz5aqIXin5Yfcgc+PCzoqW5tEGb9aDL/JWDsre7k=
+github.com/aws/aws-sdk-go v1.51.2/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
+github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb5Rw=
+github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc=
+github.com/aws/aws-sdk-go-v2/config v1.27.8 h1:0r8epOsiJ7YJz65MGcb8i91ehFp4kvvFe2qkq5oYeRI=
+github.com/aws/aws-sdk-go-v2/config v1.27.8/go.mod h1:XsmYKxYNuIhLsFddpNds+j9H5XKzjWDdg/SZngiwFio=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.8 h1:WUdNLXbyNbU07V/WFrSOBXqZTDgmmMNMgUFzpYOKJhw=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.8/go.mod h1:iPZzLpaBIfhyvVS/XGD3JvR1GP3YdHTqpySKDlqkfs8=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.4 h1:S+L2QSKhUuShih3aq9P/mkzDBiOO5tTyVg+vXREfsfg=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.4/go.mod h1:nQ3how7DMnFMWiU1SpECohgC82fpn4cKZ875NDMmwtA=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 h1:I/5wmGMffY4happ8NOCuIUEWGUvvFp5NSeQcXl9RHcI=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26/go.mod h1:FR8f4turZtNy6baO0KJ5FJUmXH/cSkI9fOngs0yl6mA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 h1:zXFLuEuMMUOvEARXFUVJdfqZ4bvvSgdGRq/ATcrQxzM=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26/go.mod h1:3o2Wpy0bogG1kyOPrgkXA8pgIfEEv0+m19O9D5+W8y8=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 h1:GeNJsIFHB+WW5ap2Tec4K6dzcVTsRbsT1Lra46Hv9ME=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26/go.mod h1:zfgMpwHDXX2WGoG84xG2H+ZlPTkJUU4YUvx2svLQYWo=
+github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.4 h1:f4pkN5PVSqlGxD2gZvboz6SRaeoykgknflMPBVuhcGs=
+github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.4/go.mod h1:NZBgGUf6LD2KS6Ns5xTK+cR1LK5hZwNkeOt8nDKXzMA=
+github.com/aws/aws-sdk-go-v2/service/cloudformation v1.48.0 h1:uMlYsoHdd2Gr9sDGq2ieUR5jVu7F5AqPYz6UBJmdRhY=
+github.com/aws/aws-sdk-go-v2/service/cloudformation v1.48.0/go.mod h1:G2qcp9xrwch6TH9AlzWoYbV9QScyZhLCoMCQ1+BD404=
+github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.36.3 h1:l3vM7tnmYWZBdyN1d2Q4gTCnDNbwKNtns4oCFt0zfQk=
+github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.36.3/go.mod h1:xeAHc7vhdOYwpG2t4uXdnGhOvOIpJ8n+A5AHnCkk8iw=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.151.1 h1:Ky/RdoVNuWli0Qzvn2q7iXAPJ7Lf+YL22D6q1SVXU3Y=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.151.1/go.mod h1:TeZ9dVQzGaLG+SBIgdLIDbJ6WmfFvksLeG3EHGnNfZM=
+github.com/aws/aws-sdk-go-v2/service/ecr v1.27.3 h1:gfgt0D8MGL3gHrJPEv4rcWptA4Nz7uYn25ls8lLiANw=
+github.com/aws/aws-sdk-go-v2/service/ecr v1.27.3/go.mod h1:O5Fvd41s5KfDG093xLM7FhGiH6EmhmEli5D5MQH3TWw=
+github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.23.3 h1:gaq/4fd2/bQeJ33m4csgL7DJHrrmvGhqnrsxchNr46c=
+github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.23.3/go.mod h1:vn+Rz9fAFGJtDXbBmYdTc71Q8iF/W/uK1/ec93hinD8=
+github.com/aws/aws-sdk-go-v2/service/eks v1.53.0 h1:ACTxnLwL6YNmuYbxtp/VR3HGL9SWXU6VZkXPjWST9ZQ=
+github.com/aws/aws-sdk-go-v2/service/eks v1.53.0/go.mod h1:ZzOjZXGGUQxOq+T3xmfPLKCZe4OaB5vm1LdGaC8IPn4=
+github.com/aws/aws-sdk-go-v2/service/iam v1.38.3 h1:2sFIoFzU1IEL9epJWubJm9Dhrn45aTNEJuwsesaCGnk=
+github.com/aws/aws-sdk-go-v2/service/iam v1.38.3/go.mod h1:KzlNINwfr/47tKkEhgk0r10/OZq3rjtyWy0txL3lM+I=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 h1:tB4tNw83KcajNAzaIMhkhVI2Nt8fAZd5A5ro113FEMY=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7/go.mod h1:lvpyBGkZ3tZ9iSsUIcC2EWp+0ywa7aK3BLT+FwZi+mQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 h1:8eUsivBQzZHqe/3FE+cqwfH+0p5Jo8PFM/QYQSmeZ+M=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7/go.mod h1:kLPQvGUmxn/fqiCrDeohwG33bq2pQpGeY62yRO6Nrh0=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEHWM0bJ1QcBzxLrL/k2DHvGYhb8+W1w=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c=
+github.com/aws/aws-sdk-go-v2/service/kms v1.27.9 h1:W9PbZAZAEcelhhjb7KuwUtf+Lbc+i7ByYJRuWLlnxyQ=
+github.com/aws/aws-sdk-go-v2/service/kms v1.27.9/go.mod h1:2tFmR7fQnOdQlM2ZCEPpFnBIQD1U8wmXmduBgZbOag0=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 h1:aOVVZJgWbaH+EJYPvEgkNhCEbXXvH7+oML36oaPK3zE=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q=
+github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2 h1:z6Pq4+jtKlhK4wWJGHRGwMLGjC1HZwAO3KJr/Na0tSU=
+github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2/go.mod h1:DSmu/VZzpQlAubWBbAvNpt+S4k/XweglJi4XaDGyvQk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 h1:mnbuWHOcM70/OFUlZZ5rcdfA8PflGXXiefU/O+1S3+8=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.3/go.mod h1:5HFu51Elk+4oRBZVxmHrSds5jFXmFj8C3w7DVF2gnrs=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 h1:uLq0BKatTmDzWa/Nu4WO0M1AaQDaPpwTKAeByEc6WFM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3/go.mod h1:b+qdhjnxj8GSR6t5YfphOffeoQSQ1KmpoVVuBn+PWxs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 h1:J/PpTf/hllOjx8Xu9DMflff3FajfLxqM5+tepvVXmxg=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod h1:0ih0Z83YDH/QeQ6Ori2yGE2XvWYv/Xm+cZc01LC6oK0=
+github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
+github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240318154307-a1a918375412 h1:tfbmGNeOidVXzO1I7zo/WsT5QX7Aa0BGTbnEAE4FG3E=
+github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240318154307-a1a918375412/go.mod h1:kcUkjB9HwuV7PSck2b60kJtgDy+eTHWuAP0kb93FXsk=
+github.com/awslabs/operatorpkg v0.0.0-20241205163410-0fff9f28d115 h1:9nhjY3dzCpEmhpQ0vMlhB7wqucAiftLjAIEQu8uT2J4=
+github.com/awslabs/operatorpkg v0.0.0-20241205163410-0fff9f28d115/go.mod h1:TTs6HGuqmgdNyNlbdv29v1OoON+kQKVPojZgJaJVtNk=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU=
-github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
-github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
-github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
-github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
-github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
-github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
-github.com/briandowns/spinner v1.11.1 h1:OixPqDEcX3juo5AjQZAnFPbeUA0jvkp2qzB5gOZJ/L0=
-github.com/briandowns/spinner v1.11.1/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ=
-github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70=
-github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
-github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng=
-github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
-github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ=
-github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
-github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o=
-github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
-github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34=
-github.com/caddyserver/caddy v1.0.3/go.mod h1:G+ouvOY32gENkJC+jhgl62TyhvqEsFaDiZ4uw0RzP1E=
-github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
-github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
-github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
-github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg=
-github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
-github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/buildkite/agent/v3 v3.66.0 h1:Yw1dxN2lhUYRzs0g2QwXho60nqSuN4mRzt1aYHR+rjM=
+github.com/buildkite/agent/v3 v3.66.0/go.mod h1:P8fwXRxHrApHhoA4H1tlixB7+GucL9oqlalR9bfofJY=
+github.com/buildkite/go-pipeline v0.4.1 h1:RZ1afSHOt5mUcTzhFab0WxKm90vvJsOUAPpfQBETgkE=
+github.com/buildkite/go-pipeline v0.4.1/go.mod h1:/8zdWlpn40HsxZql5iUbr00P8/Fm/yud9+Bqrar8S3s=
+github.com/buildkite/interpolate v0.0.0-20200526001904-07f35b4ae251 h1:k6UDF1uPYOs0iy1HPeotNa155qXRWrzKnqAaGXHLZCE=
+github.com/buildkite/interpolate v0.0.0-20200526001904-07f35b4ae251/go.mod h1:gbPR1gPu9dB96mucYIR7T3B7p/78hRVSOuzIWLHK2Y4=
+github.com/buildkite/roko v1.2.0 h1:hbNURz//dQqNl6Eo9awjQOVOZwSDJ8VEbBDxSfT9rGQ=
+github.com/buildkite/roko v1.2.0/go.mod h1:23R9e6nHxgedznkwwfmqZ6+0VJZJZ2Sg/uVcp2cP46I=
+github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
+github.com/bytecodealliance/wasmtime-go/v3 v3.0.2 h1:3uZCA/BLTIu+DqCfguByNMJa2HVHpXvjfy0Dy7g6fuA=
+github.com/bytecodealliance/wasmtime-go/v3 v3.0.2/go.mod h1:RnUjnIXxEJcL6BgCvNyzCCRzZcxCgsZCi+RNlvYor5Q=
+github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M=
+github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
-github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
-github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
-github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
-github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
-github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 h1:krfRl01rzPzxSxyLyrChD+U+MzsBXbm0OwYYB67uF+4=
+github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589/go.mod h1:OuDyvmLnMCwa2ep4Jkm6nyA0ocJuZlGyk2gGseVzERM=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs=
-github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo=
-github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg=
-github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc=
-github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
-github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
-github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
+github.com/clbanning/mxj/v2 v2.5.5/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s=
+github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME=
+github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1wt9Z3DP8O6W3rvwCt0REIlshg1InHImaLW0t3ObY0=
+github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
+github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
+github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
-github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
-github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
-github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
-github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
-github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
-github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
-github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s=
-github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM=
-github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=
-github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
-github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
-github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
-github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
-github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw=
-github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ=
-github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
-github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.12/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.7.12 h1:+KQsnv4VnzyxWcfO9mlxxELaoztsDEjOuCMPAuPqgU0=
-github.com/containerd/containerd v1.7.12/go.mod h1:/5OMpE1p0ylxtEUGY8kuCYkDRzJm9NO1TFMWjUpdevk=
-github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
-github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM=
-github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
-github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
-github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
-github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
-github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
-github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
-github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
-github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
-github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
-github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
-github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg=
-github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=
-github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
-github.com/coredns/caddy v1.1.0/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
-github.com/coredns/corefile-migration v1.0.10/go.mod h1:RMy/mXdeDlYwzt0vdMEJvT2hGJ2I86/eO0UdXmH9XNI=
-github.com/coredns/corefile-migration v1.0.14/go.mod h1:XnhgULOEouimnzgn0t4WPuFDN2/PJQcTxdWKC5eXNGE=
-github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
-github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
-github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
-github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
-github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
-github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
-github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
-github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
-github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
-github.com/crowdstrike/falcon-operator v0.9.5 h1:Ai7bAS3VI5v0mmiTjg0Ng57WQYnz7CKeBI2Att2Ez7o=
-github.com/crowdstrike/falcon-operator v0.9.5/go.mod h1:hkglGFZuORnsj6BDmghP9z7cRgu6z+h2SzwYAJEwOu4=
-github.com/crowdstrike/gofalcon v0.4.2 h1:lMO1AVgFOrmDCYWFmFBxbwDvGO2fECWkdHwIe4DyLoM=
-github.com/crowdstrike/gofalcon v0.4.2/go.mod h1:7+jUPekHO7/KvQ8pXw675S7TZMdGhP92NsmOyzGNy+s=
-github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
-github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg=
+github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc=
+github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE=
+github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4=
+github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ=
+github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w=
+github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU=
+github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk=
+github.com/coreos/go-oidc/v3 v3.9.0 h1:0J/ogVOd4y8P0f0xUh8l9t07xRP/d8tccvjHl2dcsSo=
+github.com/coreos/go-oidc/v3 v3.9.0/go.mod h1:rTKz2PYwftcrtoCzV5g5kvfJoWcm0Mk8AF8y1iAQro4=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f h1:eHnXnuK47UlSTOQexbzxAZfekVz6i+LKRdj1CU5DPaM=
+github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw=
github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
+github.com/danieljoos/wincred v1.2.1 h1:dl9cBrupW8+r5250DYkYxocLeZ1Y4vB1kxgtjxw8GQs=
+github.com/danieljoos/wincred v1.2.1/go.mod h1:uGaFL9fDn3OLTvzCGulzE+SzjEe5NGlh5FdCcyfPwps=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ=
-github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/digitalocean/godo v1.58.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU=
-github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc=
-github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI=
-github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
-github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
-github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
-github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY=
-github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/distribution v2.8.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
-github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v20.10.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
-github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A=
-github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0=
-github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
-github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
-github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8=
-github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
-github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8=
-github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
-github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
-github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4=
-github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
-github.com/dougm/pretty v0.0.0-20171025230240-2ee9d7453c02/go.mod h1:7NQ3kWOx2cZOSjtcveTa5nqupVr2s6/83sG+rTlI7uA=
-github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
-github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
-github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
-github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/depcheck-test/depcheck-test v0.0.0-20220607135614-199033aaa936 h1:foGzavPWwtoyBvjWyKJYDYsyzy+23iBV7NKTwdk+LRY=
+github.com/depcheck-test/depcheck-test v0.0.0-20220607135614-199033aaa936/go.mod h1:ttKPnOepYt4LLzD+loXQ1rT6EmpyIYHro7TAJuIIlHo=
+github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg=
+github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw=
+github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
+github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
+github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g=
+github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
+github.com/digitorus/pkcs7 v0.0.0-20230713084857-e76b763bdc49/go.mod h1:SKVExuS+vpu2l9IoOc0RwqE7NYnb0JlcFHFnEJkVDzc=
+github.com/digitorus/pkcs7 v0.0.0-20230818184609-3a137a874352 h1:ge14PCmCvPjpMQMIAH7uKg0lrtNSOdpYsRXlwk3QbaE=
+github.com/digitorus/pkcs7 v0.0.0-20230818184609-3a137a874352/go.mod h1:SKVExuS+vpu2l9IoOc0RwqE7NYnb0JlcFHFnEJkVDzc=
+github.com/digitorus/timestamp v0.0.0-20231217203849-220c5c2851b7 h1:lxmTCgmHE1GUYL7P0MlNa00M67axePTq+9nBSGddR8I=
+github.com/digitorus/timestamp v0.0.0-20231217203849-220c5c2851b7/go.mod h1:GvWntX9qiTlOud0WkQ6ewFm0LPy5JUR1Xo0Ngbd1w6Y=
+github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
+github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
+github.com/docker/cli v25.0.4+incompatible h1:DatRkJ+nrFoYL2HZUzjM5Z5sAmcA5XGp+AW0oEw2+cA=
+github.com/docker/cli v25.0.4+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
+github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI=
+github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker-credential-helpers v0.8.1 h1:j/eKUktUltBtMzKqmfLB0PAgqYyMHOp5vfsD1807oKo=
+github.com/docker/docker-credential-helpers v0.8.1/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
-github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
-github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
-github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts=
-github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
-github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
-github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
-github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
-github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
-github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
+github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
+github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
+github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk=
+github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/emicklei/proto v1.13.2 h1:z/etSFO3uyXeuEsVPzfl56WNgzcvIr42aQazXaQmFZY=
+github.com/emicklei/proto v1.13.2/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
-github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
-github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
-github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34=
-github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI=
-github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q=
-github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo=
-github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w=
-github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss=
-github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss=
-github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs=
-github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE=
-github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw=
-github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI=
-github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro=
-github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
-github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
-github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
-github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
-github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
-github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
-github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
+github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
+github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
+github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
+github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
-github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
-github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
-github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI=
-github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4=
-github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
-github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
-github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
-github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
-github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI=
+github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
-github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
-github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
-github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
-github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
-github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
-github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
-github.com/go-acme/lego v2.5.0+incompatible/go.mod h1:yzMNe9CasVUhkquNvti5nAtPmG94USbYxYrZfTkIn0M=
-github.com/go-bindata/go-bindata v3.1.1+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
-github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
+github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
+github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
+github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
+github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
+github.com/gliderlabs/ssh v0.3.6 h1:ZzjlDa05TcFRICb3anf/dSPN3ewz1Zx6CMLPWgkm3b8=
+github.com/gliderlabs/ssh v0.3.6/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8=
+github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
+github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
-github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g=
-github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks=
-github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
-github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
-github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs=
-github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw=
-github.com/go-ini/ini v1.55.0 h1:0wVcG9udk2C3TGgmdIGKK9ScOZHZB5nbG+gwji9fhhc=
-github.com/go-ini/ini v1.55.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
-github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o=
-github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
-github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
-github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
-github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U=
-github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk=
-github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
-github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
+github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
+github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
+github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
+github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
+github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4=
+github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY=
+github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
+github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k=
+github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
-github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
-github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4=
github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
-github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
-github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
-github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
-github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
-github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
-github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU=
-github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ=
-github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk=
-github.com/go-openapi/analysis v0.20.0/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og=
-github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY=
-github.com/go-openapi/analysis v0.21.4 h1:ZDFLvSNxpDaomuCueM0BlSXxpANBlFYiBvr+GXrvIHc=
-github.com/go-openapi/analysis v0.21.4/go.mod h1:4zQ35W4neeZTqh3ol0rv/O8JBbka9QyAgQRPp9y3pfo=
-github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
-github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
-github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
-github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
-github.com/go-openapi/errors v0.19.4/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
-github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
-github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
-github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
-github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
-github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
-github.com/go-openapi/errors v0.20.4 h1:unTcVm6PispJsMECE3zWgvG4xTiKda1LIR5rCRWLG6M=
-github.com/go-openapi/errors v0.20.4/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk=
-github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
-github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
-github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
-github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
-github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
-github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
-github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
-github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
-github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
-github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
-github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns=
-github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo=
-github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
-github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
-github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
-github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
-github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
-github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
-github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs=
-github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI=
-github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk=
-github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY=
-github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc=
-github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc=
-github.com/go-openapi/loads v0.20.0/go.mod h1:2LhKquiE513rN5xC6Aan6lYOSddlL8Mp20AW9kpviM4=
-github.com/go-openapi/loads v0.20.2/go.mod h1:hTVUotJ+UonAMMZsvakEgmWKgtulweO9vYP2bQYKA/o=
-github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g=
-github.com/go-openapi/loads v0.21.2 h1:r2a/xFIYeZ4Qd2TnGpWDIQNcP80dIaZgf704za8enro=
-github.com/go-openapi/loads v0.21.2/go.mod h1:Jq58Os6SSGz0rzh62ptiu8Z31I+OTHqmULx5e/gJbNw=
-github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA=
-github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64=
-github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4=
-github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo=
-github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98=
-github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk=
-github.com/go-openapi/runtime v0.26.0 h1:HYOFtG00FM1UvqrcxbEJg/SwvDRvYLQKGhw2zaQjTcc=
-github.com/go-openapi/runtime v0.26.0/go.mod h1:QgRGeZwrUcSHdeh4Ka9Glvo0ug1LC5WyE+EV88plZrQ=
-github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
-github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
-github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
-github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
-github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
-github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
-github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU=
-github.com/go-openapi/spec v0.20.0/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU=
-github.com/go-openapi/spec v0.20.1/go.mod h1:93x7oh+d+FQsmsieroS4cmR3u0p/ywH649a3qwC9OsQ=
-github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg=
-github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I=
-github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA=
-github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8=
-github.com/go-openapi/spec v0.20.9/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA=
-github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
-github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
-github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
-github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
-github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
-github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
-github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
-github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc=
-github.com/go-openapi/strfmt v0.20.0/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc=
-github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg=
-github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k=
-github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg=
-github.com/go-openapi/strfmt v0.21.7 h1:rspiXgNWgeUzhjo1YU01do6qsahtJNByjLVbPLNHb8k=
-github.com/go-openapi/strfmt v0.21.7/go.mod h1:adeGTkxE44sPyLk0JV235VQAO/ZXUr8KAzYjclFs3ew=
-github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
-github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
-github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY=
-github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY=
-github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M=
-github.com/go-openapi/swag v0.19.13/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
-github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
-github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
-github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
-github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
-github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU=
-github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
-github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
-github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
-github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo=
-github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
-github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
-github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8=
-github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4=
-github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI=
-github.com/go-openapi/validate v0.20.1/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE9E4k54HpKcJ0=
-github.com/go-openapi/validate v0.20.2/go.mod h1:e7OJoKNgd0twXZwIn0A43tHbvIcr/rZIVCbJBpTUoY0=
-github.com/go-openapi/validate v0.22.1 h1:G+c2ub6q47kfX1sOBLwIQwzBVt8qmOAARyo/9Fqs9NU=
-github.com/go-openapi/validate v0.22.1/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg=
-github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
-github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M=
-github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M=
-github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
-github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
-github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
-github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
-github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
+github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
+github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU=
+github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo=
+github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w=
+github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE=
+github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
+github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
+github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
+github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=
+github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco=
+github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs=
+github.com/go-openapi/runtime v0.28.0 h1:gpPPmWSNGo214l6n8hzdXYhPuJcGtziTOgUpvsFWGIQ=
+github.com/go-openapi/runtime v0.28.0/go.mod h1:QN7OzcS+XuYmkQLw05akXk0jRH/eZ3kb18+1KwW9gyc=
+github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY=
+github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk=
+github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c=
+github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4=
+github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
+github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
+github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58=
+github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ=
+github.com/go-piv/piv-go v1.11.0 h1:5vAaCdRTFSIW4PeqMbnsDlUZ7odMYWnHBDGdmtU/Zhg=
+github.com/go-piv/piv-go v1.11.0/go.mod h1:NZ2zmjVkfFaL/CF8cVQ/pXdXtuj110zEKGdJM6fJZZM=
+github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
+github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
+github.com/go-rod/rod v0.114.7 h1:h4pimzSOUnw7Eo41zdJA788XsawzHjJMyzCE3BrBww0=
+github.com/go-rod/rod v0.114.7/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
-github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
-github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
-github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
-github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
-github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
-github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
-github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
-github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
-github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
-github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
-github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
-github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
-github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
-github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
-github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
-github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
-github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
-github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
-github.com/gobuffalo/logger v1.0.6 h1:nnZNpxYo0zx+Aj9RfMPBm+x9zAU2OayFh/xrAWi34HU=
-github.com/gobuffalo/logger v1.0.6/go.mod h1:J31TBEHR1QLV2683OXTAItYIg8pv2JMHnF/quuAbMjs=
-github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
-github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
-github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
-github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
-github.com/gobuffalo/packd v1.0.1 h1:U2wXfRr4E9DH8IdsDLlRFwTZTK7hLfq9qT/QHXGVe/0=
-github.com/gobuffalo/packd v1.0.1/go.mod h1:PP2POP3p3RXGz7Jh6eYEf93S7vA2za6xM7QT85L4+VY=
-github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
-github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
-github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XEWlY=
-github.com/gobuffalo/packr/v2 v2.8.3/go.mod h1:0SahksCVcx4IMnigTjiFuyldmTrdTctXsOdiU5KwbKc=
-github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
+github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
+github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
-github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
-github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
-github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
-github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
-github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
+github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
+github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
-github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
-github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4=
-github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
+github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw=
+github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
-github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4=
+github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
-github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
-github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -1492,431 +403,175 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
-github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho=
-github.com/golangplus/bytes v1.0.0/go.mod h1:AdRaCFwmc/00ZzELMWb01soso6W1R/++O1XL80yAn+A=
-github.com/golangplus/fmt v1.0.0/go.mod h1:zpM0OfbMCjPtd2qkTD/jX2MgiFCqklhSUFyDW44gVQE=
-github.com/golangplus/testing v1.0.0/go.mod h1:ZDreixUV3YzhoVraIDyOzHrr76p6NUh6k/pPg/Q3gYA=
-github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k=
-github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
-github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
-github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
-github.com/google/cadvisor v0.38.8/go.mod h1:1OFB9sOOMkBdUBGCO/1SArawTnDscgMzTodacVDe8mA=
-github.com/google/cadvisor v0.44.1/go.mod h1:GQ9KQfz0iNHQk3D6ftzJWK4TXabfIgM10Oy3FkR+Gzg=
-github.com/google/cel-go v0.17.7/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY=
-github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/certificate-transparency-go v1.1.8 h1:LGYKkgZF7satzgTak9R4yzfJXEeYVAjV6/EAEJOf1to=
+github.com/google/certificate-transparency-go v1.1.8/go.mod h1:bV/o8r0TBKRf1X//iiiSgWrvII4d7/8OiA+3vG26gI8=
+github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM=
github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
-github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ=
-github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
-github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
+github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 h1:0VpGH+cDhbDtdcweoyCVsF3fhN8kejK6rFe/2FFX2nU=
+github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49/go.mod h1:BkkQ4L1KS1xMt2aWSPStnn55ChGC0DPOn2FQYj+f25M=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/go-containerregistry v0.19.1 h1:yMQ62Al6/V0Z7CqIrrS1iYoA5/oQCm88DeNujc7C1KY=
+github.com/google/go-containerregistry v0.19.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI=
+github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=
+github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=
+github.com/google/go-github/v58 v58.0.0 h1:Una7GGERlF/37XfkPwpzYJe0Vp4dt2k1kCjlxwjIvzw=
+github.com/google/go-github/v58 v58.0.0/go.mod h1:k4hxDKEfoWpSqFlc8LTpGd9fu2KrV1YAa6Hi6FmDNY4=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
-github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
+github.com/google/gofuzz v1.2.1-0.20210504230335-f78f29fc09ea h1:VcIYpAGBae3Z6BVncE0OnTE/ZjlDXqtYhOZky88neLM=
+github.com/google/gofuzz v1.2.1-0.20210504230335-f78f29fc09ea/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/licenseclassifier/v2 v2.0.0 h1:1Y57HHILNf4m0ABuMVb6xk4vAJYEUO0gDxNpog0pyeA=
+github.com/google/licenseclassifier/v2 v2.0.0/go.mod h1:cOjbdH0kyC9R22sdQbYsFkto4NGCAc+ZSwbeThazEtM=
+github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210323184331-8eee2492667d/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM=
-github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
-github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
+github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
+github.com/google/tink/go v1.7.0 h1:6Eox8zONGebBFcCBqkVmt60LaWZa6xg1cl/DwAh/J1w=
+github.com/google/tink/go v1.7.0/go.mod h1:GAUOd+QE3pgj9q8VKIGTCP33c/B7eb4NhxLcgTJZStM=
+github.com/google/trillian v1.6.0 h1:jMBeDBIkINFvS2n6oV5maDqfRlxREAc6CW9QYWQ0qT4=
+github.com/google/trillian v1.6.0/go.mod h1:Yu3nIMITzNhhMJEHjAtp6xKiu+H/iHu2Oq5FjV2mCWI=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
-github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
-github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
-github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
-github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
-github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
-github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
-github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM=
-github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c=
-github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
-github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY=
-github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
-github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
-github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
-github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw=
-github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
-github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
-github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
-github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
-github.com/gophercloud/gophercloud v0.16.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4=
+github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
+github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
+github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA=
+github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0=
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4=
-github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
-github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
-github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo=
-github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM=
-github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
-github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
-github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
-github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
-github.com/hashicorp/consul/api v1.8.1/go.mod h1:sDjTOq0yUyv5G4h+BqSea7Fn6BU+XbolEz1952UB+mk=
-github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/consul/sdk v0.7.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
-github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
+github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M=
+github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
-github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
-github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=
-github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
-github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
-github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
-github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY=
-github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
-github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
-github.com/heketi/heketi v9.0.1-0.20190917153846-c2e2a4ab7ab9+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o=
-github.com/heketi/heketi v10.3.0+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o=
-github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4=
-github.com/hetznercloud/hcloud-go v1.24.0/go.mod h1:3YmyK8yaZZ48syie6xpm3dt26rtB6s65AisBHylXYFA=
+github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 h1:UpiO20jno/eV1eVZcxqWnUohyKRe1g8FPV/xH1s/2qs=
+github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8=
+github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
+github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
+github.com/hashicorp/go-sockaddr v1.0.5 h1:dvk7TIXCZpmfOlM+9mlcrWmWjw/wlKT+VDq2wMvfPJU=
+github.com/hashicorp/go-sockaddr v1.0.5/go.mod h1:uoUUmtwU7n9Dv3O4SNLeFvg0SxQ3lyjsj6+CCykpaxI=
+github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM=
+github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM=
+github.com/hashicorp/vault/api v1.10.0 h1:/US7sIjWN6Imp4o/Rj1Ce2Nr5bki/AXi9vAW3p2tOJQ=
+github.com/hashicorp/vault/api v1.10.0/go.mod h1:jo5Y/ET+hNyz+JnKDt8XLAdKs+AM0G5W0Vp1IrFI8N8=
+github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef h1:A9HsByNhogrvm9cWb28sjiS3i7tcKCkflWFEkHfuAgM=
+github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
-github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
-github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
-github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
-github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
-github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
+github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
+github.com/in-toto/in-toto-golang v0.9.0 h1:tHny7ac4KgtsfrG6ybU8gVOZux2H8jN05AXJ9EBM1XU=
+github.com/in-toto/in-toto-golang v0.9.0/go.mod h1:xsBVrVsHNsB61++S6Dy2vWosKhuA3lUTQd+eF9HdeMo=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY=
-github.com/influxdata/influxdb v1.8.4/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI=
-github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
-github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk=
-github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE=
-github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8=
-github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE=
-github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0=
-github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po=
-github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5/go.mod h1:DM4VvS+hD/kDi1U1QsX2fnZowwBhqD0Dk3bRPKF/Oc8=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
-github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 h1:TMtDYDHKYY15rFihtRfck/bfFqNfvcabqvXAFQfAUpY=
+github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267/go.mod h1:h1nSAbGFqGVzn6Jyl1R/iCcBUHN4g+gW1u9CoBTrb9E=
+github.com/jellydator/ttlcache/v3 v3.2.0 h1:6lqVJ8X3ZaUwvzENqPAobDsXNExfUJd61u++uW8a3LE=
+github.com/jellydator/ttlcache/v3 v3.2.0/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVKySzCVW6+0gA2n4=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
-github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
-github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
-github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
+github.com/jmhodges/clock v1.2.0 h1:eq4kys+NI0PLngzaHEe7AmPT90XMGIEySD1JfV1PDIs=
+github.com/jmhodges/clock v1.2.0/go.mod h1:qKjhA7x7u/lQpPB1XAqX1b1lCI/w3/fNuYpI/ZjLynI=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
-github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU=
-github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU=
-github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
-github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
-github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
-github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
-github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
-github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
-github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
+github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
-github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
-github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
-github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
-github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
-github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 h1:BcxbplxjtczA1a6d3wYoa7a0WL3rq9DKBMGHeKyjEF0=
-github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/knqyf263/go-rpmdb v0.1.0 h1:pOgjtOGtW0B+ibY905hP3ETrYFmLZsHiReKsplcs+to=
+github.com/knqyf263/go-rpmdb v0.1.0/go.mod h1:9LQcoMCMQ9vrF7HcDtXfvqGO4+ddxFQ8+YF/0CVGDww=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+github.com/kubeflow/common v0.4.6 h1:yzJf/HEdS6ginD0GlVkgbOFie0Sp66VdGjXidAGZIlk=
+github.com/kubeflow/common v0.4.6/go.mod h1:43MAof/uhpJA2C0urynqatE3oKFQc7m2HLmJty7waqY=
+github.com/kubeflow/mpi-operator v0.4.0 h1:PS4jLoMuRyrk/DHuYkI0D46sQQYpQt375HjOV4KVMFs=
+github.com/kubeflow/mpi-operator v0.4.0/go.mod h1:/A4mTy/RYh2UIgaGUiXUaW70eThjsogu80WbbcZpuMg=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
-github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
-github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
-github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw=
-github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/letsencrypt/boulder v0.0.0-20240318162201-5e68cbe552b9 h1:t+GNF6j5rgBKp7duiRWnIrwPzU22SkX7ZJJkX1+jTyM=
+github.com/letsencrypt/boulder v0.0.0-20240318162201-5e68cbe552b9/go.mod h1:ZpN5WFiDxRIR34bTEsaMzgRo4/JTuIikrl7cQqLp0BY=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc=
-github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
-github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
-github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
-github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
-github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
-github.com/lpabon/godbc v0.1.1/go.mod h1:Jo9QV0cf3U6jZABgiJ2skINAXb9j8m51r07g4KI92ZA=
-github.com/lucas-clemente/aes12 v0.0.0-20171027163421-cd47fb39b79f/go.mod h1:JpH9J1c9oX6otFSgdUHwUBUizmKlrMjxWnIAjff4m04=
-github.com/lucas-clemente/quic-clients v0.1.0/go.mod h1:y5xVIEoObKqULIKivu+gD/LU90pL73bTdtQjPBvtCBk=
-github.com/lucas-clemente/quic-go v0.10.2/go.mod h1:hvaRS9IHjFLMq76puFJeWNfmn+H70QZ/CXoxqw9bzao=
-github.com/lucas-clemente/quic-go-certificates v0.0.0-20160823095156-d2f86524cced/go.mod h1:NCcRLrOTZbzhZvixZLlERbJtDtYsmMw8Jc4vS8Z0g58=
-github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a h1:weJVJJRzAJBFRlAiJQROKQs8oC9vOxvm4rZmBBk0ONw=
-github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
-github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
-github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
-github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o=
-github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk=
-github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
-github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
-github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
-github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
+github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
+github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/manifoldco/promptui v0.8.0 h1:R95mMF+McvXZQ7j1g8ucVZE1gLP3Sv6j9vlF9kyRqQo=
-github.com/manifoldco/promptui v0.8.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ=
-github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI=
-github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc=
-github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
-github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY=
-github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI=
-github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI=
-github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
-github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
-github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
-github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
-github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
-github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
-github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
-github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
-github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
-github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
-github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
-github.com/mholt/archiver/v3 v3.3.0 h1:vWjhY8SQp5yzM9P6OJ/eZEkmi3UAbRrxCq48MxjAzig=
-github.com/mholt/archiver/v3 v3.3.0/go.mod h1:YnQtqsp+94Rwd0D/rk5cnLrxusUBUXg+08Ebtr1Mqao=
-github.com/mholt/certmagic v0.6.2-0.20190624175158-6a42ef9fe8c2/go.mod h1:g4cOPxcjV0oFq3qwpjSA30LReKD8AoIfwAY9VvG35NY=
-github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/dns v1.1.3/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
-github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
-github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
-github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY=
-github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
-github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
-github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
-github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
-github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
-github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
-github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
-github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
-github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
-github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
-github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4=
+github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY=
+github.com/miekg/pkcs11 v1.0.3-0.20190429190417-a667d056470f/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU=
+github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
-github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
-github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
-github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
-github.com/mitchellh/ioprogress v0.0.0-20180201004757-6a23b12fa88e h1:Qa6dnn8DlasdXRnacluu8HzPts0S1I9zvvUPDbBnXFI=
-github.com/mitchellh/ioprogress v0.0.0-20180201004757-6a23b12fa88e/go.mod h1:waEya8ee1Ro/lgxpVhkJI4BVASzkm3UZqkx/cFJiYHM=
-github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
+github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
-github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
-github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
-github.com/moby/ipvs v1.0.1/go.mod h1:2pngiyseZbIKXNv7hsKj3O9UEz30c53MT9005gt2hxQ=
-github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
-github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
-github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
-github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
-github.com/moby/sys/mountinfo v0.1.3/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o=
-github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
-github.com/moby/sys/mountinfo v0.6.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
-github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
-github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
-github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
-github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
-github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
-github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
-github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
+github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8=
+github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -1924,343 +579,167 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
-github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
-github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
-github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
-github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
-github.com/mrunalp/fileutils v0.0.0-20200520151820-abd8a0e76976/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0=
-github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
-github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
-github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mozillazg/docker-credential-acr-helper v0.3.0 h1:DVWFZ3/O8BP6Ue3iS/Olw+G07u1hCq1EOVCDZZjCIBI=
+github.com/mozillazg/docker-credential-acr-helper v0.3.0/go.mod h1:cZlu3tof523ujmLuiNUb6JsjtHcNA70u1jitrrdnuyA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU=
-github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
-github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
-github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
-github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=
-github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k=
-github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
-github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
-github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
-github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/nwaples/rardecode v1.0.0 h1:r7vGuS5akxOnR4JQSkko62RJ1ReCMXxQRPtxsiFMBOs=
-github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
+github.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481 h1:Up6+btDp321ZG5/zdSLo48H9Iaq0UQGthrhWC6pCxzE=
+github.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481/go.mod h1:yKZQO8QE2bHlgozqWDiRVqTFlLQSj30K/6SAK8EeYFw=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
+github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
github.com/octago/sflags v0.2.0 h1:XceYzkRXGAHa/lSFmKLcaxSrsh4MTuOMQdIGsUD0wlk=
github.com/octago/sflags v0.2.0/go.mod h1:G0bjdxh4qPRycF74a2B8pU36iTp9QHGx0w0dFZXPt80=
-github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
-github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
-github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
-github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
-github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
-github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/oleiade/reflections v1.0.1 h1:D1XO3LVEYroYskEsoSiGItp9RUxG6jWnCVvrqH0HHQM=
+github.com/oleiade/reflections v1.0.1/go.mod h1:rdFxbxq4QXVZWj0F+e9jqjDkc7dbp97vkRixKo2JR60=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
-github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU=
-github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk=
-github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0=
-github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
-github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw=
-github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo=
-github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc=
-github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk=
-github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo=
-github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts=
-github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
-github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0=
-github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
-github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
-github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY=
-github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw=
-github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg=
+github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
-github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
-github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc=
-github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM=
-github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
-github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
-github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
-github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw=
-github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw=
-github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ=
-github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
-github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4=
-github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ=
-github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
-github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
-github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
-github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
-github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/onsi/gomega v1.36.0 h1:Pb12RlruUtj4XUuPUqeEWc6j5DkVVVA49Uf6YLfC95Y=
+github.com/onsi/gomega v1.36.0/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
+github.com/open-policy-agent/opa v0.62.1 h1:UcxBQ0fe6NEjkYc775j4PWoUFFhx4f6yXKIKSTAuTVk=
+github.com/open-policy-agent/opa v0.62.1/go.mod h1:YqiSIIuvKwyomtnnXkJvy0E3KtVKbavjPJ/hNMuOmeM=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
-github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI=
-github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
-github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
-github.com/opencontainers/runc v1.0.0-rc92/go.mod h1:X1zlU4p7wOlX4+WRCz+hvlRv8phdL7UqbYD+vQwNMmE=
-github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc=
-github.com/opencontainers/runc v1.1.1/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc=
-github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.3-0.20200728170252-4d89ac9fbff6/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
-github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
-github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
-github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU=
-github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
-github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
-github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
-github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
+github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
-github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA=
-github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
-github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
-github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
-github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
-github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
-github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
-github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
-github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
-github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
-github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
-github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
-github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc=
-github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI=
-github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
-github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
-github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY=
-github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
-github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
-github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
-github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
-github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
-github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/package-url/packageurl-go v0.1.2 h1:0H2DQt6DHd/NeRlVwW4EZ4oEI6Bn40XlNPRqegcxuo4=
+github.com/package-url/packageurl-go v0.1.2/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
+github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
+github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
+github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
+github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
-github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
-github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
-github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
-github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY=
-github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg=
-github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI=
-github.com/prometheus/alertmanager v0.21.0/go.mod h1:h7tJ81NA0VLWvWEayi1QltevFkLF3KxmC/malTcT8Go=
-github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
-github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
-github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
-github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
-github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4=
-github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU=
-github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
-github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
-github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
-github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
-github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
+github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
-github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
-github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
-github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
-github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
-github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
-github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
-github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
-github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
-github.com/prometheus/common v0.20.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
-github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
-github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
-github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
-github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=
-github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
-github.com/prometheus/exporter-toolkit v0.5.1/go.mod h1:OCkM4805mmisBhLmVFw858QYi3v0wKdY6/UxrT0pZVg=
-github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
-github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
-github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
-github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
-github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
-github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
-github.com/prometheus/prometheus v1.8.2-0.20210331101223-3cafc58827d1/go.mod h1:sf7j/iAbhZahjeC0s3wwMmp5dksrJ/Za1UKdR+j6Hmw=
-github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/quobyte/api v0.1.8/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI=
-github.com/rasky/go-xdr v0.0.0-20170217172119-4930550ba2e2/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
-github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
-github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+github.com/protocolbuffers/txtpbfmt v0.0.0-20240116145035-ef3ab179eed6 h1:MAzmm+JtFxQwTPb1cVMLkemw2OxLy5AB/d/rxtAwGQQ=
+github.com/protocolbuffers/txtpbfmt v0.0.0-20240116145035-ef3ab179eed6/go.mod h1:jgxiZysxFPM+iWKwQwPR+y+Jvo54ARd4EisXxKYpB5c=
+github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
+github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
-github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc=
-github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
-github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
-github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
-github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
-github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
-github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
-github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
-github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
-github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0=
-github.com/rubenv/sql-migrate v1.5.2/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWxs+kmzlg0Is=
-github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto=
-github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w=
-github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk=
-github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
-github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210223165440-c65ae3540d44/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8=
-github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
-github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
-github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
-github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
-github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
+github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
+github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
+github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
+github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
+github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc=
+github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU=
+github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A=
+github.com/sassoftware/relic v7.2.1+incompatible/go.mod h1:CWfAxv73/iLZ17rbyhIEq3K9hs5w6FpNMdUT//qR+zk=
+github.com/sassoftware/relic/v7 v7.6.1 h1:O5s8ewCgq5QYNpv45dK4u6IpBmDM9RIcsbf/G1uXepQ=
+github.com/sassoftware/relic/v7 v7.6.1/go.mod h1:NxwtWxWxlUa9as2qZi635Ye6bBT/tGnMALLq7dSfOOU=
+github.com/secure-systems-lab/go-securesystemslib v0.8.0 h1:mr5An6X45Kb2nddcFlbmfHkLguCE9laoZCUzEEpIZXA=
+github.com/secure-systems-lab/go-securesystemslib v0.8.0/go.mod h1:UH2VZVuJfCYR8WgMlCU1uFsOUU+KeyrTWcSS73NBOzU=
+github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c=
+github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
-github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
-github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
-github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
-github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw=
-github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw=
-github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
+github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
+github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI=
+github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE=
+github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y=
+github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk=
+github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
+github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
+github.com/sigstore/cosign/v2 v2.2.3 h1:WX7yawI+EXu9h7S5bZsfYCbB9XW6Jc43ctKy/NoOSiA=
+github.com/sigstore/cosign/v2 v2.2.3/go.mod h1:WpMn4MBt0cI23GdHsePwO4NxhX1FOz1ITGB3ALUjFaI=
+github.com/sigstore/fulcio v1.4.4 h1:RjfymVe5t3a2CUBfLYo+7xEYuBusZa/XmFGxiYTsAqI=
+github.com/sigstore/fulcio v1.4.4/go.mod h1:yYtN6mvEFMSS/m7IM6+3rosUa30+0kgn4hIFbzZARZA=
+github.com/sigstore/rekor v1.3.5 h1:QoVXcS7NppKY+rpbEFVHr4evGDZBBSh65X0g8PXoUkQ=
+github.com/sigstore/rekor v1.3.5/go.mod h1:CWqOk/fmnPwORQmm7SyDgB54GTJizqobbZ7yOP1lvw8=
+github.com/sigstore/sigstore v1.8.2 h1:0Ttjcn3V0fVQXlYq7+oHaaHkGFIt3ywm7SF4JTU/l8c=
+github.com/sigstore/sigstore v1.8.2/go.mod h1:CHVcSyknCcjI4K2ZhS1SI28r0tcQyBlwtALG536x1DY=
+github.com/sigstore/sigstore/pkg/signature/kms/aws v1.8.1 h1:rEDdUefulkIQaMJyzLwtgPDLNXBIltBABiFYfb0YmgQ=
+github.com/sigstore/sigstore/pkg/signature/kms/aws v1.8.1/go.mod h1:RCdYCc1IxCYWzh2IdzdA6Yf7JIY0cMRqH08fpQYechw=
+github.com/sigstore/sigstore/pkg/signature/kms/azure v1.8.1 h1:DvRWG99QGWZC5mp42SEde2Xke/Q384Idnj2da7yB+Mk=
+github.com/sigstore/sigstore/pkg/signature/kms/azure v1.8.1/go.mod h1:s13mo3a0UCQS3+PAUUZfvKe48sMDMsHk2GE1b2YfPcU=
+github.com/sigstore/sigstore/pkg/signature/kms/gcp v1.8.1 h1:lwdRsJv1UbBemuk7w5YfXAQilQxMoFevrzamdPbG0wY=
+github.com/sigstore/sigstore/pkg/signature/kms/gcp v1.8.1/go.mod h1:2OaSQ80EcdyVRSQ3T4d1lsc6Scopblsiq8U2AEk5K1A=
+github.com/sigstore/sigstore/pkg/signature/kms/hashivault v1.8.1 h1:9Ki0qudKpc1FQdef7xHO2bkLyTuw+qNUpWRzjBEmF4c=
+github.com/sigstore/sigstore/pkg/signature/kms/hashivault v1.8.1/go.mod h1:nhIgyu4YwwNgalIwTGsoAzam16jjAn3ADRSWKbWPwGI=
+github.com/sigstore/timestamp-authority v1.2.2 h1:X4qyutnCQqJ0apMewFyx+3t7Tws00JQ/JonBiu3QvLE=
+github.com/sigstore/timestamp-authority v1.2.2/go.mod h1:nEah4Eq4wpliDjlY342rXclGSO7Kb9hoRrl9tqLW13A=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A=
+github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo=
+github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA=
+github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
+github.com/smallstep/assert v0.0.0-20200723003110-82e2b9b3b262 h1:unQFBIznI+VYD1/1fApl1A+9VcBk+9dcqGfnePY87LY=
+github.com/smallstep/assert v0.0.0-20200723003110-82e2b9b3b262/go.mod h1:MyOHs9Po2fbM1LHej6sBUT8ozbxmMOFG+E+rx/GSGuc=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0=
github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
-github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
-github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
-github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y=
-github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
-github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
-github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
-github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
-github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
-github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
-github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
-github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
-github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
-github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
+github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
+github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
+github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
+github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
+github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
+github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
+github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
-github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
-github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
-github.com/storageos/go-api v2.2.0+incompatible/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY=
-github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
-github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
-github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
+github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ=
+github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk=
+github.com/spiffe/go-spiffe/v2 v2.1.7 h1:VUkM1yIyg/x8X7u1uXqSRVRCdMdfRIEdFBzpqoeASGk=
+github.com/spiffe/go-spiffe/v2 v2.1.7/go.mod h1:QJDGdhXllxjxvd5B+2XnhhXB/+rC8gr+lNrtOryiWeE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -2268,1090 +747,285 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/thecodeteam/goscaleio v0.1.0/go.mod h1:68sdkZAsK8bvEwBlbQnlLS+xU+hvLYM/iQ8KXej1AwM=
-github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
-github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
-github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
-github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
-github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
-github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
-github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
-github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
-github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
-github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
-github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
-github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
-github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
-github.com/vmware/govmomi v0.30.6/go.mod h1:epgoslm97rLECMV4D+08ORzUBEU7boFSepKjt7AYVGg=
-github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk=
-github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
-github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
-github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
-github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
-github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
-github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
-github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
-github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
-github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
+github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs=
+github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48=
+github.com/tchap/go-patricia/v2 v2.3.1 h1:6rQp39lgIYZ+MHmdEq4xzuk1t7OdC35z/xm0BGhTkes=
+github.com/tchap/go-patricia/v2 v2.3.1/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k=
+github.com/thales-e-security/pool v0.0.2 h1:RAPs4q2EbWsTit6tpzuvTFlgFRJ3S8Evf5gtvVDbmPg=
+github.com/thales-e-security/pool v0.0.2/go.mod h1:qtpMm2+thHtqhLzTwgDBj/OuNnMpupY8mv0Phz0gjhU=
+github.com/theupdateframework/go-tuf v0.7.0 h1:CqbQFrWo1ae3/I0UCblSbczevCCbS31Qvs5LdxRWqRI=
+github.com/theupdateframework/go-tuf v0.7.0/go.mod h1:uEB7WSY+7ZIugK6R1hiBMBjQftaFzn7ZCDJcp1tCUug=
+github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C1wj2THlRK+oAhjeS/TRQwMfkIuet3w0=
+github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs=
+github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w=
+github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho=
+github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE=
+github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
+github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
+github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4=
+github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A=
+github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts=
+github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk=
+github.com/vladimirvivien/gexe v0.2.0 h1:nbdAQ6vbZ+ZNsolCgSVb9Fno60kzSuvtzVh6Ytqi/xY=
+github.com/vladimirvivien/gexe v0.2.0/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+github.com/xanzy/go-gitlab v0.100.0 h1:jaOtYj5nWI19+9oVVmgy233pax2oYqucwetogYU46ks=
+github.com/xanzy/go-gitlab v0.100.0/go.mod h1:ETg8tcj4OhrB84UEgeE8dSuV/0h4BBL1uOV/qK0vlyI=
+github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
+github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
-github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
-github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
-github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4=
-github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
-github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
-github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
-github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
-github.com/xlab/treeprint v1.0.0/go.mod h1:IoImgRak9i3zJyuxOKUP1v4UZd1tMoKkq/Cimt1uhCg=
-github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
-github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yashtewari/glob-intersection v0.2.0 h1:8iuHdN88yYuCzCdjt0gDe+6bAhUwBeEWqThExu54RFg=
+github.com/yashtewari/glob-intersection v0.2.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok=
+github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ=
+github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
+github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
+github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
+github.com/ysmood/got v0.34.1 h1:IrV2uWLs45VXNvZqhJ6g2nIhY+pgIG1CUoOcqfXFl1s=
+github.com/ysmood/got v0.34.1/go.mod h1:yddyjq/PmAf08RMLSwDjPyCvHvYed+WjHnQxpH851LM=
+github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
+github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
+github.com/ysmood/leakless v0.8.0 h1:BzLrVoiwxikpgEQR0Lk8NyBN5Cit2b1z+u0mgL4ZJak=
+github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI=
-github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
-github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE=
-github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
-github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f h1:ERexzlUfuTvpE74urLSbIQW0Z/6hF9t8U4NsJLaioAY=
-github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
-github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
-github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
-go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
-go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
-go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
-go.etcd.io/etcd v3.3.27+incompatible h1:5hMrpf6REqTHV2LW2OclNpRtxI0k9ZplMemJsMSWju0=
-go.etcd.io/etcd v3.3.27+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
-go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
-go.etcd.io/etcd/api/v3 v3.5.10 h1:szRajuUUbLyppkhs9K6BRtjY37l66XQQmw7oZRANE4k=
-go.etcd.io/etcd/api/v3 v3.5.10/go.mod h1:TidfmT4Uycad3NM/o25fG3J07odo4GBB9hoxaodFCtI=
-go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
-go.etcd.io/etcd/client/pkg/v3 v3.5.10 h1:kfYIdQftBnbAq8pUWFXfpuuxFSKzlmM5cSn76JByiT0=
-go.etcd.io/etcd/client/pkg/v3 v3.5.10/go.mod h1:DYivfIviIuQ8+/lCq4vcxuseg2P2XbHygkKwFo9fc8U=
-go.etcd.io/etcd/client/v2 v2.305.10/go.mod h1:m3CKZi69HzilhVqtPDcjhSGp+kA1OmbNn0qamH80xjA=
-go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q=
-go.etcd.io/etcd/client/v3 v3.5.10 h1:W9TXNZ+oB3MCd/8UjxHTWK5J9Nquw9fQBLJd5ne5/Ao=
-go.etcd.io/etcd/client/v3 v3.5.10/go.mod h1:RVeBnDz2PUEZqTpgqwAtUd8nAPf5kjyFyND7P1VkOKc=
-go.etcd.io/etcd/pkg/v3 v3.5.10/go.mod h1:TKTuCKKcF1zxmfKWDkfz5qqYaE3JncKKZPFf8c1nFUs=
-go.etcd.io/etcd/raft/v3 v3.5.10/go.mod h1:odD6kr8XQXTy9oQnyMPBOr0TVe+gT0neQhElQ6jbGRc=
-go.etcd.io/etcd/server/v3 v3.5.10/go.mod h1:gBplPHfs6YI0L+RpGkTQO7buDbHv5HJGG/Bst0/zIPo=
-go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M=
-go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
-go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
-go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
-go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
-go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
-go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
-go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
-go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
-go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg=
-go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng=
-go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8=
-go.mongodb.org/mongo-driver v1.11.3 h1:Ql6K6qYHEzB6xvu4+AU0BoRoqf9vFPcc4o7MUIdPW8Y=
-go.mongodb.org/mongo-driver v1.11.3/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
-go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
-go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
+github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+github.com/zalando/go-keyring v0.2.2 h1:f0xmpYiSrHtSNAVgwip93Cg8tuF45HJM6rHq/A5RI/4=
+github.com/zalando/go-keyring v0.2.2/go.mod h1:sI3evg9Wvpw3+n4SqplGSJUMwtDeROfD4nsFz4z9PG0=
+github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs=
+github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
+gitlab.alpinelinux.org/alpine/go v0.10.0 h1:/ekBiNqDSXZpK+AfZx4lrtVwKTDrWz3N3ck0S+fCxwU=
+gitlab.alpinelinux.org/alpine/go v0.10.0/go.mod h1:LKzOqYjGTZNLwcHl+c2I5VNioQio7agzRFvlGB9Owk4=
+go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80=
+go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
-go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
-go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU=
-go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4=
-go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI=
-go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
-go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
-go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I=
-go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
-go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4=
-go.opentelemetry.io/otel/metric v1.18.0/go.mod h1:nNSpsVDjWGfb7chbRLUNW+PBNdcSTHD4Uu5pfFMOI0k=
-go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
-go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
-go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
-go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
-go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
-go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI=
-go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A=
-go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
-go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
-go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
-go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk=
-go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0=
-go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0=
-go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
-go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
-go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
-go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
-go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg=
-go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
-go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
-go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
-go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg=
+go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
+go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
+go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ=
+go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q=
+go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s=
+go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE=
+go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg=
+go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g=
+go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
+go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
+go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY=
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds=
-go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
-go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
-go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
-go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
+go.step.sm/crypto v0.43.1 h1:18Z/M49SnFDPXvFbfoN/ugE1i0J7phLWARhSQs/XSDI=
+go.step.sm/crypto v0.43.1/go.mod h1:9n90D/SWjH1hTyQn1hgviUGyK8YRv743S8UZHYbt4BU=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
-go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
-go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
-go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
-golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
+golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
-golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
-golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20210220032938-85be41e4509f/go.mod h1:I6l2HNBLBZEcrOoCpyKLdY2lHoRZ8lI4x60KMCQDft4=
-golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
-golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
-golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
-golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
-golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
-golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
-golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
-golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
-golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
-golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
-golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
+golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
-golang.org/x/net v0.0.0-20210324051636-2c4c8ecb7826/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
-golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
-golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
-golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
-golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
-golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
-golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
-golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
-golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
-golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
-golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
-golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
-golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
-golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
-golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
-golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
-golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
-golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
-golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
-golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ=
-golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA=
+golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
+golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
-golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
+golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201110211018-35f3e6cf4a65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210314195730-07df6a141424/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
-golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
-golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo=
-golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
-golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
-golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
-golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
+golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
-golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
+golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
+golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=
+golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200513201620-d5fe73897c97/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
-golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
-golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
-golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
-golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
-golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
-golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
-golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
-golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
-golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
-golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM=
-golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
-golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA=
-golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
+golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ=
+golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=
+golang.org/x/tools/go/vcs v0.1.0-deprecated h1:cOIJqWBl99H1dH5LWizPa+0ImeeJq3t3cJjaeOWUAL4=
+golang.org/x/tools/go/vcs v0.1.0-deprecated/go.mod h1:zUrvATBAvEI9535oC0yWYsLsHIV4Z7g63sNPVMtuBy8=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
-golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
-golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
-gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
-gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
-gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
-gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU=
-gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU=
-gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
-gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0=
-gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA=
-gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
-gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
-gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ=
-gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
-gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY=
-gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo=
-google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.1-0.20200106000736-b8fc810ca6b5/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
-google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
-google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
-google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
-google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
-google.golang.org/api v0.42.0/go.mod h1:+Oj4s6ch2SEGtPjGqfUfZonBH0GjQH89gTeKKAEGZKI=
-google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
-google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I=
-google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo=
-google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4=
-google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw=
-google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU=
-google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k=
-google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
-google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
-google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
-google.golang.org/api v0.60.0/go.mod h1:d7rl65NZAkEQ90JFzqBjcRq1TVeG5ZoGV3sSpEnnVb4=
-google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
-google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo=
-google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g=
-google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
-google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
-google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
-google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
-google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
-google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
-google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
-google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
-google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g=
-google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
-google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
-google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI=
-google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s=
-google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s=
-google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s=
-google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08=
-google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70=
-google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo=
-google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0=
-google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
-google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
-google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
-google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
-google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0=
-google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg=
-google.golang.org/api v0.118.0/go.mod h1:76TtD3vkgmZ66zZzp72bUUklpmQmKlhh6sYtIjYK+5E=
-google.golang.org/api v0.122.0/go.mod h1:gcitW0lvnyWjSp9nKxAbdHKIZ6vF4aajGueeslZOyms=
-google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2BlP4=
-google.golang.org/api v0.125.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw=
-google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw=
+google.golang.org/api v0.170.0 h1:zMaruDePM88zxZBG+NG8+reALO2rfLhe/JShitLyT48=
+google.golang.org/api v0.170.0/go.mod h1:/xql9M2btF85xac/VAm4PsLMTLVGUOpq4BE9R8jyNy8=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
-google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
-google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
-google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210312152112-fc591d9ea70f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
-google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
-google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
-google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
-google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
-google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
-google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
-google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
-google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
-google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w=
-google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211021150943-2b146023228c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E=
-google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
-google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
-google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
-google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
-google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE=
-google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc=
-google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk=
-google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk=
-google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk=
-google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk=
-google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk=
-google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo=
-google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo=
-google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo=
-google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo=
-google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo=
-google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw=
-google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI=
-google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI=
-google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U=
-google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM=
-google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM=
-google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s=
-google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s=
-google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo=
-google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
-google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
-google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
-google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
-google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
-google.golang.org/genproto v0.0.0-20221201204527-e3fa12d562f3/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
-google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE=
-google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
-google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
-google.golang.org/genproto v0.0.0-20230112194545-e10362b5ecf9/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
-google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
-google.golang.org/genproto v0.0.0-20230123190316-2c411cf9d197/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
-google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
-google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
-google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
-google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
-google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
-google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
-google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
-google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA=
-google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
-google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
-google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
-google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
-google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
-google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
-google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
-google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY=
-google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk=
-google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk=
-google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
-google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
-google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:O9kGHb51iE/nOGvQaDUuadVYqovW56s5emA88lQnj6Y=
-google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0=
-google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8=
-google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8=
-google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
-google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
-google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
-google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
-google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ=
-google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ=
-google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ=
-google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4=
-google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE=
-google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
-google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
+google.golang.org/genproto v0.0.0-20240318140521-94a12d6c2237 h1:PgNlNSx2Nq2/j4juYzQBG0/Zdr+WP4z5N01Vk4VYBCY=
+google.golang.org/genproto v0.0.0-20240318140521-94a12d6c2237/go.mod h1:9sVD8c25Af3p0rGs7S7LLsxWKFiJt/65LdSyqXBkX/Y=
+google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw=
+google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
-google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
-google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
-google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
-google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
-google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
-google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
-google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
-google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
-google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
-google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
-google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
-google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
-google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
-google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
-google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
-google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
-google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
-google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
-google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
-google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
-google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
-google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
-google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
-google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk=
-google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE=
-google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
+google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
+google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -3360,232 +1034,118 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
-google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
+google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=
+gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE=
-gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
-gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
+gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs=
+gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
-gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/mcuadros/go-syslog.v2 v2.2.1/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U=
-gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
-gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
-gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
-gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
-gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
-gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
-gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
-gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
-gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
-helm.sh/helm/v3 v3.14.3 h1:HmvRJlwyyt9HjgmAuxHbHv3PhMz9ir/XNWHyXfmnOP4=
-helm.sh/helm/v3 v3.14.3/go.mod h1:v6myVbyseSBJTzhmeE39UcPLNv6cQK6qss3dvgAySaE=
-honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
+gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
-k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw=
-k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80=
-k8s.io/apiextensions-apiserver v0.29.3 h1:9HF+EtZaVpFjStakF4yVufnXGPRppWFEQ87qnO91YeI=
-k8s.io/apiextensions-apiserver v0.29.3/go.mod h1:po0XiY5scnpJfFizNGo6puNU6Fq6D70UJY2Cb2KwAVc=
-k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU=
-k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU=
-k8s.io/apiserver v0.29.3 h1:xR7ELlJ/BZSr2n4CnD3lfA4gzFivh0wwfNfz9L0WZcE=
-k8s.io/apiserver v0.29.3/go.mod h1:hrvXlwfRulbMbBgmWRQlFru2b/JySDpmzvQwwk4GUOs=
-k8s.io/cli-runtime v0.29.3 h1:r68rephmmytoywkw2MyJ+CxjpasJDQY7AGc3XY2iv1k=
-k8s.io/cli-runtime v0.29.3/go.mod h1:aqVUsk86/RhaGJwDhHXH0jcdqBrgdF3bZWk4Z9D4mkM=
-k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg=
-k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0=
-k8s.io/cloud-provider v0.29.3/go.mod h1:daDV1WkAO6pTrdsn7v8TpN/q9n75ExUC4RJDl7vlPKk=
-k8s.io/cluster-bootstrap v0.29.3/go.mod h1:aPAg1VtXx3uRrx5qU2jTzR7p1rf18zLXWS+pGhiqPto=
-k8s.io/code-generator v0.29.3/go.mod h1:x47ofBhN4gxYFcxeKA1PYXeaPreAGaDN85Y/lNUsPoM=
-k8s.io/component-base v0.29.3 h1:Oq9/nddUxlnrCuuR2K/jp6aflVvc0uDvxMzAWxnGzAo=
-k8s.io/component-base v0.29.3/go.mod h1:Yuj33XXjuOk2BAaHsIGHhCKZQAgYKhqIxIjIr2UXYio=
-k8s.io/component-helpers v0.29.3/go.mod h1:yiDqbRQrnQY+sPju/bL7EkwDJb6LVOots53uZNMZBos=
-k8s.io/controller-manager v0.29.3/go.mod h1:RNxpf0d1WAo59sOLd32isWJP0oZ7Zxr+q4VEEaSq4gk=
-k8s.io/cri-api v0.29.3/go.mod h1:3X7EnhsNaQnCweGhQCJwKNHlH7wHEYuKQ19bRvXMoJY=
-k8s.io/csi-translation-lib v0.29.3/go.mod h1:snAzieA58/oiQXQZr27b0+b6/3+ZzitwI+57cUsMKKQ=
-k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
-k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
-k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
-k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
-k8s.io/heapster v1.2.0-beta.1/go.mod h1:h1uhptVXMwC8xtZBYsPXKVi8fpdlYkTs6k949KozGrM=
+k8s.io/api v0.31.3 h1:umzm5o8lFbdN/hIXbrK9oRpOproJO62CV1zqxXrLgk8=
+k8s.io/api v0.31.3/go.mod h1:UJrkIp9pnMOI9K2nlL6vwpxRzzEX5sWgn8kGQe92kCE=
+k8s.io/apiextensions-apiserver v0.31.3 h1:+GFGj2qFiU7rGCsA5o+p/rul1OQIq6oYpQw4+u+nciE=
+k8s.io/apiextensions-apiserver v0.31.3/go.mod h1:2DSpFhUZZJmn/cr/RweH1cEVVbzFw9YBu4T+U3mf1e4=
+k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4=
+k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
+k8s.io/cli-runtime v0.28.3 h1:lvuJYVkwCqHEvpS6KuTZsUVwPePFjBfSGvuaLl2SxzA=
+k8s.io/cli-runtime v0.28.3/go.mod h1:jeX37ZPjIcENVuXDDTskG3+FnVuZms5D9omDXS/2Jjc=
+k8s.io/client-go v0.31.3 h1:CAlZuM+PH2cm+86LOBemaJI/lQ5linJ6UFxKX/SoG+4=
+k8s.io/client-go v0.31.3/go.mod h1:2CgjPUTpv3fE5dNygAr2NcM8nhHzXvxB8KL5gYc3kJs=
+k8s.io/cloud-provider v0.31.3 h1:7C3CHQUUwnv/HWWVIaibZH06iPg663RYQ6C6Zy4FnO8=
+k8s.io/cloud-provider v0.31.3/go.mod h1:c7csKppoVb9Ej6upJ28AvHy4B3BtlRMzXfgezsDdPKw=
+k8s.io/component-base v0.31.3 h1:DMCXXVx546Rfvhj+3cOm2EUxhS+EyztH423j+8sOwhQ=
+k8s.io/component-base v0.31.3/go.mod h1:xME6BHfUOafRgT0rGVBGl7TuSg8Z9/deT7qq6w7qjIU=
+k8s.io/csi-translation-lib v0.31.3 h1:hxcPRNdtEsk766jCXSKjgH1V8jUNx5tVqdooQ1Ars/M=
+k8s.io/csi-translation-lib v0.31.3/go.mod h1:0B1gQwd868XUIDwJYy5gB2jDXWEwlcWvSsfcQEgzbRk=
k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
-k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
-k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
-k8s.io/kms v0.29.3/go.mod h1:TBGbJKpRUMk59neTMDMddjIDL+D4HuFUbpuiuzmOPg0=
-k8s.io/kube-aggregator v0.29.3/go.mod h1:xGJqV/SJJ1fbwTGfQLAZfwgqX1EMoaqfotDTkDrqqSk=
-k8s.io/kube-controller-manager v0.29.3/go.mod h1:Tf+BQG52nu8f2Ra9he9MsNFJ2sK7U3TSVvmTB+OVgiw=
-k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
-k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk=
-k8s.io/kube-openapi v0.0.0-20230109183929-3758b55a6596/go.mod h1:/BYxry62FuDzmI+i9B+X2pqfySRmSOW2ARmj5Zbqhj0=
-k8s.io/kube-openapi v0.0.0-20230531092745-9b4dcd38a4bf/go.mod h1:l8HTwL5fqnlns4jOveW1L75eo7R9KFHxiE0bsPGy428=
-k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
-k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
-k8s.io/kube-proxy v0.29.3/go.mod h1:VyAG5i+d+crJ0WM6Ik/4wSq+VttAFDfD4n3z+0hEQl8=
-k8s.io/kube-scheduler v0.29.3/go.mod h1:1NLHViSwFFddWHH4U9UGD57clINAtje/PEs6PjOYQZg=
-k8s.io/kubectl v0.29.3 h1:RuwyyIU42MAISRIePaa8Q7A3U74Q9P4MoJbDFz9o3us=
-k8s.io/kubectl v0.29.3/go.mod h1:yCxfY1dbwgVdEt2zkJ6d5NNLOhhWgTyrqACIoFhpdd4=
-k8s.io/kubelet v0.29.3/go.mod h1:jDiGuTkFOUynyBKzOoC1xRSWlgAZ9UPcTYeFyjr6vas=
-k8s.io/kubernetes v1.20.12/go.mod h1:ITbWRVYhYMeehJRseDAcHSwdk1+LoOL7p35hjnBQfMs=
-k8s.io/kubernetes v1.24.3 h1:RVuivOFTujzUDYZV8UFXCeD05US8DowBbxV+97JWEFg=
-k8s.io/kubernetes v1.24.3/go.mod h1:8e8maMiZzBR2/8Po5Uulx+MXZUYJuN3vtKwD4Ct1Xi0=
-k8s.io/legacy-cloud-providers v0.29.3/go.mod h1:IFA6hqla2IgCPD14aF5uQ8+gXtqRiCLln7Khguu2idw=
-k8s.io/metrics v0.29.3/go.mod h1:kb3tGGC4ZcIDIuvXyUE291RwJ5WmDu0tB4wAVZM6h2I=
-k8s.io/mount-utils v0.29.3/go.mod h1:9IWJTMe8tG0MYMLEp60xK9GYVeCdA3g4LowmnVi+t9Y=
-k8s.io/perf-tests/clusterloader2 v0.0.0-20220805114947-bdcf75fa01d0 h1:OZavpq3kBtBjzSU3xyVTNh+IHAwQR72aR1glBP6HKrY=
-k8s.io/perf-tests/clusterloader2 v0.0.0-20220805114947-bdcf75fa01d0/go.mod h1:4MME66dJLDfWXtQgS3GRnGqFKyAlXz/ZFhHNABgh2Jg=
-k8s.io/pod-security-admission v0.29.3/go.mod h1:tipz/v8IXwAPKS5J4FEMAYBVhMfmSWs8/03Hwup+Po4=
-k8s.io/sample-apiserver v0.29.3/go.mod h1:Jrl7BuQhoasdW+FYMkZnikgPZBcYMiLjXh7QkwTTi18=
-k8s.io/system-validators v1.2.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q=
-k8s.io/system-validators v1.7.0/go.mod h1:gP1Ky+R9wtrSiFbrpEPwWMeYz9yqyy1S/KOh0Vci7WI=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
-k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
-lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
-modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
-modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
-modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
-modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
-modernc.org/cc/v3 v3.37.0/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20=
-modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
-modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc=
-modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw=
-modernc.org/ccgo/v3 v3.0.0-20220904174949-82d86e1b6d56/go.mod h1:YSXjPL62P2AMSxBphRHPn7IkzhVHqkvOnRKAKh+W6ZI=
-modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
-modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
-modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws=
-modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo=
-modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g=
-modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
-modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
-modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
-modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
-modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA=
-modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A=
-modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU=
-modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU=
-modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA=
-modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0=
-modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s=
-modernc.org/libc v1.17.4/go.mod h1:WNg2ZH56rDEwdropAJeZPQkXmDwh+JCA1s/htl6r2fA=
-modernc.org/libc v1.18.0/go.mod h1:vj6zehR5bfc98ipowQOM2nIDUZnVew/wNC/2tOGS+q0=
-modernc.org/libc v1.20.3/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0=
-modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI=
-modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug=
-modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
-modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
-modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
-modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
-modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw=
-modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw=
-modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
-modernc.org/memory v1.3.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
-modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
-modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
-modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
+k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
+k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
+k8s.io/release v0.16.5 h1:aITTxdJ0JwKDD5/cb3jNuCbb3MfQrdq9BdfEOh2SFmw=
+k8s.io/release v0.16.5/go.mod h1:PVmynXUd/jSgARt/DsTmG2la/MyIu5YvN1VeXtxiJtM=
+k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=
+k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+modernc.org/cc/v4 v4.19.3 h1:vE9kmJqUcyvNOf8F2Hn8od14SOMq34BiqcZ2tMzLk5c=
+modernc.org/cc/v4 v4.19.3/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
+modernc.org/ccgo/v4 v4.11.0 h1:2uc2kRvZLC/oHylsrirRW6f1I4wljQST2BBbm+aKiXM=
+modernc.org/ccgo/v4 v4.11.0/go.mod h1:GwrfAtnU6PdZkCWD4XI8wB1T5Xj3fSw9lO/40H1ldys=
+modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
+modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
+modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
+modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
+modernc.org/libc v1.45.2 h1:oRlBu8xlBen2awVAWuLOkvYNBPaIKFxFOj9wA/jaXHM=
+modernc.org/libc v1.45.2/go.mod h1:YkRHLoN4L70OdO1cVmM83KZhRbRvsc3XogfVzbTXBwE=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
+modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
+modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
-modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4=
-modernc.org/sqlite v1.18.2/go.mod h1:kvrTLEWgxUcHa2GfHBQtanR1H9ht3hTJNtKpzH9k1u0=
-modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
-modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
-modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
-modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw=
-modernc.org/tcl v1.13.2/go.mod h1:7CLiGIPo1M8Rv1Mitpv5akc2+8fxUd2y2UzC/MfMzy0=
-modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
+modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
+modernc.org/sqlite v1.29.5 h1:8l/SQKAjDtZFo9lkJLdk8g9JEOeYRG4/ghStDCCTiTE=
+modernc.org/sqlite v1.29.5/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U=
+modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
+modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
-modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8=
-oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY=
-oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y=
-sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0=
-sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s=
+sigs.k8s.io/bom v0.6.0 h1:IPMPHx6XdmMeW2oEeF66DgNyP5d4RxfuXwiC1qn+n9o=
+sigs.k8s.io/bom v0.6.0/go.mod h1:MV0D3vdGlkaPgi5EwpwMBeQ8n8QS8Q2u1lJ5LyE7RLM=
+sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw=
+sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM=
+sigs.k8s.io/e2e-framework v0.3.0 h1:eqQALBtPCth8+ulTs6lcPK7ytV5rZSSHJzQHZph4O7U=
+sigs.k8s.io/e2e-framework v0.3.0/go.mod h1:C+ef37/D90Dc7Xq1jQnNbJYscrUGpxrWog9bx2KIa+c=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
-sigs.k8s.io/kubetest2 v0.0.0-20220728001911-c76fb417aa01 h1:WYiH7R3KJw1+osbyp6AlZaE5EMmKFwFj5wRHSDDYyK8=
-sigs.k8s.io/kubetest2 v0.0.0-20220728001911-c76fb417aa01/go.mod h1:YPdzgDKlnoGYtZnH03w01YMSf6tpetPibc0xjNI3sOc=
-sigs.k8s.io/kustomize/api v0.13.4/go.mod h1:Bkaavz5RKK6ZzP0zgPrB7QbpbBJKiHuD3BB0KujY7Ls=
+sigs.k8s.io/karpenter v1.1.1 h1:QPpVC8DsaLgJ/YWcFpZKE4m3jD+Qp88/GtSPvMfffck=
+sigs.k8s.io/karpenter v1.1.1/go.mod h1:NQouOJNK6s1d4EIKa5cY7nAV3IG74qZ6gPzHBeCZNPw=
+sigs.k8s.io/kubetest2 v0.0.0-20240309080311-0d7ca9ccb41e h1:qvGAwPBj9Yi/XXYcMX3IQNN2IntLiz4ywruWpG+MQk4=
+sigs.k8s.io/kubetest2 v0.0.0-20240309080311-0d7ca9ccb41e/go.mod h1:0FsjmDUaeJjXDmIiNAYusNykIgkqouCX0cyOGEWOFkc=
sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0=
sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3/go.mod h1:9n16EZKMhXBNSiUC5kSdFQJkdH3zbxS/JoO619G1VAY=
-sigs.k8s.io/kustomize/cmd/config v0.11.2/go.mod h1:PCpHxyu10daTnbMfn3xhH1vppn7L8jsS3qpRKXb7Lkc=
-sigs.k8s.io/kustomize/kustomize/v5 v5.0.4-0.20230601165947-6ce0bf390ce3/go.mod h1:/d88dHCvoy7d0AKFT0yytezSGZKjsZBVs9YTkBHSGFk=
-sigs.k8s.io/kustomize/kyaml v0.14.2/go.mod h1:AN1/IpawKilWD7V+YvQwRGUvuUOOWpjsHu6uHwonSF4=
sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 h1:W6cLQc5pnqM7vh3b7HvGNfXrJ/xL6BDMS0v1V/HHg5U=
sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
+sigs.k8s.io/promo-tools/v3 v3.6.0 h1:C2L08ezrWm1aZI8Emd3iZPZQserLPRgzuqQVxvI0PUI=
+sigs.k8s.io/promo-tools/v3 v3.6.0/go.mod h1:XJ3jy0hJYs+hWKt8XsLHFzGQV8PUtvllvbxjN/E5RXI=
+sigs.k8s.io/release-sdk v0.11.0 h1:a+zjOO3tHm1NiVZgNcUWq5QrKmv7b63UZXw+XGdPGfk=
+sigs.k8s.io/release-sdk v0.11.0/go.mod h1:sjbFpskyVjCXcFBnI3Bj1iGQHGjDYPoHVyld/pT+TvU=
+sigs.k8s.io/release-utils v0.7.7 h1:JKDOvhCk6zW8ipEOkpTGDH/mW3TI+XqtPp16aaQ79FU=
+sigs.k8s.io/release-utils v0.7.7/go.mod h1:iU7DGVNi3umZJ8q6aHyUFzsDUIaYwNnNKGHo3YE5E3s=
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
-sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
-sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
+software.sslmate.com/src/go-pkcs12 v0.2.0 h1:nlFkj7bTysH6VkC4fGphtjXRbezREPgrHuJG20hBGPE=
+software.sslmate.com/src/go-pkcs12 v0.2.0/go.mod h1:23rNcYsMabIc1otwLpTkCCPwUq6kQsTyowttG/as0kQ=
diff --git a/hack/build.sh b/hack/build.sh
deleted file mode 100755
index e70e0040a..000000000
--- a/hack/build.sh
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-if ! [[ "$0" =~ hack/build.sh ]]; then
- echo "must be run from repository root"
- exit 255
-fi
-
-GIT_COMMIT=${GIT_COMMIT:-$(git rev-parse --short=12 HEAD || echo "GitNotFound")}
-RELEASE_VERSION=${RELEASE_VERSION:-v$(date -u '+%Y%m%d.%H%M%S')}
-BUILD_TIME=${BUILD_TIME:-$(date -u '+%Y-%m-%d_%H:%M:%S')}
-echo "GIT_COMMIT:" ${GIT_COMMIT}
-echo "RELEASE_VERSION:" ${RELEASE_VERSION}
-echo "BUILD_TIME:" ${BUILD_TIME}
-
-DEFAULT_ARCHS='amd64 arm64'
-DEFAULT_TARGETS='linux darwin'
-DEFAULT_WHAT='aws-k8s-tester cw-utils ec2-utils ecr-utils eks-utils etcd-utils s3-utils sts-utils'
-
-ARCHS=${ARCHS:-$DEFAULT_ARCHS}
-TARGETS=${TARGETS:-$DEFAULT_TARGETS}
-WHAT=${WHAT:-$DEFAULT_WHAT}
-
-echo ""
-echo "Usage: \`make TARGETS='linux' WHAT='aws-k8s-tester cw-utils'\`"
-echo "DEFAULT_ARCHS=$DEFAULT_ARCHS"
-echo "DEFAULT_TARGETS=$DEFAULT_TARGETS"
-echo "DEFAULT_WHAT=$DEFAULT_WHAT"
-echo ""
-
-mkdir -p ./bin
-
-PACKAGE_NAME='github.com/aws/aws-k8s-tester'
-for arch in ${ARCHS}; do
- for os in ${TARGETS}; do
- for bin in ${WHAT}; do
- echo "=== Building arch=${arch}, os=${os}, target=${bin} ==="
- CGO_ENABLED=0 \
- GOARCH=${arch} \
- GOOS=${os} \
- GOWORK=off \
- go build \
- -v \
- -ldflags "-s -w \
- -X ${PACKAGE_NAME}/version.GitCommit=${GIT_COMMIT} \
- -X ${PACKAGE_NAME}/version.ReleaseVersion=${RELEASE_VERSION} \
- -X ${PACKAGE_NAME}/version.BuildTime=${BUILD_TIME}" \
- -o ./bin/${bin}-${RELEASE_VERSION}-${os}-${arch} \
- ./cmd/${bin}
- done
- done
-done
-
-WHAT="k8s-tester"
-PACKAGE_NAME='github.com/aws/aws-k8s-tester/k8s-tester'
-pushd ./k8s-tester/cmd/k8s-tester
-for arch in ${ARCHS}; do
- for os in ${TARGETS}; do
- for bin in ${WHAT}; do
- echo "=== Building arch=${arch}, os=${os}, target=${bin} ==="
- CGO_ENABLED=0 \
- GOARCH=${arch} \
- GOOS=${os} \
- GOWORK=off \
- go build \
- -v \
- -ldflags "-s -w \
- -X ${PACKAGE_NAME}/version.GitCommit=${GIT_COMMIT} \
- -X ${PACKAGE_NAME}/version.ReleaseVersion=${RELEASE_VERSION} \
- -X ${PACKAGE_NAME}/version.BuildTime=${BUILD_TIME}" \
- -o ../../../bin/${bin}-${RELEASE_VERSION}-${os}-${arch} \
- .
- done
- done
-done
-popd
-
-echo ""
-echo "Success! Your shiny new binaries are ready."
-echo $(find ./bin -type f)
diff --git a/hack/ec2config.gen.sh b/hack/ec2config.gen.sh
deleted file mode 100755
index 9fe0fe220..000000000
--- a/hack/ec2config.gen.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-if ! [[ "$0" =~ hack/ec2config.gen.sh ]]; then
- echo "must be run from repository root"
- exit 255
-fi
-
-rm -f ec2config/README.md
-go run ec2config/gen/main.go
-cat ec2config/README.md
-
-go install -v ./cmd/ec2-utils
-ec2-utils create config --path ./ec2config/default.yaml
-rm -f ./ec2config/default.ssh.sh
diff --git a/hack/eksconfig.gen.sh b/hack/eksconfig.gen.sh
deleted file mode 100755
index cf8b00757..000000000
--- a/hack/eksconfig.gen.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-if ! [[ "$0" =~ hack/eksconfig.gen.sh ]]; then
- echo "must be run from repository root"
- exit 255
-fi
-
-rm -f eksconfig/README.md
-go run eksconfig/gen/main.go
-cat eksconfig/README.md
-
-go install -v ./cmd/aws-k8s-tester
-aws-k8s-tester eks create config --path ./eksconfig/default.yaml
-rm -f ./eksconfig/default.kubectl.sh
-rm -f ./eksconfig/default.ssh.sh
diff --git a/hack/fmt.sh b/hack/fmt.sh
deleted file mode 100755
index c32b827f4..000000000
--- a/hack/fmt.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-if ! [[ "$0" =~ hack/fmt.sh ]]; then
- echo "must be run from repository root"
- exit 255
-fi
-
-goimports -w ./cmd
-gofmt -s -w ./cmd
-
-goimports -w ./client
-gofmt -s -w ./client
-
-goimports -w ./e2e
-gofmt -s -w ./e2e
-
-goimports -w ./ec2
-gofmt -s -w ./ec2
-
-goimports -w ./ec2config
-gofmt -s -w ./ec2config
-
-goimports -w ./eks
-gofmt -s -w ./eks
-
-goimports -w ./eksconfig
-gofmt -s -w ./eksconfig
-
-goimports -w ./k8s-tester
-gofmt -s -w ./k8s-tester
-
-goimports -w ./pkg
-gofmt -s -w ./pkg
-
-goimports -w ./ssh
-gofmt -s -w ./ssh
-
-goimports -w ./utils
-gofmt -s -w ./utils
-
-goimports -w ./version
-gofmt -s -w ./version
diff --git a/e2e2/scripts/go-test.sh b/hack/go-test.sh
similarity index 100%
rename from e2e2/scripts/go-test.sh
rename to hack/go-test.sh
diff --git a/hack/install.sh b/hack/install.sh
deleted file mode 100755
index 3b9ac2d41..000000000
--- a/hack/install.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-if ! [[ "$0" =~ hack/install.sh ]]; then
- echo "must be run from repository root"
- exit 255
-fi
-
-go install -v ./cmd/...
diff --git a/hack/unit.sh b/hack/unit.sh
deleted file mode 100755
index ed88d7fe0..000000000
--- a/hack/unit.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-if ! [[ "$0" =~ hack/unit.sh ]]; then
- echo "must be run from repository root"
- exit 255
-fi
-
-make clean
-
-echo "Running fmt tests..."
-IGNORE_PKGS="(vendor)"
-FORMATTABLE=$(find . -name \*.go | while read -r a; do echo "$(dirname "$a")/*.go"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | sed "s|\./||g")
-FMT=($FORMATTABLE)
-
-function gofmt_pass {
- fmtRes=$(gofmt -l -s -d "${FMT[@]}")
- if [ -n "${fmtRes}" ]; then
- echo -e "gofmt checking failed:\\n${fmtRes}"
- exit 255
- fi
-}
-
-function govet_pass {
- vetRes=$(go vet ./...)
- if [ -n "${vetRes}" ]; then
- echo -e "govet checking failed:\\n${vetRes}"
- exit 255
- fi
-}
-
-gofmt_pass
-govet_pass
-
-echo "Running unit tests..."
-go test -v ./eksconfig/...
-go test -v -race ./eksconfig/...
-go test -v ./ec2config/...
-go test -v -race ./ec2config/...
diff --git a/hack/updatedep.sh b/hack/updatedep.sh
deleted file mode 100755
index 883a9a1fe..000000000
--- a/hack/updatedep.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-<
diff --git a/kubetest2/internal/awssdk/config.go b/internal/awssdk/config.go
similarity index 100%
rename from kubetest2/internal/awssdk/config.go
rename to internal/awssdk/config.go
diff --git a/kubetest2/internal/deployers/eksapi/addons.go b/internal/deployers/eksapi/addons.go
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/addons.go
rename to internal/deployers/eksapi/addons.go
diff --git a/kubetest2/internal/deployers/eksapi/auth_map_role.go b/internal/deployers/eksapi/auth_map_role.go
similarity index 68%
rename from kubetest2/internal/deployers/eksapi/auth_map_role.go
rename to internal/deployers/eksapi/auth_map_role.go
index cdf84af9a..8fe6089cc 100644
--- a/kubetest2/internal/deployers/eksapi/auth_map_role.go
+++ b/internal/deployers/eksapi/auth_map_role.go
@@ -3,15 +3,15 @@ package eksapi
import (
"bytes"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/deployers/eksapi/templates"
+ "github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates"
)
func generateAuthMapRole(nodeNameStrategy string, rolearn string) (string, error) {
template := templates.AuthMapRole
buf := bytes.Buffer{}
if err := template.Execute(&buf, templates.AuthMapRoleTemplateData{
- NodeNameStrategy: nodeNameStrategy,
- Rolearn: rolearn,
+ NodeNameStrategy: nodeNameStrategy,
+ Rolearn: rolearn,
}); err != nil {
return "", err
}
diff --git a/kubetest2/internal/deployers/eksapi/auth_map_role_test.go b/internal/deployers/eksapi/auth_map_role_test.go
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/auth_map_role_test.go
rename to internal/deployers/eksapi/auth_map_role_test.go
diff --git a/kubetest2/internal/deployers/eksapi/aws.go b/internal/deployers/eksapi/aws.go
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/aws.go
rename to internal/deployers/eksapi/aws.go
diff --git a/kubetest2/internal/deployers/eksapi/cluster.go b/internal/deployers/eksapi/cluster.go
similarity index 99%
rename from kubetest2/internal/deployers/eksapi/cluster.go
rename to internal/deployers/eksapi/cluster.go
index 1fccee6c4..89ec0dc91 100644
--- a/kubetest2/internal/deployers/eksapi/cluster.go
+++ b/internal/deployers/eksapi/cluster.go
@@ -6,7 +6,7 @@ import (
"fmt"
"time"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/util"
+ "github.com/aws/aws-k8s-tester/internal/util"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/eks"
ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types"
diff --git a/kubetest2/internal/deployers/eksapi/deployer.go b/internal/deployers/eksapi/deployer.go
similarity index 98%
rename from kubetest2/internal/deployers/eksapi/deployer.go
rename to internal/deployers/eksapi/deployer.go
index 13ee7a2b0..f67a8d20f 100644
--- a/kubetest2/internal/deployers/eksapi/deployer.go
+++ b/internal/deployers/eksapi/deployer.go
@@ -6,10 +6,10 @@ import (
"path/filepath"
"time"
- "github.com/aws/aws-k8s-tester/kubetest2/internal"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/awssdk"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/metrics"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/util"
+ "github.com/aws/aws-k8s-tester/internal"
+ "github.com/aws/aws-k8s-tester/internal/awssdk"
+ "github.com/aws/aws-k8s-tester/internal/metrics"
+ "github.com/aws/aws-k8s-tester/internal/util"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types"
diff --git a/kubetest2/internal/deployers/eksapi/infra.go b/internal/deployers/eksapi/infra.go
similarity index 98%
rename from kubetest2/internal/deployers/eksapi/infra.go
rename to internal/deployers/eksapi/infra.go
index 224a9c92e..71679ea57 100644
--- a/kubetest2/internal/deployers/eksapi/infra.go
+++ b/internal/deployers/eksapi/infra.go
@@ -21,8 +21,8 @@ import (
"github.com/aws/aws-sdk-go/aws/arn"
"k8s.io/klog"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/deployers/eksapi/templates"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/metrics"
+ "github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates"
+ "github.com/aws/aws-k8s-tester/internal/metrics"
)
const (
diff --git a/kubetest2/internal/deployers/eksapi/janitor.go b/internal/deployers/eksapi/janitor.go
similarity index 95%
rename from kubetest2/internal/deployers/eksapi/janitor.go
rename to internal/deployers/eksapi/janitor.go
index 745f4b4c8..720505630 100644
--- a/kubetest2/internal/deployers/eksapi/janitor.go
+++ b/internal/deployers/eksapi/janitor.go
@@ -7,8 +7,8 @@ import (
"strings"
"time"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/awssdk"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/metrics"
+ "github.com/aws/aws-k8s-tester/internal/awssdk"
+ "github.com/aws/aws-k8s-tester/internal/metrics"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/cloudformation"
cloudformationtypes "github.com/aws/aws-sdk-go-v2/service/cloudformation/types"
diff --git a/kubetest2/internal/deployers/eksapi/k8s.go b/internal/deployers/eksapi/k8s.go
similarity index 98%
rename from kubetest2/internal/deployers/eksapi/k8s.go
rename to internal/deployers/eksapi/k8s.go
index ce4084758..2fc6fd7d1 100644
--- a/kubetest2/internal/deployers/eksapi/k8s.go
+++ b/internal/deployers/eksapi/k8s.go
@@ -8,8 +8,8 @@ import (
"strings"
"time"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/metrics"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/util"
+ "github.com/aws/aws-k8s-tester/internal/metrics"
+ "github.com/aws/aws-k8s-tester/internal/util"
"github.com/aws/aws-sdk-go-v2/service/ec2"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
diff --git a/kubetest2/internal/deployers/eksapi/kubeconfig.go b/internal/deployers/eksapi/kubeconfig.go
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/kubeconfig.go
rename to internal/deployers/eksapi/kubeconfig.go
diff --git a/kubetest2/internal/deployers/eksapi/logs.go b/internal/deployers/eksapi/logs.go
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/logs.go
rename to internal/deployers/eksapi/logs.go
diff --git a/kubetest2/internal/deployers/eksapi/logs_ssm_doc.json b/internal/deployers/eksapi/logs_ssm_doc.json
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/logs_ssm_doc.json
rename to internal/deployers/eksapi/logs_ssm_doc.json
diff --git a/kubetest2/internal/deployers/eksapi/metrics.go b/internal/deployers/eksapi/metrics.go
similarity index 92%
rename from kubetest2/internal/deployers/eksapi/metrics.go
rename to internal/deployers/eksapi/metrics.go
index ace0e1639..2fd34c828 100644
--- a/kubetest2/internal/deployers/eksapi/metrics.go
+++ b/internal/deployers/eksapi/metrics.go
@@ -3,7 +3,7 @@ package eksapi
import (
"path"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/metrics"
+ "github.com/aws/aws-k8s-tester/internal/metrics"
cloudwatchtypes "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
)
diff --git a/kubetest2/internal/deployers/eksapi/node.go b/internal/deployers/eksapi/node.go
similarity index 99%
rename from kubetest2/internal/deployers/eksapi/node.go
rename to internal/deployers/eksapi/node.go
index 7a41242b1..5e8b68f2d 100644
--- a/kubetest2/internal/deployers/eksapi/node.go
+++ b/internal/deployers/eksapi/node.go
@@ -26,7 +26,7 @@ import (
"k8s.io/utils/pointer"
karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/deployers/eksapi/templates"
+ "github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates"
apierrors "k8s.io/apimachinery/pkg/api/errors"
)
diff --git a/kubetest2/internal/deployers/eksapi/static_cluster.go b/internal/deployers/eksapi/static_cluster.go
similarity index 98%
rename from kubetest2/internal/deployers/eksapi/static_cluster.go
rename to internal/deployers/eksapi/static_cluster.go
index 168a6665c..cfe5e81bf 100644
--- a/kubetest2/internal/deployers/eksapi/static_cluster.go
+++ b/internal/deployers/eksapi/static_cluster.go
@@ -8,7 +8,7 @@ import (
"strings"
"time"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/deployers/eksapi/templates"
+ "github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates"
v1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
diff --git a/kubetest2/internal/deployers/eksapi/templates/auth_map_role.yaml.template b/internal/deployers/eksapi/templates/auth_map_role.yaml.template
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/templates/auth_map_role.yaml.template
rename to internal/deployers/eksapi/templates/auth_map_role.yaml.template
diff --git a/kubetest2/internal/deployers/eksapi/templates/busybox_deployment.yaml.template b/internal/deployers/eksapi/templates/busybox_deployment.yaml.template
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/templates/busybox_deployment.yaml.template
rename to internal/deployers/eksapi/templates/busybox_deployment.yaml.template
diff --git a/kubetest2/internal/deployers/eksapi/templates/infra.yaml b/internal/deployers/eksapi/templates/infra.yaml
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/templates/infra.yaml
rename to internal/deployers/eksapi/templates/infra.yaml
diff --git a/kubetest2/internal/deployers/eksapi/templates/nvidia_static_cluster_nodepool.yaml.template b/internal/deployers/eksapi/templates/nvidia_static_cluster_nodepool.yaml.template
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/templates/nvidia_static_cluster_nodepool.yaml.template
rename to internal/deployers/eksapi/templates/nvidia_static_cluster_nodepool.yaml.template
diff --git a/kubetest2/internal/deployers/eksapi/templates/templates.go b/internal/deployers/eksapi/templates/templates.go
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/templates/templates.go
rename to internal/deployers/eksapi/templates/templates.go
diff --git a/kubetest2/internal/deployers/eksapi/templates/templates_test.go b/internal/deployers/eksapi/templates/templates_test.go
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/templates/templates_test.go
rename to internal/deployers/eksapi/templates/templates_test.go
diff --git a/kubetest2/internal/deployers/eksapi/templates/unmanaged-nodegroup-efa.yaml b/internal/deployers/eksapi/templates/unmanaged-nodegroup-efa.yaml
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/templates/unmanaged-nodegroup-efa.yaml
rename to internal/deployers/eksapi/templates/unmanaged-nodegroup-efa.yaml
diff --git a/kubetest2/internal/deployers/eksapi/templates/unmanaged-nodegroup.yaml.template b/internal/deployers/eksapi/templates/unmanaged-nodegroup.yaml.template
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/templates/unmanaged-nodegroup.yaml.template
rename to internal/deployers/eksapi/templates/unmanaged-nodegroup.yaml.template
diff --git a/kubetest2/internal/deployers/eksapi/templates/userdata_bootstrap.sh.mimepart.template b/internal/deployers/eksapi/templates/userdata_bootstrap.sh.mimepart.template
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/templates/userdata_bootstrap.sh.mimepart.template
rename to internal/deployers/eksapi/templates/userdata_bootstrap.sh.mimepart.template
diff --git a/kubetest2/internal/deployers/eksapi/templates/userdata_bottlerocket.toml.template b/internal/deployers/eksapi/templates/userdata_bottlerocket.toml.template
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/templates/userdata_bottlerocket.toml.template
rename to internal/deployers/eksapi/templates/userdata_bottlerocket.toml.template
diff --git a/kubetest2/internal/deployers/eksapi/templates/userdata_nodeadm.yaml.mimepart.template b/internal/deployers/eksapi/templates/userdata_nodeadm.yaml.mimepart.template
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/templates/userdata_nodeadm.yaml.mimepart.template
rename to internal/deployers/eksapi/templates/userdata_nodeadm.yaml.mimepart.template
diff --git a/kubetest2/internal/deployers/eksapi/userdata.go b/internal/deployers/eksapi/userdata.go
similarity index 92%
rename from kubetest2/internal/deployers/eksapi/userdata.go
rename to internal/deployers/eksapi/userdata.go
index ff7b7ef43..17f012dc2 100644
--- a/kubetest2/internal/deployers/eksapi/userdata.go
+++ b/internal/deployers/eksapi/userdata.go
@@ -5,7 +5,7 @@ import (
"fmt"
"text/template"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/deployers/eksapi/templates"
+ "github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates"
)
func generateUserData(format string, cluster *Cluster) (string, bool, error) {
diff --git a/kubetest2/internal/deployers/eksapi/userdata_test.go b/internal/deployers/eksapi/userdata_test.go
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/userdata_test.go
rename to internal/deployers/eksapi/userdata_test.go
diff --git a/kubetest2/internal/deployers/eksapi/vpccni.go b/internal/deployers/eksapi/vpccni.go
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/vpccni.go
rename to internal/deployers/eksapi/vpccni.go
diff --git a/kubetest2/internal/deployers/eksapi/vpccni_test.go b/internal/deployers/eksapi/vpccni_test.go
similarity index 100%
rename from kubetest2/internal/deployers/eksapi/vpccni_test.go
rename to internal/deployers/eksapi/vpccni_test.go
diff --git a/kubetest2/internal/deployers/eksctl/build.go b/internal/deployers/eksctl/build.go
similarity index 100%
rename from kubetest2/internal/deployers/eksctl/build.go
rename to internal/deployers/eksctl/build.go
diff --git a/kubetest2/internal/deployers/eksctl/cluster_config.go b/internal/deployers/eksctl/cluster_config.go
similarity index 100%
rename from kubetest2/internal/deployers/eksctl/cluster_config.go
rename to internal/deployers/eksctl/cluster_config.go
diff --git a/kubetest2/internal/deployers/eksctl/deployer.go b/internal/deployers/eksctl/deployer.go
similarity index 93%
rename from kubetest2/internal/deployers/eksctl/deployer.go
rename to internal/deployers/eksctl/deployer.go
index 5c82b7bf5..dabc25713 100644
--- a/kubetest2/internal/deployers/eksctl/deployer.go
+++ b/internal/deployers/eksctl/deployer.go
@@ -4,8 +4,8 @@ import (
"flag"
"path/filepath"
- "github.com/aws/aws-k8s-tester/kubetest2/internal"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/awssdk"
+ "github.com/aws/aws-k8s-tester/internal"
+ "github.com/aws/aws-k8s-tester/internal/awssdk"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/eks"
"github.com/octago/sflags/gen/gpflag"
diff --git a/kubetest2/internal/deployers/eksctl/down.go b/internal/deployers/eksctl/down.go
similarity index 85%
rename from kubetest2/internal/deployers/eksctl/down.go
rename to internal/deployers/eksctl/down.go
index 9b61f669f..d2ba2089a 100644
--- a/kubetest2/internal/deployers/eksctl/down.go
+++ b/internal/deployers/eksctl/down.go
@@ -1,7 +1,7 @@
package eksctl
import (
- "github.com/aws/aws-k8s-tester/kubetest2/internal/util"
+ "github.com/aws/aws-k8s-tester/internal/util"
"k8s.io/klog"
)
diff --git a/kubetest2/internal/deployers/eksctl/up.go b/internal/deployers/eksctl/up.go
similarity index 98%
rename from kubetest2/internal/deployers/eksctl/up.go
rename to internal/deployers/eksctl/up.go
index 174b96e25..15c47ca14 100644
--- a/kubetest2/internal/deployers/eksctl/up.go
+++ b/internal/deployers/eksctl/up.go
@@ -5,7 +5,7 @@ import (
"fmt"
"os"
- "github.com/aws/aws-k8s-tester/kubetest2/internal/util"
+ "github.com/aws/aws-k8s-tester/internal/util"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/eks"
ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types"
diff --git a/e2e2/internal/framework_extensions/client.go b/internal/e2e/client.go
similarity index 99%
rename from e2e2/internal/framework_extensions/client.go
rename to internal/e2e/client.go
index 2edf5df95..2ef549deb 100644
--- a/e2e2/internal/framework_extensions/client.go
+++ b/internal/e2e/client.go
@@ -1,4 +1,4 @@
-package frameworkext
+package e2e
import (
"bytes"
diff --git a/e2e2/internal/framework_extensions/conditions.go b/internal/e2e/conditions.go
similarity index 98%
rename from e2e2/internal/framework_extensions/conditions.go
rename to internal/e2e/conditions.go
index 3f7456bb4..114fda038 100644
--- a/e2e2/internal/framework_extensions/conditions.go
+++ b/internal/e2e/conditions.go
@@ -1,4 +1,4 @@
-package frameworkext
+package e2e
import (
"context"
diff --git a/e2e2/internal/framework_extensions/doc.go b/internal/e2e/doc.go
similarity index 77%
rename from e2e2/internal/framework_extensions/doc.go
rename to internal/e2e/doc.go
index c0f165f2b..71efa0718 100644
--- a/e2e2/internal/framework_extensions/doc.go
+++ b/internal/e2e/doc.go
@@ -1,2 +1,2 @@
// Package frameworkext contains extensions to sigs.k8s.io/e2e-framework
-package frameworkext
+package e2e
diff --git a/kubetest2/internal/metrics/cloudwatch.go b/internal/metrics/cloudwatch.go
similarity index 100%
rename from kubetest2/internal/metrics/cloudwatch.go
rename to internal/metrics/cloudwatch.go
diff --git a/kubetest2/internal/metrics/noop.go b/internal/metrics/noop.go
similarity index 100%
rename from kubetest2/internal/metrics/noop.go
rename to internal/metrics/noop.go
diff --git a/kubetest2/internal/metrics/registry.go b/internal/metrics/registry.go
similarity index 100%
rename from kubetest2/internal/metrics/registry.go
rename to internal/metrics/registry.go
diff --git a/kubetest2/internal/testers/ginkgov1/LICENSE.original b/internal/testers/ginkgov1/LICENSE.original
similarity index 100%
rename from kubetest2/internal/testers/ginkgov1/LICENSE.original
rename to internal/testers/ginkgov1/LICENSE.original
diff --git a/kubetest2/internal/testers/ginkgov1/README.md b/internal/testers/ginkgov1/README.md
similarity index 100%
rename from kubetest2/internal/testers/ginkgov1/README.md
rename to internal/testers/ginkgov1/README.md
diff --git a/kubetest2/internal/testers/ginkgov1/ginkgo.go b/internal/testers/ginkgov1/ginkgo.go
similarity index 100%
rename from kubetest2/internal/testers/ginkgov1/ginkgo.go
rename to internal/testers/ginkgov1/ginkgo.go
diff --git a/kubetest2/internal/testers/ginkgov1/kubectl/kubectl.go b/internal/testers/ginkgov1/kubectl/kubectl.go
similarity index 100%
rename from kubetest2/internal/testers/ginkgov1/kubectl/kubectl.go
rename to internal/testers/ginkgov1/kubectl/kubectl.go
diff --git a/kubetest2/internal/testers/ginkgov1/package.go b/internal/testers/ginkgov1/package.go
similarity index 100%
rename from kubetest2/internal/testers/ginkgov1/package.go
rename to internal/testers/ginkgov1/package.go
diff --git a/kubetest2/internal/testers/multi/cmd.go b/internal/testers/multi/cmd.go
similarity index 98%
rename from kubetest2/internal/testers/multi/cmd.go
rename to internal/testers/multi/cmd.go
index 41e50f26f..b50299586 100644
--- a/kubetest2/internal/testers/multi/cmd.go
+++ b/internal/testers/multi/cmd.go
@@ -7,7 +7,7 @@ import (
"path/filepath"
"strings"
- "github.com/aws/aws-k8s-tester/kubetest2/internal"
+ "github.com/aws/aws-k8s-tester/internal"
"github.com/octago/sflags/gen/gpflag"
"k8s.io/klog"
"sigs.k8s.io/kubetest2/pkg/app/shim"
diff --git a/kubetest2/internal/util/exec.go b/internal/util/exec.go
similarity index 100%
rename from kubetest2/internal/util/exec.go
rename to internal/util/exec.go
diff --git a/kubetest2/internal/util/http.go b/internal/util/http.go
similarity index 100%
rename from kubetest2/internal/util/http.go
rename to internal/util/http.go
diff --git a/kubetest2/internal/util/http_test.go b/internal/util/http_test.go
similarity index 100%
rename from kubetest2/internal/util/http_test.go
rename to internal/util/http_test.go
diff --git a/kubetest2/internal/util/lang.go b/internal/util/lang.go
similarity index 100%
rename from kubetest2/internal/util/lang.go
rename to internal/util/lang.go
diff --git a/kubetest2/internal/util/path.go b/internal/util/path.go
similarity index 100%
rename from kubetest2/internal/util/path.go
rename to internal/util/path.go
diff --git a/kubetest2/internal/util/version.go b/internal/util/version.go
similarity index 100%
rename from kubetest2/internal/util/version.go
rename to internal/util/version.go
diff --git a/kubetest2/internal/version.go b/internal/version.go
similarity index 100%
rename from kubetest2/internal/version.go
rename to internal/version.go
diff --git a/k8s-tester/Makefile b/k8s-tester/Makefile
deleted file mode 100644
index 32abe8ece..000000000
--- a/k8s-tester/Makefile
+++ /dev/null
@@ -1,33 +0,0 @@
-.PHONY: docker
-
-ACCOUNT_ID ?= $(shell aws sts get-caller-identity --query Account --output text)
-REGION ?= us-west-2
-ECR_HOST ?= amazonaws.com
-
-# build custom "busybox" image
-ORIGINAL_BUSYBOX_IMG ?= gcr.io/google-containers/busybox:latest
-ECR_BUSYBOX_IMG_NAME ?= busybox
-ECR_BUSYBOX_TAG ?= latest
-busybox:
- docker pull $(ORIGINAL_BUSYBOX_IMG)
- docker tag $(ORIGINAL_BUSYBOX_IMG) $(ACCOUNT_ID).dkr.ecr.$(REGION).$(ECR_HOST)/$(ECR_BUSYBOX_IMG_NAME):$(ECR_BUSYBOX_TAG)
- eval $$(aws ecr get-login --registry-ids $(ACCOUNT_ID) --no-include-email --region $(REGION))
- docker push $(ACCOUNT_ID).dkr.ecr.$(REGION).$(ECR_HOST)/$(ECR_BUSYBOX_IMG_NAME):$(ECR_BUSYBOX_TAG);
-
-# build custom "php-apache" image
-ECR_PHP_APACHE_IMG_NAME ?= php-apache
-ECR_PHP_APACHE_TAG ?= latest
-php-apache:
- docker build --network host -t $(ECR_PHP_APACHE_IMG_NAME):$(ECR_PHP_APACHE_TAG) ./php-apache
- docker tag $(ECR_PHP_APACHE_IMG_NAME):$(ECR_PHP_APACHE_TAG) $(ACCOUNT_ID).dkr.ecr.$(REGION).$(ECR_HOST)/$(ECR_PHP_APACHE_IMG_NAME):$(ECR_PHP_APACHE_TAG)
- eval $$(aws ecr get-login --registry-ids $(ACCOUNT_ID) --no-include-email --region $(REGION))
- docker push $(ACCOUNT_ID).dkr.ecr.$(REGION).$(ECR_HOST)/$(ECR_PHP_APACHE_IMG_NAME):$(ECR_PHP_APACHE_TAG);
-
-# build custom "stress" image
-ECR_K8S_TESTER_STRESS_IMG_NAME ?= k8s-tester-stress
-ECR_K8S_TESTER_STRESS_TAG ?= latest
-k8s-tester-stress:
- DOCKER_BUILDKIT=0 docker build --network host -t $(ECR_K8S_TESTER_STRESS_IMG_NAME):$(ECR_K8S_TESTER_STRESS_TAG) -f ./Dockerfile.k8s-tester-stress .
- docker tag $(ECR_K8S_TESTER_STRESS_IMG_NAME):$(ECR_K8S_TESTER_STRESS_TAG) $(ACCOUNT_ID).dkr.ecr.$(REGION).$(ECR_HOST)/$(ECR_K8S_TESTER_STRESS_IMG_NAME):$(ECR_K8S_TESTER_STRESS_TAG)
- eval $$(aws ecr get-login --registry-ids $(ACCOUNT_ID) --no-include-email --region $(REGION))
- docker push $(ACCOUNT_ID).dkr.ecr.$(REGION).$(ECR_HOST)/$(ECR_K8S_TESTER_STRESS_IMG_NAME):$(ECR_K8S_TESTER_STRESS_TAG);
diff --git a/k8s-tester/README.md b/k8s-tester/README.md
deleted file mode 100644
index 7f93effef..000000000
--- a/k8s-tester/README.md
+++ /dev/null
@@ -1,525 +0,0 @@
-
-`k8s-tester` implements defines Kubernetes "tester client" interface without "cluster provisioner" dependency. This replaces all test cases under `eks/*` (< `aws-k8s-tester` v1.6). The tester assumes an existing Kubernetes cluster (e.g., EKS, vanilla Kubernetes) and worker nodes to run testing components.
-
-Each test case:
- - MUST comply with `"github.com/aws/aws-k8s-tester/k8s-tester/tester.Tester"` interface
- - MUST be generic enough to run against any Kubernetes cluster on AWS
- - MUST implement clean-up in a non-destrutive way
- - MUST implement a package that can be easily imported as a library (e.g., integrates with EKS tester)
- - MUST control their own dependencies (e.g., vending Kubernetes client-go) in case a user does not want to carry out other dependencies
- - MAY require certain AWS API calls and assume correct IAM or instance role for required AWS actions
- - MAY implement a CLI with the sub-commands of "apply" and "delete"
-
-To add a new tester,
-- Create a new directory under `github.com/aws/aws-k8s-tester/k8s-tester`.
-- Implement [`github.com/aws/aws-k8s-tester/k8s-tester/tester.Tester`](https://pkg.go.dev/github.com/aws/aws-k8s-tester/k8s-tester/tester#Tester) interface within the new package `github.com/aws/aws-k8s-tester/k8s-tester/NEW-TESTER`.
-- (Optional) Implement a stand-alone CLI for the test case under `github.com/aws/aws-k8s-tester/k8s-tester/NEW-TESTER/cmd/k8s-tester-NEW-TESTER`.
-- Import the new configuration struct to `k8s-tester/config.go` with test cases in `k8s-tester/config_test.go`.
-- Add the new tester to `github.com/aws/aws-k8s-tester/k8s-tester/tester.go`.
-- Update `github.com/aws/aws-k8s-tester/k8s-tester/go.mod`.
-- Run `github.com/aws/aws-k8s-tester/k8s-tester/vend.sh`.
-- Add the new tester to `github.com/aws/aws-k8s-tester/k8s-tester/cmd/readme-gen/main.go`.
-- Update `github.com/aws/aws-k8s-tester/k8s-tester/cmd/readme-gen/go.mod`.
-- Run `github.com/aws/aws-k8s-tester/k8s-tester/cmd/readme-gen/vend.sh`.
-- Update and run `github.com/aws/aws-k8s-tester/k8s-tester/fmt.sh`.
-- Update `github.com/aws/aws-k8s-tester/k8s-tester/cmd/k8s-tester/go.mod`.
-- Run `github.com/aws/aws-k8s-tester/k8s-tester/cmd/k8s-tester/vend.sh`.
-- Run `github.com/aws/aws-k8s-tester/k8s-tester/gen.sh`.
-
-See example commits:
-- [`k8s-tester/clusterloader`](https://github.com/aws/aws-k8s-tester/commit/7b9113c21f440623ec01bdea5d81a74176100746).
-- [`k8s-tester/stress`](https://github.com/aws/aws-k8s-tester/commit/310f44bc0da12ca093b02f74680b34131d6283a6).
-- [`k8s-tester/stress/in-cluster`](https://github.com/aws/aws-k8s-tester/commit/e0b5fa0b0fb97851d86d268d093f4754617c638b).
-- [`k8s-tester/csrs`](https://github.com/aws/aws-k8s-tester/commit/90ef22a2e6505189f998d1f6ed738fe05f73d56d).
-- [`k8s-tester/falco`](https://github.com/aws/aws-k8s-tester/pull/221).
-- [`k8s-tester/nlb-guestbook`](https://github.com/aws/aws-k8s-tester/commit/6c985cfabff769c020d2f1f131c4106607fa5d95).
-- [`k8s-tester/wordpress`](https://github.com/aws/aws-k8s-tester/commit/b5a8f3e6533e199413269a27041aa70604318f57).
-- [`k8s-tester/prometheus-grafana`](https://github.com/aws/aws-k8s-tester/commit/TODO).
-- [`k8s-tester/jupyter-hub`](https://github.com/aws/aws-k8s-tester/commit/TODO).
-- [`k8s-tester/cni-vpc`](https://github.com/aws/aws-k8s-tester/commit/TODO).
-- [`k8s-tester/alb-2048`](https://github.com/aws/aws-k8s-tester/commit/TODO).
-- [`k8s-tester/fargate`](https://github.com/aws/aws-k8s-tester/commit/TODO).
-- [`k8s-tester/irsa`](https://github.com/aws/aws-k8s-tester/commit/TODO).
-- [`k8s-tester/irsa-fargate`](https://github.com/aws/aws-k8s-tester/commit/TODO).
-- [`k8s-tester/gpu`](https://github.com/aws/aws-k8s-tester/commit/TODO).
-- [`k8s-tester/cuda-vector-add`](https://github.com/aws/aws-k8s-tester/commit/TODO).
-- [`k8s-tester/app-mesh`](https://github.com/aws/aws-k8s-tester/commit/TODO).
-
-### Environmental variables
-
-Total 30 test cases!
-
-```
-*----------------------------------*----------------------*----------------------------------------*---------------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*----------------------------------*----------------------*----------------------------------------*---------------*
-| K8S_TESTER_PROMPT | SETTABLE VIA ENV VAR | *k8s_tester.Config.Prompt | bool |
-| K8S_TESTER_CLUSTER_NAME | SETTABLE VIA ENV VAR | *k8s_tester.Config.ClusterName | string |
-| K8S_TESTER_CONFIG_PATH | SETTABLE VIA ENV VAR | *k8s_tester.Config.ConfigPath | string |
-| K8S_TESTER_LOG_COLOR | SETTABLE VIA ENV VAR | *k8s_tester.Config.LogColor | bool |
-| K8S_TESTER_LOG_COLOR_OVERRIDE | SETTABLE VIA ENV VAR | *k8s_tester.Config.LogColorOverride | string |
-| K8S_TESTER_LOG_LEVEL | SETTABLE VIA ENV VAR | *k8s_tester.Config.LogLevel | string |
-| K8S_TESTER_LOG_OUTPUTS | SETTABLE VIA ENV VAR | *k8s_tester.Config.LogOutputs | []string |
-| K8S_TESTER_KUBECTL_DOWNLOAD_URL | SETTABLE VIA ENV VAR | *k8s_tester.Config.KubectlDownloadURL | string |
-| K8S_TESTER_KUBECTL_PATH | SETTABLE VIA ENV VAR | *k8s_tester.Config.KubectlPath | string |
-| K8S_TESTER_KUBECONFIG_PATH | SETTABLE VIA ENV VAR | *k8s_tester.Config.KubeconfigPath | string |
-| K8S_TESTER_KUBECONFIG_CONTEXT | SETTABLE VIA ENV VAR | *k8s_tester.Config.KubeconfigContext | string |
-| K8S_TESTER_CLIENTS | SETTABLE VIA ENV VAR | *k8s_tester.Config.Clients | int |
-| K8S_TESTER_CLIENT_QPS | SETTABLE VIA ENV VAR | *k8s_tester.Config.ClientQPS | float32 |
-| K8S_TESTER_CLIENT_BURST | SETTABLE VIA ENV VAR | *k8s_tester.Config.ClientBurst | int |
-| K8S_TESTER_CLIENT_TIMEOUT | SETTABLE VIA ENV VAR | *k8s_tester.Config.ClientTimeout | time.Duration |
-| K8S_TESTER_CLIENT_TIMEOUT_STRING | READ-ONLY | *k8s_tester.Config.ClientTimeoutString | string |
-| K8S_TESTER_MINIMUM_NODES | SETTABLE VIA ENV VAR | *k8s_tester.Config.MinimumNodes | int |
-| K8S_TESTER_TOTAL_NODES | READ-ONLY | *k8s_tester.Config.TotalNodes | int |
-*----------------------------------*----------------------*----------------------------------------*---------------*
-
-*--------------------------------------------------*----------------------*---------------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*--------------------------------------------------*----------------------*---------------------------------------*---------*
-| K8S_TESTER_ADD_ON_CLOUDWATCH_AGENT_ENABLE | SETTABLE VIA ENV VAR | *cloudwatch_agent.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_CLOUDWATCH_AGENT_REGION | SETTABLE VIA ENV VAR | *cloudwatch_agent.Config.Region | string |
-| K8S_TESTER_ADD_ON_CLOUDWATCH_AGENT_CLUSTER_NAME | READ-ONLY | *cloudwatch_agent.Config.ClusterName | string |
-| K8S_TESTER_ADD_ON_CLOUDWATCH_AGENT_MINIMUM_NODES | SETTABLE VIA ENV VAR | *cloudwatch_agent.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_CLOUDWATCH_AGENT_NAMESPACE | SETTABLE VIA ENV VAR | *cloudwatch_agent.Config.Namespace | string |
-*--------------------------------------------------*----------------------*---------------------------------------*---------*
-
-*--------------------------------------------*----------------------*---------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*--------------------------------------------*----------------------*---------------------------------*---------*
-| K8S_TESTER_ADD_ON_FLUENT_BIT_ENABLE | SETTABLE VIA ENV VAR | *fluent_bit.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_FLUENT_BIT_MINIMUM_NODES | SETTABLE VIA ENV VAR | *fluent_bit.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_FLUENT_BIT_NAMESPACE | SETTABLE VIA ENV VAR | *fluent_bit.Config.Namespace | string |
-*--------------------------------------------*----------------------*---------------------------------*---------*
-
-*------------------------------------------------*----------------------*-------------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*------------------------------------------------*----------------------*-------------------------------------*---------*
-| K8S_TESTER_ADD_ON_METRICS_SERVER_ENABLE | SETTABLE VIA ENV VAR | *metrics_server.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_METRICS_SERVER_MINIMUM_NODES | SETTABLE VIA ENV VAR | *metrics_server.Config.MinimumNodes | int |
-*------------------------------------------------*----------------------*-------------------------------------*---------*
-
-*------------------------------------------------*----------------------*-----------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*------------------------------------------------*----------------------*-----------------------------------*---------*
-| K8S_TESTER_ADD_ON_KUBECOST_ENABLE | SETTABLE VIA ENV VAR | *kubecost.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_KUBECOST_MINIMUM_NODES | SETTABLE VIA ENV VAR | *kubecost.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_KUBECOST_HELM_CHART_REPO_URL | SETTABLE VIA ENV VAR | *kubecost.Config.HelmChartRepoURL | string |
-| K8S_TESTER_ADD_ON_KUBECOST_NAMESPACE | SETTABLE VIA ENV VAR | *kubecost.Config.Namespace | string |
-*------------------------------------------------*----------------------*-----------------------------------*---------*
-
-*-------------------------------------*----------------------*--------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-------------------------------------*----------------------*--------------------------*---------*
-| K8S_TESTER_ADD_ON_CNI_ENABLE | SETTABLE VIA ENV VAR | *cni.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_CNI_MINIMUM_NODES | SETTABLE VIA ENV VAR | *cni.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_CNI_NAMESPACE | SETTABLE VIA ENV VAR | *cni.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_CNI_CNI_NAMESPACE | SETTABLE VIA ENV VAR | *cni.Config.CNINamespace | string |
-*-------------------------------------*----------------------*--------------------------*---------*
-
-*-------------------------------------------------------------------*----------------------*-----------------------------------------------------*---------------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-------------------------------------------------------------------*----------------------*-----------------------------------------------------*---------------*
-| K8S_TESTER_ADD_ON_CONFORMANCE_ENABLE | SETTABLE VIA ENV VAR | *conformance.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_CONFORMANCE_MINIMUM_NODES | SETTABLE VIA ENV VAR | *conformance.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_CONFORMANCE_NAMESPACE | SETTABLE VIA ENV VAR | *conformance.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_PATH | SETTABLE VIA ENV VAR | *conformance.Config.SonobuoyPath | string |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_DOWNLOAD_URL | SETTABLE VIA ENV VAR | *conformance.Config.SonobuoyDownloadURL | string |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_RUN_TIMEOUT | SETTABLE VIA ENV VAR | *conformance.Config.SonobuoyRunTimeout | time.Duration |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_RUN_TIMEOUT_STRING | READ-ONLY | *conformance.Config.SonobuoyRunTimeoutString | string |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_DELETE_TIMEOUT | SETTABLE VIA ENV VAR | *conformance.Config.SonobuoyDeleteTimeout | time.Duration |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_DELETE_TIMEOUT_STRING | READ-ONLY | *conformance.Config.SonobuoyDeleteTimeoutString | string |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_RUN_MODE | SETTABLE VIA ENV VAR | *conformance.Config.SonobuoyRunMode | string |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_RUN_E2E_FOCUS | SETTABLE VIA ENV VAR | *conformance.Config.SonobuoyRunE2EFocus | string |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_RUN_E2E_SKIP | SETTABLE VIA ENV VAR | *conformance.Config.SonobuoyRunE2ESkip | string |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_RUN_KUBE_CONFORMANCE_IMAGE | SETTABLE VIA ENV VAR | *conformance.Config.SonobuoyRunKubeConformanceImage | string |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_RUN_E2E_REPO_CONFIG | SETTABLE VIA ENV VAR | *conformance.Config.SonobuoyRunE2ERepoConfig | string |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_RUN_IMAGE | SETTABLE VIA ENV VAR | *conformance.Config.SonobuoyRunImage | string |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_RUN_SYSTEMD_LOGS_IMAGE | SETTABLE VIA ENV VAR | *conformance.Config.SonobuoyRunSystemdLogsImage | string |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_RESULTS_TAR_GZ_PATH | SETTABLE VIA ENV VAR | *conformance.Config.SonobuoyResultsTarGzPath | string |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_RESULTS_E2E_LOG_PATH | SETTABLE VIA ENV VAR | *conformance.Config.SonobuoyResultsE2ELogPath | string |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_RESULTS_JUNIT_XML_PATH | SETTABLE VIA ENV VAR | *conformance.Config.SonobuoyResultsJunitXMLPath | string |
-| K8S_TESTER_ADD_ON_CONFORMANCE_SONOBUOY_RESULTS_OUTPUT_DIR | SETTABLE VIA ENV VAR | *conformance.Config.SonobuoyResultsOutputDir | string |
-*-------------------------------------------------------------------*----------------------*-----------------------------------------------------*---------------*
-
-*-----------------------------------------------*----------------------*----------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-----------------------------------------------*----------------------*----------------------------------*---------*
-| K8S_TESTER_ADD_ON_CSI_EBS_ENABLE | SETTABLE VIA ENV VAR | *csi_ebs.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_CSI_EBS_MINIMUM_NODES | SETTABLE VIA ENV VAR | *csi_ebs.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_CSI_EBS_NAMESPACE | SETTABLE VIA ENV VAR | *csi_ebs.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_CSI_EBS_HELM_CHART_REPO_URL | SETTABLE VIA ENV VAR | *csi_ebs.Config.HelmChartRepoURL | string |
-*-----------------------------------------------*----------------------*----------------------------------*---------*
-
-*-----------------------------------------------*----------------------*----------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-----------------------------------------------*----------------------*----------------------------------*---------*
-| K8S_TESTER_ADD_ON_CSI_EFS_ENABLE | SETTABLE VIA ENV VAR | *csi_efs.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_CSI_EFS_MINIMUM_NODES | SETTABLE VIA ENV VAR | *csi_efs.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_CSI_EFS_NAMESPACE | SETTABLE VIA ENV VAR | *csi_efs.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_CSI_EFS_HELM_CHART_REPO_URL | SETTABLE VIA ENV VAR | *csi_efs.Config.HelmChartRepoURL | string |
-*-----------------------------------------------*----------------------*----------------------------------*---------*
-
-*------------------------------------------------------*----------------------*-------------------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*------------------------------------------------------*----------------------*-------------------------------------------*---------*
-| K8S_TESTER_ADD_ON_KUBERNETES_DASHBOARD_ENABLE | SETTABLE VIA ENV VAR | *kubernetes_dashboard.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_KUBERNETES_DASHBOARD_MINIMUM_NODES | SETTABLE VIA ENV VAR | *kubernetes_dashboard.Config.MinimumNodes | int |
-*------------------------------------------------------*----------------------*-------------------------------------------*---------*
-
-*---------------------------------------------*----------------------*--------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*---------------------------------------------*----------------------*--------------------------------*---------*
-| K8S_TESTER_ADD_ON_FALCO_ENABLE | SETTABLE VIA ENV VAR | *falco.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_FALCO_MINIMUM_NODES | SETTABLE VIA ENV VAR | *falco.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_FALCO_HELM_CHART_REPO_URL | SETTABLE VIA ENV VAR | *falco.Config.HelmChartRepoURL | string |
-| K8S_TESTER_ADD_ON_FALCO_NAMESPACE | SETTABLE VIA ENV VAR | *falco.Config.Namespace | string |
-*---------------------------------------------*----------------------*--------------------------------*---------*
-
-*-----------------------------------------------*----------------------*-----------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-----------------------------------------------*----------------------*-----------------------------------*---------*
-| K8S_TESTER_ADD_ON_FALCON_ENABLE | SETTABLE VIA ENV VAR | *falcon.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_FALCON_FALCON_CLIENT_ID | SETTABLE VIA ENV VAR | *falcon.Config.FalconClientId | string |
-| K8S_TESTER_ADD_ON_FALCON_FALCON_CLIENT_SECRET | SETTABLE VIA ENV VAR | *falcon.Config.FalconClientSecret | string |
-*-----------------------------------------------*----------------------*-----------------------------------*---------*
-
-*-------------------------------------------------------*----------------------*-------------------------------------------*-------------------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-------------------------------------------------------*----------------------*-------------------------------------------*-------------------*
-| K8S_TESTER_ADD_ON_PHP_APACHE_ENABLE | SETTABLE VIA ENV VAR | *php_apache.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_PHP_APACHE_MINIMUM_NODES | SETTABLE VIA ENV VAR | *php_apache.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_PHP_APACHE_NAMESPACE | SETTABLE VIA ENV VAR | *php_apache.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_PHP_APACHE_DEPLOYMENT_NODE_SELECTOR | SETTABLE VIA ENV VAR | *php_apache.Config.DeploymentNodeSelector | map[string]string |
-| K8S_TESTER_ADD_ON_PHP_APACHE_DEPLOYMENT_REPLICAS | SETTABLE VIA ENV VAR | *php_apache.Config.DeploymentReplicas | int32 |
-*-------------------------------------------------------*----------------------*-------------------------------------------*-------------------*
-
-*----------------------------------------------------*----------------------*---------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*----------------------------------------------------*----------------------*---------------------------*---------*
-| K8S_TESTER_ADD_ON_PHP_APACHE_REPOSITORY_PARTITION | SETTABLE VIA ENV VAR | *ecr.Repository.Partition | string |
-| K8S_TESTER_ADD_ON_PHP_APACHE_REPOSITORY_ACCOUNT_ID | SETTABLE VIA ENV VAR | *ecr.Repository.AccountID | string |
-| K8S_TESTER_ADD_ON_PHP_APACHE_REPOSITORY_REGION | SETTABLE VIA ENV VAR | *ecr.Repository.Region | string |
-| K8S_TESTER_ADD_ON_PHP_APACHE_REPOSITORY_NAME | SETTABLE VIA ENV VAR | *ecr.Repository.Name | string |
-| K8S_TESTER_ADD_ON_PHP_APACHE_REPOSITORY_IMAGE_TAG | SETTABLE VIA ENV VAR | *ecr.Repository.ImageTag | string |
-*----------------------------------------------------*----------------------*---------------------------*---------*
-
-*----------------------------------------------------------*----------------------*----------------------------------------------*-------------------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*----------------------------------------------------------*----------------------*----------------------------------------------*-------------------*
-| K8S_TESTER_ADD_ON_NLB_GUESTBOOK_ENABLE | SETTABLE VIA ENV VAR | *nlb_guestbook.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_NLB_GUESTBOOK_ACCOUNT_ID | READ-ONLY | *nlb_guestbook.Config.AccountID | string |
-| K8S_TESTER_ADD_ON_NLB_GUESTBOOK_PARTITION | SETTABLE VIA ENV VAR | *nlb_guestbook.Config.Partition | string |
-| K8S_TESTER_ADD_ON_NLB_GUESTBOOK_REGION | SETTABLE VIA ENV VAR | *nlb_guestbook.Config.Region | string |
-| K8S_TESTER_ADD_ON_NLB_GUESTBOOK_MINIMUM_NODES | SETTABLE VIA ENV VAR | *nlb_guestbook.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_NLB_GUESTBOOK_NAMESPACE | SETTABLE VIA ENV VAR | *nlb_guestbook.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_NLB_GUESTBOOK_DEPLOYMENT_NODE_SELECTOR | SETTABLE VIA ENV VAR | *nlb_guestbook.Config.DeploymentNodeSelector | map[string]string |
-| K8S_TESTER_ADD_ON_NLB_GUESTBOOK_DEPLOYMENT_REPLICAS | SETTABLE VIA ENV VAR | *nlb_guestbook.Config.DeploymentReplicas | int32 |
-| K8S_TESTER_ADD_ON_NLB_GUESTBOOK_ELB_ARN | READ-ONLY | *nlb_guestbook.Config.ELBARN | string |
-| K8S_TESTER_ADD_ON_NLB_GUESTBOOK_ELB_NAME | READ-ONLY | *nlb_guestbook.Config.ELBName | string |
-| K8S_TESTER_ADD_ON_NLB_GUESTBOOK_ELB_URL | READ-ONLY | *nlb_guestbook.Config.ELBURL | string |
-*----------------------------------------------------------*----------------------*----------------------------------------------*-------------------*
-
-*------------------------------------------------------------*----------------------*------------------------------------------------*-------------------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*------------------------------------------------------------*----------------------*------------------------------------------------*-------------------*
-| K8S_TESTER_ADD_ON_NLB_HELLO_WORLD_ENABLE | SETTABLE VIA ENV VAR | *nlb_hello_world.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_NLB_HELLO_WORLD_ACCOUNT_ID | READ-ONLY | *nlb_hello_world.Config.AccountID | string |
-| K8S_TESTER_ADD_ON_NLB_HELLO_WORLD_PARTITION | SETTABLE VIA ENV VAR | *nlb_hello_world.Config.Partition | string |
-| K8S_TESTER_ADD_ON_NLB_HELLO_WORLD_REGION | SETTABLE VIA ENV VAR | *nlb_hello_world.Config.Region | string |
-| K8S_TESTER_ADD_ON_NLB_HELLO_WORLD_MINIMUM_NODES | SETTABLE VIA ENV VAR | *nlb_hello_world.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_NLB_HELLO_WORLD_NAMESPACE | SETTABLE VIA ENV VAR | *nlb_hello_world.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_NLB_HELLO_WORLD_DEPLOYMENT_NODE_SELECTOR | SETTABLE VIA ENV VAR | *nlb_hello_world.Config.DeploymentNodeSelector | map[string]string |
-| K8S_TESTER_ADD_ON_NLB_HELLO_WORLD_DEPLOYMENT_REPLICAS | SETTABLE VIA ENV VAR | *nlb_hello_world.Config.DeploymentReplicas | int32 |
-| K8S_TESTER_ADD_ON_NLB_HELLO_WORLD_ELB_ARN | READ-ONLY | *nlb_hello_world.Config.ELBARN | string |
-| K8S_TESTER_ADD_ON_NLB_HELLO_WORLD_ELB_NAME | READ-ONLY | *nlb_hello_world.Config.ELBName | string |
-| K8S_TESTER_ADD_ON_NLB_HELLO_WORLD_ELB_URL | READ-ONLY | *nlb_hello_world.Config.ELBURL | string |
-*------------------------------------------------------------*----------------------*------------------------------------------------*-------------------*
-
-*-------------------------------------------*----------------------*--------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-------------------------------------------*----------------------*--------------------------------*---------*
-| K8S_TESTER_ADD_ON_WORDPRESS_ENABLE | SETTABLE VIA ENV VAR | *wordpress.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_WORDPRESS_ACCOUNT_ID | READ-ONLY | *wordpress.Config.AccountID | string |
-| K8S_TESTER_ADD_ON_WORDPRESS_PARTITION | SETTABLE VIA ENV VAR | *wordpress.Config.Partition | string |
-| K8S_TESTER_ADD_ON_WORDPRESS_REGION | SETTABLE VIA ENV VAR | *wordpress.Config.Region | string |
-| K8S_TESTER_ADD_ON_WORDPRESS_MINIMUM_NODES | SETTABLE VIA ENV VAR | *wordpress.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_WORDPRESS_NAMESPACE | SETTABLE VIA ENV VAR | *wordpress.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_WORDPRESS_USER_NAME | SETTABLE VIA ENV VAR | *wordpress.Config.UserName | string |
-| K8S_TESTER_ADD_ON_WORDPRESS_PASSWORD | SETTABLE VIA ENV VAR | *wordpress.Config.Password | string |
-| K8S_TESTER_ADD_ON_WORDPRESS_ELB_ARN | READ-ONLY | *wordpress.Config.ELBARN | string |
-| K8S_TESTER_ADD_ON_WORDPRESS_ELB_NAME | READ-ONLY | *wordpress.Config.ELBName | string |
-| K8S_TESTER_ADD_ON_WORDPRESS_ELB_URL | READ-ONLY | *wordpress.Config.ELBURL | string |
-*-------------------------------------------*----------------------*--------------------------------*---------*
-
-*---------------------------------------------*----------------------*--------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*---------------------------------------------*----------------------*--------------------------------*---------*
-| K8S_TESTER_ADD_ON_VAULT_ENABLE | SETTABLE VIA ENV VAR | *vault.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_VAULT_MINIMUM_NODES | SETTABLE VIA ENV VAR | *vault.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_VAULT_HELM_CHART_REPO_URL | SETTABLE VIA ENV VAR | *vault.Config.HelmChartRepoURL | string |
-| K8S_TESTER_ADD_ON_VAULT_NAMESPACE | SETTABLE VIA ENV VAR | *vault.Config.Namespace | string |
-*---------------------------------------------*----------------------*--------------------------------*---------*
-
-*-----------------------------------------*----------------------*------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-----------------------------------------*----------------------*------------------------------*---------*
-| K8S_TESTER_ADD_ON_JOBS_PI_ENABLE | SETTABLE VIA ENV VAR | *jobs_pi.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_JOBS_PI_MINIMUM_NODES | SETTABLE VIA ENV VAR | *jobs_pi.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_JOBS_PI_NAMESPACE | SETTABLE VIA ENV VAR | *jobs_pi.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_JOBS_PI_COMPLETES | SETTABLE VIA ENV VAR | *jobs_pi.Config.Completes | int32 |
-| K8S_TESTER_ADD_ON_JOBS_PI_PARALLELS | SETTABLE VIA ENV VAR | *jobs_pi.Config.Parallels | int32 |
-*-----------------------------------------*----------------------*------------------------------*---------*
-
-*-----------------------------------------------------------*----------------------*----------------------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-----------------------------------------------------------*----------------------*----------------------------------------------*---------*
-| K8S_TESTER_ADD_ON_JOBS_ECHO_ENABLE | SETTABLE VIA ENV VAR | *jobs_echo.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_JOBS_ECHO_MINIMUM_NODES | SETTABLE VIA ENV VAR | *jobs_echo.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_JOBS_ECHO_NAMESPACE | SETTABLE VIA ENV VAR | *jobs_echo.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_JOBS_ECHO_JOB_TYPE | SETTABLE VIA ENV VAR | *jobs_echo.Config.JobType | string |
-| K8S_TESTER_ADD_ON_JOBS_ECHO_COMPLETES | SETTABLE VIA ENV VAR | *jobs_echo.Config.Completes | int32 |
-| K8S_TESTER_ADD_ON_JOBS_ECHO_PARALLELS | SETTABLE VIA ENV VAR | *jobs_echo.Config.Parallels | int32 |
-| K8S_TESTER_ADD_ON_JOBS_ECHO_ECHO_SIZE | SETTABLE VIA ENV VAR | *jobs_echo.Config.EchoSize | int32 |
-| K8S_TESTER_ADD_ON_JOBS_ECHO_SCHEDULE | SETTABLE VIA ENV VAR | *jobs_echo.Config.Schedule | string |
-| K8S_TESTER_ADD_ON_JOBS_ECHO_SUCCESSFUL_JOBS_HISTORY_LIMIT | SETTABLE VIA ENV VAR | *jobs_echo.Config.SuccessfulJobsHistoryLimit | int32 |
-| K8S_TESTER_ADD_ON_JOBS_ECHO_FAILED_JOBS_HISTORY_LIMIT | SETTABLE VIA ENV VAR | *jobs_echo.Config.FailedJobsHistoryLimit | int32 |
-*-----------------------------------------------------------*----------------------*----------------------------------------------*---------*
-
-*---------------------------------------------------*----------------------*---------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*---------------------------------------------------*----------------------*---------------------------*---------*
-| K8S_TESTER_ADD_ON_JOBS_ECHO_REPOSITORY_PARTITION | SETTABLE VIA ENV VAR | *ecr.Repository.Partition | string |
-| K8S_TESTER_ADD_ON_JOBS_ECHO_REPOSITORY_ACCOUNT_ID | SETTABLE VIA ENV VAR | *ecr.Repository.AccountID | string |
-| K8S_TESTER_ADD_ON_JOBS_ECHO_REPOSITORY_REGION | SETTABLE VIA ENV VAR | *ecr.Repository.Region | string |
-| K8S_TESTER_ADD_ON_JOBS_ECHO_REPOSITORY_NAME | SETTABLE VIA ENV VAR | *ecr.Repository.Name | string |
-| K8S_TESTER_ADD_ON_JOBS_ECHO_REPOSITORY_IMAGE_TAG | SETTABLE VIA ENV VAR | *ecr.Repository.ImageTag | string |
-*---------------------------------------------------*----------------------*---------------------------*---------*
-
-*----------------------------------------------------------------*----------------------*----------------------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*----------------------------------------------------------------*----------------------*----------------------------------------------*---------*
-| K8S_TESTER_ADD_ON_CRON_JOBS_ECHO_ENABLE | SETTABLE VIA ENV VAR | *jobs_echo.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_CRON_JOBS_ECHO_MINIMUM_NODES | SETTABLE VIA ENV VAR | *jobs_echo.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_CRON_JOBS_ECHO_NAMESPACE | SETTABLE VIA ENV VAR | *jobs_echo.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_CRON_JOBS_ECHO_JOB_TYPE | SETTABLE VIA ENV VAR | *jobs_echo.Config.JobType | string |
-| K8S_TESTER_ADD_ON_CRON_JOBS_ECHO_COMPLETES | SETTABLE VIA ENV VAR | *jobs_echo.Config.Completes | int32 |
-| K8S_TESTER_ADD_ON_CRON_JOBS_ECHO_PARALLELS | SETTABLE VIA ENV VAR | *jobs_echo.Config.Parallels | int32 |
-| K8S_TESTER_ADD_ON_CRON_JOBS_ECHO_ECHO_SIZE | SETTABLE VIA ENV VAR | *jobs_echo.Config.EchoSize | int32 |
-| K8S_TESTER_ADD_ON_CRON_JOBS_ECHO_SCHEDULE | SETTABLE VIA ENV VAR | *jobs_echo.Config.Schedule | string |
-| K8S_TESTER_ADD_ON_CRON_JOBS_ECHO_SUCCESSFUL_JOBS_HISTORY_LIMIT | SETTABLE VIA ENV VAR | *jobs_echo.Config.SuccessfulJobsHistoryLimit | int32 |
-| K8S_TESTER_ADD_ON_CRON_JOBS_ECHO_FAILED_JOBS_HISTORY_LIMIT | SETTABLE VIA ENV VAR | *jobs_echo.Config.FailedJobsHistoryLimit | int32 |
-*----------------------------------------------------------------*----------------------*----------------------------------------------*---------*
-
-*--------------------------------------------------------*----------------------*---------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*--------------------------------------------------------*----------------------*---------------------------*---------*
-| K8S_TESTER_ADD_ON_CRON_JOBS_ECHO_REPOSITORY_PARTITION | SETTABLE VIA ENV VAR | *ecr.Repository.Partition | string |
-| K8S_TESTER_ADD_ON_CRON_JOBS_ECHO_REPOSITORY_ACCOUNT_ID | SETTABLE VIA ENV VAR | *ecr.Repository.AccountID | string |
-| K8S_TESTER_ADD_ON_CRON_JOBS_ECHO_REPOSITORY_REGION | SETTABLE VIA ENV VAR | *ecr.Repository.Region | string |
-| K8S_TESTER_ADD_ON_CRON_JOBS_ECHO_REPOSITORY_NAME | SETTABLE VIA ENV VAR | *ecr.Repository.Name | string |
-| K8S_TESTER_ADD_ON_CRON_JOBS_ECHO_REPOSITORY_IMAGE_TAG | SETTABLE VIA ENV VAR | *ecr.Repository.ImageTag | string |
-*--------------------------------------------------------*----------------------*---------------------------*---------*
-
-*-------------------------------------------------------*----------------------*------------------------------------------*-----------------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-------------------------------------------------------*----------------------*------------------------------------------*-----------------*
-| K8S_TESTER_ADD_ON_CSRS_ENABLE | SETTABLE VIA ENV VAR | *csrs.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_CSRS_MINIMUM_NODES | SETTABLE VIA ENV VAR | *csrs.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_CSRS_OBJECTS | SETTABLE VIA ENV VAR | *csrs.Config.Objects | int |
-| K8S_TESTER_ADD_ON_CSRS_INITIAL_REQUEST_CONDITION_TYPE | SETTABLE VIA ENV VAR | *csrs.Config.InitialRequestConditionType | string |
-| K8S_TESTER_ADD_ON_CSRS_LATENCY_SUMMARY | READ-ONLY | *csrs.Config.LatencySummary | latency.Summary |
-*-------------------------------------------------------*----------------------*------------------------------------------*-----------------*
-
-*----------------------------------------------*----------------------*-----------------------------------*-----------------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*----------------------------------------------*----------------------*-----------------------------------*-----------------*
-| K8S_TESTER_ADD_ON_CONFIGMAPS_ENABLE | SETTABLE VIA ENV VAR | *configmaps.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_CONFIGMAPS_MINIMUM_NODES | SETTABLE VIA ENV VAR | *configmaps.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_CONFIGMAPS_NAMESPACE | SETTABLE VIA ENV VAR | *configmaps.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_CONFIGMAPS_OBJECTS | SETTABLE VIA ENV VAR | *configmaps.Config.Objects | int |
-| K8S_TESTER_ADD_ON_CONFIGMAPS_OBJECT_SIZE | SETTABLE VIA ENV VAR | *configmaps.Config.ObjectSize | int |
-| K8S_TESTER_ADD_ON_CONFIGMAPS_LATENCY_SUMMARY | READ-ONLY | *configmaps.Config.LatencySummary | latency.Summary |
-*----------------------------------------------*----------------------*-----------------------------------*-----------------*
-
-*-------------------------------------------*----------------------*--------------------------------*-----------------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-------------------------------------------*----------------------*--------------------------------*-----------------*
-| K8S_TESTER_ADD_ON_SECRETS_ENABLE | SETTABLE VIA ENV VAR | *secrets.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_SECRETS_MINIMUM_NODES | SETTABLE VIA ENV VAR | *secrets.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_SECRETS_NAMESPACE | SETTABLE VIA ENV VAR | *secrets.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_SECRETS_OBJECTS | SETTABLE VIA ENV VAR | *secrets.Config.Objects | int |
-| K8S_TESTER_ADD_ON_SECRETS_OBJECT_SIZE | SETTABLE VIA ENV VAR | *secrets.Config.ObjectSize | int |
-| K8S_TESTER_ADD_ON_SECRETS_LATENCY_SUMMARY | READ-ONLY | *secrets.Config.LatencySummary | latency.Summary |
-*-------------------------------------------*----------------------*--------------------------------*-----------------*
-
-*-------------------------------------------------------------*----------------------*------------------------------------------------*------------------------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-------------------------------------------------------------*----------------------*------------------------------------------------*------------------------*
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_ENABLE | SETTABLE VIA ENV VAR | *clusterloader.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_MINIMUM_NODES | SETTABLE VIA ENV VAR | *clusterloader.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_CLUSTERLOADER_PATH | SETTABLE VIA ENV VAR | *clusterloader.Config.ClusterloaderPath | string |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_CLUSTERLOADER_DOWNLOAD_URL | SETTABLE VIA ENV VAR | *clusterloader.Config.ClusterloaderDownloadURL | string |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_PROVIDER | SETTABLE VIA ENV VAR | *clusterloader.Config.Provider | string |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_RUNS | SETTABLE VIA ENV VAR | *clusterloader.Config.Runs | int |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_RUN_TIMEOUT | SETTABLE VIA ENV VAR | *clusterloader.Config.RunTimeout | time.Duration |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_RUN_TIMEOUT_STRING | READ-ONLY | *clusterloader.Config.RunTimeoutString | string |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_CONFIG_PATH | SETTABLE VIA ENV VAR | *clusterloader.Config.TestConfigPath | string |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_RUN_FROM_CLUSTER | SETTABLE VIA ENV VAR | *clusterloader.Config.RunFromCluster | bool |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_NODES | SETTABLE VIA ENV VAR | *clusterloader.Config.Nodes | int |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_ENABLE_EXEC_SERVICE | SETTABLE VIA ENV VAR | *clusterloader.Config.EnableExecService | bool |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_REPORT_DIR | READ-ONLY | *clusterloader.Config.TestReportDir | string |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_REPORT_DIR_TAR_GZ_PATH | READ-ONLY | *clusterloader.Config.TestReportDirTarGzPath | string |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_LOG_PATH | READ-ONLY | *clusterloader.Config.TestLogPath | string |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_POD_STARTUP_LATENCY | READ-ONLY | *clusterloader.Config.PodStartupLatency | clusterloader.PerfData |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_POD_STARTUP_LATENCY_PATH | READ-ONLY | *clusterloader.Config.PodStartupLatencyPath | string |
-*-------------------------------------------------------------*----------------------*------------------------------------------------*------------------------*
-
-*----------------------------------------------------------------------------------*----------------------*-------------------------------------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*----------------------------------------------------------------------------------*----------------------*-------------------------------------------------------------*---------*
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_OVERRIDE_PATH | READ-ONLY | *clusterloader.TestOverride.Path | string |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_OVERRIDE_NODES_PER_NAMESPACE | SETTABLE VIA ENV VAR | *clusterloader.TestOverride.NodesPerNamespace | int |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_OVERRIDE_PODS_PER_NODE | SETTABLE VIA ENV VAR | *clusterloader.TestOverride.PodsPerNode | int |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_OVERRIDE_BIG_GROUP_SIZE | SETTABLE VIA ENV VAR | *clusterloader.TestOverride.BigGroupSize | int |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_OVERRIDE_MEDIUM_GROUP_SIZE | SETTABLE VIA ENV VAR | *clusterloader.TestOverride.MediumGroupSize | int |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_OVERRIDE_SMALL_GROUP_SIZE | SETTABLE VIA ENV VAR | *clusterloader.TestOverride.SmallGroupSize | int |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_OVERRIDE_SMALL_STATEFUL_SETS_PER_NAMESPACE | SETTABLE VIA ENV VAR | *clusterloader.TestOverride.SmallStatefulSetsPerNamespace | int |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_OVERRIDE_MEDIUM_STATEFUL_SETS_PER_NAMESPACE | SETTABLE VIA ENV VAR | *clusterloader.TestOverride.MediumStatefulSetsPerNamespace | int |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_OVERRIDE_CL2_USE_HOST_NETWORK_PODS | SETTABLE VIA ENV VAR | *clusterloader.TestOverride.CL2UseHostNetworkPods | bool |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_OVERRIDE_CL2_LOAD_TEST_THROUGHPUT | SETTABLE VIA ENV VAR | *clusterloader.TestOverride.CL2LoadTestThroughput | int |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_OVERRIDE_CL2_ENABLE_PVS | SETTABLE VIA ENV VAR | *clusterloader.TestOverride.CL2EnablePVS | bool |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_OVERRIDE_CL2_SCHEDULER_THROUGHPUT_THRESHOLD | SETTABLE VIA ENV VAR | *clusterloader.TestOverride.CL2SchedulerThroughputThreshold | int |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_OVERRIDE_PROMETHEUS_SCRAPE_KUBE_PROXY | SETTABLE VIA ENV VAR | *clusterloader.TestOverride.PrometheusScrapeKubeProxy | bool |
-| K8S_TESTER_ADD_ON_CLUSTERLOADER_TEST_OVERRIDE_ENABLE_SYSTEM_POD_METRICS | SETTABLE VIA ENV VAR | *clusterloader.TestOverride.EnableSystemPodMetrics | bool |
-*----------------------------------------------------------------------------------*----------------------*-------------------------------------------------------------*---------*
-
-*-----------------------------------------------------*----------------------*----------------------------------------*-----------------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-----------------------------------------------------*----------------------*----------------------------------------*-----------------*
-| K8S_TESTER_ADD_ON_STRESS_ENABLE | SETTABLE VIA ENV VAR | *stress.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_STRESS_MINIMUM_NODES | SETTABLE VIA ENV VAR | *stress.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_STRESS_NAMESPACE | SETTABLE VIA ENV VAR | *stress.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_STRESS_SKIP_NAMESPACE_CREATION | SETTABLE VIA ENV VAR | *stress.Config.SkipNamespaceCreation | bool |
-| K8S_TESTER_ADD_ON_STRESS_ECR_BUSYBOX_IMAGE | SETTABLE VIA ENV VAR | *stress.Config.ECRBusyboxImage | string |
-| K8S_TESTER_ADD_ON_STRESS_RUN_TIMEOUT | SETTABLE VIA ENV VAR | *stress.Config.RunTimeout | time.Duration |
-| K8S_TESTER_ADD_ON_STRESS_RUN_TIMEOUT_STRING | READ-ONLY | *stress.Config.RunTimeoutString | string |
-| K8S_TESTER_ADD_ON_STRESS_OBJECT_KEY_PREFIX | SETTABLE VIA ENV VAR | *stress.Config.ObjectKeyPrefix | string |
-| K8S_TESTER_ADD_ON_STRESS_OBJECTS | SETTABLE VIA ENV VAR | *stress.Config.Objects | int |
-| K8S_TESTER_ADD_ON_STRESS_OBJECT_SIZE | SETTABLE VIA ENV VAR | *stress.Config.ObjectSize | int |
-| K8S_TESTER_ADD_ON_STRESS_UPDATE_CONCURRENCY | SETTABLE VIA ENV VAR | *stress.Config.UpdateConcurrency | int |
-| K8S_TESTER_ADD_ON_STRESS_LIST_BATCH_LIMIT | SETTABLE VIA ENV VAR | *stress.Config.ListBatchLimit | int64 |
-| K8S_TESTER_ADD_ON_STRESS_LATENCY_SUMMARY_WRITES | READ-ONLY | *stress.Config.LatencySummaryWrites | latency.Summary |
-| K8S_TESTER_ADD_ON_STRESS_LATENCY_SUMMARY_GETS | READ-ONLY | *stress.Config.LatencySummaryGets | latency.Summary |
-| K8S_TESTER_ADD_ON_STRESS_LATENCY_SUMMARY_RANGE_GETS | READ-ONLY | *stress.Config.LatencySummaryRangeGets | latency.Summary |
-*-----------------------------------------------------*----------------------*----------------------------------------*-----------------*
-
-*------------------------------------------------*----------------------*---------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*------------------------------------------------*----------------------*---------------------------*---------*
-| K8S_TESTER_ADD_ON_STRESS_REPOSITORY_PARTITION | SETTABLE VIA ENV VAR | *ecr.Repository.Partition | string |
-| K8S_TESTER_ADD_ON_STRESS_REPOSITORY_ACCOUNT_ID | SETTABLE VIA ENV VAR | *ecr.Repository.AccountID | string |
-| K8S_TESTER_ADD_ON_STRESS_REPOSITORY_REGION | SETTABLE VIA ENV VAR | *ecr.Repository.Region | string |
-| K8S_TESTER_ADD_ON_STRESS_REPOSITORY_NAME | SETTABLE VIA ENV VAR | *ecr.Repository.Name | string |
-| K8S_TESTER_ADD_ON_STRESS_REPOSITORY_IMAGE_TAG | SETTABLE VIA ENV VAR | *ecr.Repository.ImageTag | string |
-*------------------------------------------------*----------------------*---------------------------*---------*
-
-*-------------------------------------------------------------------*----------------------*-----------------------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-------------------------------------------------------------------*----------------------*-----------------------------------------------*---------*
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_ENABLE | SETTABLE VIA ENV VAR | *in_cluster.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_MINIMUM_NODES | SETTABLE VIA ENV VAR | *in_cluster.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_NAMESPACE | SETTABLE VIA ENV VAR | *in_cluster.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_COMPLETES | SETTABLE VIA ENV VAR | *in_cluster.Config.Completes | int32 |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_PARALLELS | SETTABLE VIA ENV VAR | *in_cluster.Config.Parallels | int32 |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_SCHEDULE | SETTABLE VIA ENV VAR | *in_cluster.Config.Schedule | string |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_SUCCESSFUL_JOBS_HISTORY_LIMIT | SETTABLE VIA ENV VAR | *in_cluster.Config.SuccessfulJobsHistoryLimit | int32 |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_FAILED_JOBS_HISTORY_LIMIT | SETTABLE VIA ENV VAR | *in_cluster.Config.FailedJobsHistoryLimit | int32 |
-*-------------------------------------------------------------------*----------------------*-----------------------------------------------*---------*
-
-*-----------------------------------------------------------------------------*----------------------*---------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-----------------------------------------------------------------------------*----------------------*---------------------------*---------*
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_REPOSITORY_PARTITION | SETTABLE VIA ENV VAR | *ecr.Repository.Partition | string |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_REPOSITORY_ACCOUNT_ID | SETTABLE VIA ENV VAR | *ecr.Repository.AccountID | string |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_REPOSITORY_REGION | SETTABLE VIA ENV VAR | *ecr.Repository.Region | string |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_REPOSITORY_NAME | SETTABLE VIA ENV VAR | *ecr.Repository.Name | string |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_REPOSITORY_IMAGE_TAG | SETTABLE VIA ENV VAR | *ecr.Repository.ImageTag | string |
-*-----------------------------------------------------------------------------*----------------------*---------------------------*---------*
-
-*------------------------------------------------------------------------------*----------------------*--------------------------------------------------*---------------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*------------------------------------------------------------------------------*----------------------*--------------------------------------------------*---------------*
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_CLI_RUN_TIMEOUT | SETTABLE VIA ENV VAR | *in_cluster.K8sTesterStressCLI.RunTimeout | time.Duration |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_CLI_RUN_TIMEOUT_STRING | READ-ONLY | *in_cluster.K8sTesterStressCLI.RunTimeoutString | string |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_CLI_OBJECT_KEY_PREFIX | SETTABLE VIA ENV VAR | *in_cluster.K8sTesterStressCLI.ObjectKeyPrefix | string |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_CLI_OBJECTS | SETTABLE VIA ENV VAR | *in_cluster.K8sTesterStressCLI.Objects | int |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_CLI_OBJECT_SIZE | SETTABLE VIA ENV VAR | *in_cluster.K8sTesterStressCLI.ObjectSize | int |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_CLI_UPDATE_CONCURRENCY | SETTABLE VIA ENV VAR | *in_cluster.K8sTesterStressCLI.UpdateConcurrency | int |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_CLI_LIST_BATCH_LIMIT | SETTABLE VIA ENV VAR | *in_cluster.K8sTesterStressCLI.ListBatchLimit | int64 |
-*------------------------------------------------------------------------------*----------------------*--------------------------------------------------*---------------*
-
-*-----------------------------------------------------------------------------------------*----------------------*---------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-----------------------------------------------------------------------------------------*----------------------*---------------------------*---------*
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_CLI_BUSYBOX_REPOSITORY_PARTITION | SETTABLE VIA ENV VAR | *ecr.Repository.Partition | string |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_CLI_BUSYBOX_REPOSITORY_ACCOUNT_ID | SETTABLE VIA ENV VAR | *ecr.Repository.AccountID | string |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_CLI_BUSYBOX_REPOSITORY_REGION | SETTABLE VIA ENV VAR | *ecr.Repository.Region | string |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_CLI_BUSYBOX_REPOSITORY_NAME | SETTABLE VIA ENV VAR | *ecr.Repository.Name | string |
-| K8S_TESTER_ADD_ON_STRESS_IN_CLUSTER_K8S_TESTER_STRESS_CLI_BUSYBOX_REPOSITORY_IMAGE_TAG | SETTABLE VIA ENV VAR | *ecr.Repository.ImageTag | string |
-*-----------------------------------------------------------------------------------------*----------------------*---------------------------*---------*
-
-*--------------------------------------------*----------------------*-------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*--------------------------------------------*----------------------*-------------------------------*---------*
-| K8S_TESTER_ADD_ON_AQUA_ENABLE | SETTABLE VIA ENV VAR | *aqua.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_AQUA_MINIMUM_NODES | SETTABLE VIA ENV VAR | *aqua.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_AQUA_HELM_CHART_REPO_URL | SETTABLE VIA ENV VAR | *aqua.Config.HelmChartRepoURL | string |
-| K8S_TESTER_ADD_ON_AQUA_NAMESPACE | SETTABLE VIA ENV VAR | *aqua.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_AQUA_AQUA_LICENSE | SETTABLE VIA ENV VAR | *aqua.Config.AquaLicense | string |
-| K8S_TESTER_ADD_ON_AQUA_AQUA_USERNAME | SETTABLE VIA ENV VAR | *aqua.Config.AquaUsername | string |
-| K8S_TESTER_ADD_ON_AQUA_AQUA_PASSWORD | SETTABLE VIA ENV VAR | *aqua.Config.AquaPassword | string |
-*--------------------------------------------*----------------------*-------------------------------*---------*
-
-*----------------------------------------------*----------------------*---------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*----------------------------------------------*----------------------*---------------------------------*---------*
-| K8S_TESTER_ADD_ON_ARMORY_ENABLE | SETTABLE VIA ENV VAR | *armory.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_ARMORY_MINIMUM_NODES | SETTABLE VIA ENV VAR | *armory.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_ARMORY_HELM_CHART_REPO_URL | SETTABLE VIA ENV VAR | *armory.Config.HelmChartRepoURL | string |
-| K8S_TESTER_ADD_ON_ARMORY_NAMESPACE | SETTABLE VIA ENV VAR | *armory.Config.Namespace | string |
-*----------------------------------------------*----------------------*---------------------------------*---------*
-
-*-----------------------------------------------*----------------------*-----------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*-----------------------------------------------*----------------------*-----------------------------------*---------*
-| K8S_TESTER_ADD_ON_EPSAGON_ENABLE | SETTABLE VIA ENV VAR | *epsagon.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_EPSAGON_MINIMUM_NODES | SETTABLE VIA ENV VAR | *epsagon.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_EPSAGON_HELM_CHART_REPO_URL | SETTABLE VIA ENV VAR | *epsagon.Config.HelmChartRepoURL | string |
-| K8S_TESTER_ADD_ON_EPSAGON_NAMESPACE | SETTABLE VIA ENV VAR | *epsagon.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_EPSAGON_COLLECTOR_ENDPOINT | SETTABLE VIA ENV VAR | *epsagon.Config.CollectorEndpoint | string |
-| K8S_TESTER_ADD_ON_EPSAGON_API_TOKEN | SETTABLE VIA ENV VAR | *epsagon.Config.APIToken | string |
-| K8S_TESTER_ADD_ON_EPSAGON_CLUSTER_NAME | SETTABLE VIA ENV VAR | *epsagon.Config.ClusterName | string |
-*-----------------------------------------------*----------------------*-----------------------------------*---------*
-
-*----------------------------------------------*----------------------*----------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*----------------------------------------------*----------------------*----------------------------------*---------*
-| K8S_TESTER_ADD_ON_SYSDIG_ENABLE | SETTABLE VIA ENV VAR | *sysdig.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_SYSDIG_MINIMUM_NODES | SETTABLE VIA ENV VAR | *sysdig.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_SYSDIG_HELM_CHART_REPO_URL | SETTABLE VIA ENV VAR | *sysdig.Config.HelmChartRepoURL | string |
-| K8S_TESTER_ADD_ON_SYSDIG_NAMESPACE | SETTABLE VIA ENV VAR | *sysdig.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_SYSDIG_ACCESS_KEY | SETTABLE VIA ENV VAR | *sysdig.Config.AccessKey | string |
-| K8S_TESTER_ADD_ON_SYSDIG_COLLECTOR_ENDPOINT | SETTABLE VIA ENV VAR | *sysdig.Config.CollectorEndpoint | string |
-*----------------------------------------------*----------------------*----------------------------------*---------*
-
-*----------------------------------------------*----------------------*---------------------------------*---------*
-| ENVIRONMENTAL VARIABLE | FIELD TYPE | TYPE | GO TYPE |
-*----------------------------------------------*----------------------*---------------------------------*---------*
-| K8S_TESTER_ADD_ON_SPLUNK_ENABLE | SETTABLE VIA ENV VAR | *splunk.Config.Enable | bool |
-| K8S_TESTER_ADD_ON_SPLUNK_MINIMUM_NODES | SETTABLE VIA ENV VAR | *splunk.Config.MinimumNodes | int |
-| K8S_TESTER_ADD_ON_SPLUNK_HELM_CHART_REPO_URL | SETTABLE VIA ENV VAR | *splunk.Config.HelmChartRepoURL | string |
-| K8S_TESTER_ADD_ON_SPLUNK_NAMESPACE | SETTABLE VIA ENV VAR | *splunk.Config.Namespace | string |
-| K8S_TESTER_ADD_ON_SPLUNK_ACCESS_KEY | SETTABLE VIA ENV VAR | *splunk.Config.AccessKey | string |
-| K8S_TESTER_ADD_ON_SPLUNK_SPLUNK_REALM | SETTABLE VIA ENV VAR | *splunk.Config.SplunkRealm | string |
-*----------------------------------------------*----------------------*---------------------------------*---------*
-```
diff --git a/k8s-tester/aqua/README.md b/k8s-tester/aqua/README.md
deleted file mode 100644
index f181f2100..000000000
--- a/k8s-tester/aqua/README.md
+++ /dev/null
@@ -1,43 +0,0 @@
-
-# How we added/built stand-alone tests for Aqua
-
-- git clone
-- `mkdir/aqua`
-- `cd aqua/`
-- `go mod init github.com/aws/aws-k8s-tester/k8s-tester/aqua`
-- Create a file to implement the Tester Interface. `touch tester.go`
-- copy a vend file from another package `cp ../vend.sh .`
-- Write tests
-- run `./vend.sh`
-- run `go mod tidy -v`
-
-
-
-Test/Run singe test stand-alone
-```bash
-go run cmd/k8s-tester-aqua/main.go apply \
- --kubectl-path="/usr/local/bin/kubectl" \
- --kubeconfig-path="/PATHTO/kubeconfig" \
- --log-outputs="aqua.log" \
- --aqua-license="1234567890" \
- --aqua-username="Username" \
- --aqua-password="Password"
-
-## Delete
-go run cmd/k8s-tester-aqua/main.go delete \
- --kubectl-path="/usr/local/bin/kubectl" \
- --kubeconfig-path="/PATHTO/kubeconfig" \
- --log-outputs="aqua.log" \
- --aqua-license="1234567890" \
- --aqua-username="Username" \
- --aqua-password="Password"
-```
-
-## Tests are equivilant to
-```
-
-helm repo add aqua https://helm.aquasec.com
-
-helm repo update
-
-helm upgrade --install --namespace aqua aqua . --set ke.aquasecret.kubeEnforcerToken=12345 --set imageCredentials.username=12345@gmail.com --set imageCredentials.password=12345 --set web.service.type=ClusterIP
\ No newline at end of file
diff --git a/k8s-tester/aqua/cmd/k8s-tester-aqua/main.go b/k8s-tester/aqua/cmd/k8s-tester-aqua/main.go
deleted file mode 100644
index 69206949c..000000000
--- a/k8s-tester/aqua/cmd/k8s-tester-aqua/main.go
+++ /dev/null
@@ -1,163 +0,0 @@
-// k8s-tester-aqua installs aqua using helm, and tests that it's able to function correctly.
-package main
-
-import (
- "fmt"
- "os"
-
- "github.com/aws/aws-k8s-tester/client"
- aqua "github.com/aws/aws-k8s-tester/k8s-tester/aqua"
- "github.com/aws/aws-k8s-tester/utils/log"
- "github.com/spf13/cobra"
- "go.uber.org/zap"
-)
-
-var rootCmd = &cobra.Command{
- Use: "k8s-tester-aqua",
- Short: "Kubernetes Aqua tester",
- SuggestFor: []string{"aqua"},
-}
-
-func init() {
- cobra.EnablePrefixMatching = true
-}
-
-var (
- prompt bool
- logLevel string
- logOutputs []string
- minimumNodes int
- namespace string
- kubectlDownloadURL string
- kubectlPath string
- kubeconfigPath string
- aquaLicense string
- aquaUsername string
- aquaPassword string
-)
-
-func init() {
- rootCmd.PersistentFlags().BoolVar(&prompt, "prompt", true, "'true' to enable prompt mode")
- rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", log.DefaultLogLevel, "Logging level")
- rootCmd.PersistentFlags().StringSliceVar(&logOutputs, "log-outputs", []string{"stderr"}, "Additional logger outputs")
- rootCmd.PersistentFlags().IntVar(&minimumNodes, "minimum-nodes", aqua.DefaultMinimumNodes, "minimum number of Kubernetes nodes required for installing this addon")
- rootCmd.PersistentFlags().StringVar(&namespace, "namespace", "test-namespace", "'true' to auto-generate path for create config/cluster, overwrites existing --path value")
- rootCmd.PersistentFlags().StringVar(&kubectlDownloadURL, "kubectl-download-url", client.DefaultKubectlDownloadURL(), "kubectl download URL")
- rootCmd.PersistentFlags().StringVar(&kubectlPath, "kubectl-path", client.DefaultKubectlPath(), "kubectl path")
- rootCmd.PersistentFlags().StringVar(&kubeconfigPath, "kubeconfig-path", "", "KUBECONFIG path")
- rootCmd.PersistentFlags().StringVar(&aquaLicense, "aqua-license", "", "aquaLicense for helm chart")
- rootCmd.PersistentFlags().StringVar(&aquaUsername, "aqua-username", "", "aquaUsername from success center")
- rootCmd.PersistentFlags().StringVar(&aquaPassword, "aqua-password", "", "aquaPassword from success center")
-
- rootCmd.AddCommand(
- newApply(),
- newDelete(),
- )
-}
-
-func main() {
- if err := rootCmd.Execute(); err != nil {
- fmt.Fprintf(os.Stderr, "k8s-tester-aqua failed %v\n", err)
- os.Exit(1)
- }
- os.Exit(0)
-}
-
-var helmChartRepoURL string
-
-func newApply() *cobra.Command {
- cmd := &cobra.Command{
- Use: "apply",
- Short: "Apply tests",
- Run: createApplyFunc,
- }
- cmd.PersistentFlags().StringVar(&helmChartRepoURL, "helm-chart-repo-url", aqua.DefaultHelmChartRepoURL, "helm chart repo URL")
- return cmd
-}
-
-func createApplyFunc(cmd *cobra.Command, args []string) {
- lg, logWriter, _, err := log.NewWithStderrWriter(logLevel, logOutputs)
- if err != nil {
- panic(err)
- }
- _ = zap.ReplaceGlobals(lg)
-
- cli, err := client.New(&client.Config{
- Logger: lg,
- KubectlDownloadURL: kubectlDownloadURL,
- KubectlPath: kubectlPath,
- KubeconfigPath: kubeconfigPath,
- })
- if err != nil {
- lg.Panic("failed to create client", zap.Error(err))
- }
-
- cfg := &aqua.Config{
- Prompt: prompt,
- Logger: lg,
- LogWriter: logWriter,
- MinimumNodes: minimumNodes,
- Namespace: namespace,
- HelmChartRepoURL: helmChartRepoURL,
- Client: cli,
- AquaLicense: aquaLicense,
- AquaUsername: aquaUsername,
- AquaPassword: aquaPassword,
- }
-
- ts := aqua.New(cfg)
- if err := ts.Apply(); err != nil {
- fmt.Fprintf(os.Stderr, "failed to apply (%v)\n", err)
- os.Exit(1)
- }
-
- fmt.Printf("\n*********************************\n")
- fmt.Printf("'k8s-tester-aqua apply' success\n")
-}
-
-func newDelete() *cobra.Command {
- cmd := &cobra.Command{
- Use: "delete",
- Short: "Delete resources",
- Run: createDeleteFunc,
- }
- return cmd
-}
-
-func createDeleteFunc(cmd *cobra.Command, args []string) {
- lg, logWriter, _, err := log.NewWithStderrWriter(logLevel, logOutputs)
- if err != nil {
- panic(err)
- }
- _ = zap.ReplaceGlobals(lg)
-
- cli, err := client.New(&client.Config{
- Logger: lg,
- KubectlDownloadURL: kubectlDownloadURL,
- KubectlPath: kubectlPath,
- KubeconfigPath: kubeconfigPath,
- })
- if err != nil {
- lg.Panic("failed to create client", zap.Error(err))
- }
-
- cfg := &aqua.Config{
- Prompt: prompt,
- Logger: lg,
- LogWriter: logWriter,
- Namespace: namespace,
- Client: cli,
- AquaLicense: aquaLicense,
- AquaUsername: aquaUsername,
- AquaPassword: aquaPassword,
- }
-
- ts := aqua.New(cfg)
- if err := ts.Delete(); err != nil {
- fmt.Fprintf(os.Stderr, "failed to delete (%v)\n", err)
- os.Exit(1)
- }
-
- fmt.Printf("\n*********************************\n")
- fmt.Printf("'k8s-tester-aqua delete' success\n")
-}
diff --git a/k8s-tester/aqua/tester.go b/k8s-tester/aqua/tester.go
deleted file mode 100644
index 26d479601..000000000
--- a/k8s-tester/aqua/tester.go
+++ /dev/null
@@ -1,321 +0,0 @@
-// package aqua installs aqua helm charts.
-// ref https://github.com/aquasecurity/aqua-helm
-package aqua
-
-import (
- "context"
- "errors"
- "fmt"
- "io"
- "path"
- "reflect"
- "strings"
- "time"
-
- "github.com/aws/aws-k8s-tester/client"
- "github.com/aws/aws-k8s-tester/k8s-tester/helm"
- k8s_tester "github.com/aws/aws-k8s-tester/k8s-tester/tester"
- "github.com/aws/aws-k8s-tester/utils/rand"
- utils_time "github.com/aws/aws-k8s-tester/utils/time"
- "github.com/manifoldco/promptui"
- "go.uber.org/zap"
- "k8s.io/utils/exec"
-)
-
-type Config struct {
- Enable bool `json:"enable"`
- Prompt bool `json:"-"`
-
- Stopc chan struct{} `json:"-"`
- Logger *zap.Logger `json:"-"`
- LogWriter io.Writer `json:"-"`
- Client client.Client `json:"-"`
-
- // MinimumNodes is the minimum number of Kubernetes nodes required for installing this addon.
- MinimumNodes int `json:"minimum_nodes"`
- // HelmChartRepoURL is the helm chart repo URL.
- HelmChartRepoURL string `json:"helm_chart_repo_url"`
- // Namespace to create test resources.
- Namespace string `json:"namespace"`
- // AquaLicense is the license used from the suceess center for Kubenenforcer
- AquaLicense string `json:"aqua_license"`
- // AquaUsername is the username for the suceess center used to pull images
- AquaUsername string `json:"aqua_username"`
- // AquaUsername is the password for the suceess center used to pull images
- AquaPassword string `json:"aqua_password"`
-}
-
-func (cfg *Config) ValidateAndSetDefaults() error {
- if cfg.MinimumNodes == 0 {
- cfg.MinimumNodes = DefaultMinimumNodes
- }
- if cfg.Namespace == "" {
- return errors.New("empty Namespace")
- }
- if cfg.AquaUsername == "" {
- return errors.New("empty Aqua Usernmae")
- }
- if cfg.AquaPassword == "" {
- return errors.New("empty Aqua Password")
- }
- return nil
-}
-
-const chartName = "server"
-
-const (
- DefaultMinimumNodes int = 1
- chartRepoName = "aqua"
- chartRepoURL = "https://helm.aquasec.com"
- DefaultHelmChartRepoURL = "https://github.com/aquasecurity/aqua-helm/archive/refs/tags/6.0.2.tar.gz"
-)
-
-func NewDefault() *Config {
- return &Config{
- Enable: false,
- Prompt: false,
- MinimumNodes: DefaultMinimumNodes,
- HelmChartRepoURL: DefaultHelmChartRepoURL,
- Namespace: pkgName + "-" + rand.String(10) + "-" + utils_time.GetTS(10),
- }
-}
-
-func New(cfg *Config) k8s_tester.Tester {
- return &tester{
- cfg: cfg,
- }
-}
-
-type tester struct {
- cfg *Config
-}
-
-var pkgName = path.Base(reflect.TypeOf(tester{}).PkgPath())
-
-func Env() string {
- return "ADD_ON_" + strings.ToUpper(strings.Replace(pkgName, "-", "_", -1))
-}
-
-func (ts *tester) Name() string { return pkgName }
-
-func (ts *tester) Enabled() bool { return ts.cfg.Enable }
-
-func (ts *tester) Apply() error {
- if ok := ts.runPrompt("apply"); !ok {
- return errors.New("cancelled")
- }
-
- if nodes, err := client.ListNodes(ts.cfg.Client.KubernetesClient()); len(nodes) < ts.cfg.MinimumNodes || err != nil {
- return fmt.Errorf("failed to validate minimum nodes requirement %d (nodes %v, error %v)", ts.cfg.MinimumNodes, len(nodes), err)
- }
- if err := client.CreateNamespace(ts.cfg.Logger, ts.cfg.Client.KubernetesClient(), ts.cfg.Namespace); err != nil {
- return err
- }
- if err := ts.checkForStorageClass(); err != nil {
- return err
- }
- if err := helm.AddUpdate(ts.cfg.Logger, chartRepoName, chartRepoURL); err != nil {
- return err
- }
- if err := ts.createHelmAqua(); err != nil {
- return err
- }
- return nil
-}
-
-func (ts *tester) Delete() error {
- if ok := ts.runPrompt("delete"); !ok {
- return errors.New("cancelled")
- }
-
- var errs []string
-
- if err := ts.deleteHelmAqua(); err != nil {
- errs = append(errs, err.Error())
- }
-
- if err := client.DeleteNamespaceAndWait(
- ts.cfg.Logger,
- ts.cfg.Client.KubernetesClient(),
- ts.cfg.Namespace,
- client.DefaultNamespaceDeletionInterval,
- client.DefaultNamespaceDeletionTimeout,
- client.WithForceDelete(true),
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete namespace (%v)", err))
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- return nil
-}
-
-func (ts *tester) runPrompt(action string) (ok bool) {
- if ts.cfg.Prompt {
- msg := fmt.Sprintf("Ready to %q resources, should we continue?", action)
- prompt := promptui.Select{
- Label: msg,
- Items: []string{
- "No, cancel it!",
- fmt.Sprintf("Yes, let's %q!", action),
- },
- }
- idx, answer, err := prompt.Run()
- if err != nil {
- panic(err)
- }
- if idx != 1 {
- fmt.Printf("cancelled %q [index %d, answer %q]\n", action, idx, answer)
- return false
- }
- }
- return true
-}
-
-func (ts *tester) checkForStorageClass() (err error) {
- storageclass, err := client.ListStorageClass(
- ts.cfg.Logger,
- ts.cfg.Client.KubernetesClient(),
- 5,
- 5*time.Second,
- )
- for _, class := range storageclass {
- if class.ObjectMeta.Annotations["storageclass.kubernetes.io/is-default-class"] == "true" {
- ts.cfg.Logger.Info("found default STORAGE CLASS, proceeding to tests")
- return nil
- } else {
- return errors.New("No Default StroageClass")
- }
- }
- return nil
-}
-
-// https://github.com/aquasecurity/aqua-helm/tree/6.2/aqua-quickstart
-func (ts *tester) createHelmAqua() error {
- values := map[string]interface{}{
- // "ke": map[string]interface{}{
- // "aquasecret": map[string]interface{}{
- // "kubeEnforcerToken": ts.cfg.AquaLicense,
- // },
- // },
- "platform": "k8s",
- "imageCredentials": map[string]interface{}{
- "username": ts.cfg.AquaUsername,
- "password": ts.cfg.AquaPassword,
- },
- "web": map[string]interface{}{
- "service": map[string]interface{}{
- "type": "ClusterIP",
- },
- },
- }
- getAllArgs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "get",
- "all",
- }
- getAllCmd := strings.Join(getAllArgs, " ")
-
- descArgsDs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "describe",
- "deployment.apps/aqua-console",
- }
- descCmdDs := strings.Join(descArgsDs, " ")
-
- descArgsPods := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "describe",
- "pods",
- "--selector=app=aqua-console",
- }
- descCmdPods := strings.Join(descArgsPods, " ")
-
- logArgs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "logs",
- "--selector=app=aqua-console",
- "--all-containers=true",
- "--timestamps",
- }
- logsCmd := strings.Join(logArgs, " ")
-
- return helm.Install(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Stopc: ts.cfg.Stopc,
- Timeout: 10 * time.Minute,
- KubeconfigPath: ts.cfg.Client.Config().KubeconfigPath,
- Namespace: ts.cfg.Namespace,
- ChartRepoURL: chartRepoURL,
- ChartName: chartName,
- ReleaseName: chartName,
- Values: values,
- LogFunc: func(format string, v ...interface{}) {
- ts.cfg.Logger.Info(fmt.Sprintf("[install] "+format, v...))
- },
- QueryFunc: func() {
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, getAllArgs[0], getAllArgs[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl get all' failed", zap.Error(err))
- ts.cfg.Logger.Warn("Aqua Tests::", zap.String("TEST", "FAILED"))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getAllCmd, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, descArgsDs[0], descArgsDs[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe daemonset' failed", zap.Error(err))
- ts.cfg.Logger.Warn("Aqua Tests::", zap.String("TEST", "FAILED"))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmdDs, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, descArgsPods[0], descArgsPods[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe pods' failed", zap.Error(err))
- ts.cfg.Logger.Warn("Aqua Tests::", zap.String("TEST", "FAILED"))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmdPods, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, logArgs[0], logArgs[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl logs' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", logsCmd, out)
- },
- QueryInterval: 30 * time.Second,
- })
-}
-
-func (ts *tester) deleteHelmAqua() error {
- return helm.Uninstall(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Timeout: 15 * time.Minute,
- KubeconfigPath: ts.cfg.Client.Config().KubeconfigPath,
- Namespace: ts.cfg.Namespace,
- ChartName: chartName,
- ReleaseName: chartName,
- })
-}
diff --git a/k8s-tester/aqua/vend.sh b/k8s-tester/aqua/vend.sh
deleted file mode 100755
index 7937e2980..000000000
--- a/k8s-tester/aqua/vend.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-< 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- return nil
-}
-
-func (ts *tester) runPrompt(action string) (ok bool) {
- if ts.cfg.Prompt {
- msg := fmt.Sprintf("Ready to %q resources, should we continue?", action)
- prompt := promptui.Select{
- Label: msg,
- Items: []string{
- "No, cancel it!",
- fmt.Sprintf("Yes, let's %q!", action),
- },
- }
- idx, answer, err := prompt.Run()
- if err != nil {
- panic(err)
- }
- if idx != 1 {
- fmt.Printf("cancelled %q [index %d, answer %q]\n", action, idx, answer)
- return false
- }
- }
- return true
-}
-
-// https://github.com/armory/spinnaker-operator/blob/master/deploy/operator/helm/values.yaml
-func (ts *tester) createHelmSpinnaker() error {
- values := map[string]interface{}{
- "image": map[string]interface{}{
- "tag": "0.28.1",
- },
- }
-
- getAllArgs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "get",
- "all",
- }
- getAllCmd := strings.Join(getAllArgs, " ")
-
- descArgsDs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "describe",
- "deployment.apps/spin-deck",
- }
- descCmdDs := strings.Join(descArgsDs, " ")
-
- descArgsPods := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "describe",
- "pods",
- "--selector=app=spin",
- }
- descCmdPods := strings.Join(descArgsPods, " ")
-
- logArgs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "logs",
- "--selector=app=spin",
- "--all-containers=true",
- "--timestamps",
- }
- logsCmd := strings.Join(logArgs, " ")
-
- return helm.Install(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Stopc: ts.cfg.Stopc,
- Timeout: 10 * time.Minute,
- KubeconfigPath: ts.cfg.Client.Config().KubeconfigPath,
- Namespace: ts.cfg.Namespace,
- ChartRepoURL: ts.cfg.HelmChartRepoURL,
- ChartName: chartName,
- ReleaseName: chartName,
- Values: values,
- LogFunc: func(format string, v ...interface{}) {
- ts.cfg.Logger.Info(fmt.Sprintf("[install] "+format, v...))
- },
- QueryFunc: func() {
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, getAllArgs[0], getAllArgs[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl get all' failed", zap.Error(err))
- ts.cfg.Logger.Warn("Spinnaker Tests::", zap.String("TEST", "FAILED"))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getAllCmd, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, descArgsDs[0], descArgsDs[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe daemonset' failed", zap.Error(err))
- ts.cfg.Logger.Warn("Spinnaker Tests::", zap.String("TEST", "FAILED"))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmdDs, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, descArgsPods[0], descArgsPods[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe pods' failed", zap.Error(err))
- ts.cfg.Logger.Warn("Spinnaker Tests::", zap.String("TEST", "FAILED"))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmdPods, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, logArgs[0], logArgs[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl logs' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", logsCmd, out)
- },
- QueryInterval: 30 * time.Second,
- })
-}
-
-func (ts *tester) deleteHelmSpinnaker() error {
- return helm.Uninstall(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Timeout: 15 * time.Minute,
- KubeconfigPath: ts.cfg.Client.Config().KubeconfigPath,
- Namespace: ts.cfg.Namespace,
- ChartName: chartName,
- ReleaseName: chartName,
- })
-}
diff --git a/k8s-tester/armory/vend.sh b/k8s-tester/armory/vend.sh
deleted file mode 100755
index 7937e2980..000000000
--- a/k8s-tester/armory/vend.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-< 0 {
- if nodes, err := client.ListNodes(ts.cfg.Client.KubernetesClient()); len(nodes) < ts.cfg.MinimumNodes || err != nil {
- return fmt.Errorf("failed to validate minimum nodes requirement %d (nodes %v, error %v)", ts.cfg.MinimumNodes, len(nodes), err)
- }
- }
-
- if err := client.CreateNamespace(ts.cfg.Logger, ts.cfg.Client.KubernetesClient(), ts.cfg.Namespace); err != nil {
- return err
- }
-
- if err := ts.createServiceAccount(); err != nil {
- return err
- }
-
- if err := ts.createRBACClusterRole(); err != nil {
- return err
- }
-
- if err := ts.createRBACClusterRoleBinding(); err != nil {
- return err
- }
-
- if err := ts.createConfigMapConfig(); err != nil {
- return err
- }
-
- if err := ts.createDaemonSet(); err != nil {
- return err
- }
-
- if err := ts.checkPods(); err != nil {
- return err
- }
-
- return nil
-}
-
-func (ts *tester) Delete() error {
- if ok := ts.runPrompt("delete"); !ok {
- return errors.New("cancelled")
- }
-
- var errs []string
-
- if err := ts.deleteDaemonSet(); err != nil {
- errs = append(errs, err.Error())
- }
- time.Sleep(time.Minute)
-
- if err := ts.deleteConfigMapConfig(); err != nil {
- errs = append(errs, err.Error())
- }
-
- if err := ts.deleteRBACClusterRoleBinding(); err != nil {
- errs = append(errs, err.Error())
- }
-
- if err := ts.deleteRBACClusterRole(); err != nil {
- errs = append(errs, err.Error())
- }
-
- if err := ts.deleteServiceAccount(); err != nil {
- errs = append(errs, err.Error())
- }
-
- if err := client.DeleteNamespaceAndWait(
- ts.cfg.Logger,
- ts.cfg.Client.KubernetesClient(),
- ts.cfg.Namespace,
- client.DefaultNamespaceDeletionInterval,
- client.DefaultNamespaceDeletionTimeout,
- client.WithForceDelete(true),
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete namespace (%v)", err))
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- return nil
-}
-
-func (ts *tester) runPrompt(action string) (ok bool) {
- if ts.cfg.Prompt {
- msg := fmt.Sprintf("Ready to %q resources for the namespace %q, should we continue?", action, ts.cfg.Namespace)
- prompt := promptui.Select{
- Label: msg,
- Items: []string{
- "No, cancel it!",
- fmt.Sprintf("Yes, let's %q!", action),
- },
- }
- idx, answer, err := prompt.Run()
- if err != nil {
- panic(err)
- }
- if idx != 1 {
- fmt.Printf("cancelled %q [index %d, answer %q]\n", action, idx, answer)
- return false
- }
- }
- return true
-}
-
-const (
- cwAgentServiceAccountName = "amazon-cloudwatch-agent-service-account"
- cwAgentRBACRoleName = "amazon-cloudwatch-agent-rbac-role"
- cwAgentRBACClusterRoleBindingName = "amazon-cloudwatch-agent-rbac-role-binding"
- cwAgentConfigMapNameConfig = "amazon-cloudwatch-configmap-config"
- cwAgentConfigMapFileNameConfig = "cwagentconfig.json"
- cwAgentAppName = "amazon-cloudwatch"
- cwAgentDaemonSetName = "amazon-cloudwatch"
-)
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) createServiceAccount() error {
- ts.cfg.Logger.Info("creating cw agent ServiceAccount")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.Client.KubernetesClient().
- CoreV1().
- ServiceAccounts(ts.cfg.Namespace).
- Create(
- ctx,
- &core_v1.ServiceAccount{
- TypeMeta: meta_v1.TypeMeta{
- APIVersion: "v1",
- Kind: "ServiceAccount",
- },
- ObjectMeta: meta_v1.ObjectMeta{
- Name: cwAgentServiceAccountName,
- Namespace: ts.cfg.Namespace,
- Labels: map[string]string{
- "app.kubernetes.io/name": cwAgentAppName,
- },
- },
- },
- meta_v1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create cw agent ServiceAccount (%v)", err)
- }
-
- ts.cfg.Logger.Info("created cw agent ServiceAccount")
- return nil
-}
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) deleteServiceAccount() error {
- ts.cfg.Logger.Info("deleting cw agent ServiceAccount")
- foreground := meta_v1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.Client.KubernetesClient().
- CoreV1().
- ServiceAccounts(ts.cfg.Namespace).
- Delete(
- ctx,
- cwAgentServiceAccountName,
- meta_v1.DeleteOptions{
- GracePeriodSeconds: int64Ref(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !k8s_errors.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete cw agent ServiceAccount (%v)", err)
- }
- ts.cfg.Logger.Info("deleted cw agent ServiceAccount", zap.Error(err))
-
- return nil
-}
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) createRBACClusterRole() error {
- ts.cfg.Logger.Info("creating cw agent RBAC ClusterRole")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.Client.KubernetesClient().
- RbacV1().
- ClusterRoles().
- Create(
- ctx,
- &rbac_v1.ClusterRole{
- TypeMeta: meta_v1.TypeMeta{
- APIVersion: "rbac.authorization.k8s.io/v1",
- Kind: "ClusterRole",
- },
- // "ClusterRole" is a non-namespaced resource.
- // ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole
- ObjectMeta: meta_v1.ObjectMeta{
- Name: cwAgentRBACRoleName,
- Namespace: "default",
- Labels: map[string]string{
- "app.kubernetes.io/name": cwAgentAppName,
- },
- },
- Rules: []rbac_v1.PolicyRule{
- {
- // "" indicates the core API group
- // ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole
- APIGroups: []string{
- "",
- },
- Resources: []string{
- "pods",
- "nodes",
- "endpoints",
- },
- Verbs: []string{
- "list",
- "watch",
- },
- },
- {
- APIGroups: []string{
- "apps",
- },
- Resources: []string{
- "replicasets",
- },
- Verbs: []string{
- "list",
- "watch",
- },
- },
- {
- APIGroups: []string{
- "batch",
- },
- Resources: []string{
- "jobs",
- },
- Verbs: []string{
- "list",
- "watch",
- },
- },
- {
- // "" indicates the core API group
- // ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole
- APIGroups: []string{
- "",
- },
- Resources: []string{
- "nodes/stats",
- "configmaps",
- "events",
- },
- Verbs: []string{
- "create",
- },
- },
- {
- // "" indicates the core API group
- // ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole
- APIGroups: []string{
- "",
- },
- Resources: []string{
- "configmaps",
- },
- ResourceNames: []string{
- "cwagent-clusterleader",
- },
- Verbs: []string{
- "get",
- "update",
- },
- },
- },
- },
- meta_v1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create cw agent RBAC ClusterRole (%v)", err)
- }
-
- ts.cfg.Logger.Info("created cw agent RBAC ClusterRole")
- return nil
-}
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) deleteRBACClusterRole() error {
- ts.cfg.Logger.Info("deleting cw agent RBAC ClusterRole")
- foreground := meta_v1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.Client.KubernetesClient().
- RbacV1().
- ClusterRoles().
- Delete(
- ctx,
- cwAgentRBACRoleName,
- meta_v1.DeleteOptions{
- GracePeriodSeconds: int64Ref(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !k8s_errors.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete cw agent RBAC ClusterRole (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted cw agent RBAC ClusterRole", zap.Error(err))
- return nil
-}
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) createRBACClusterRoleBinding() error {
- ts.cfg.Logger.Info("creating cw agent RBAC ClusterRoleBinding")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.Client.KubernetesClient().
- RbacV1().
- ClusterRoleBindings().
- Create(
- ctx,
- &rbac_v1.ClusterRoleBinding{
- TypeMeta: meta_v1.TypeMeta{
- APIVersion: "rbac.authorization.k8s.io/v1",
- Kind: "ClusterRoleBinding",
- },
- ObjectMeta: meta_v1.ObjectMeta{
- Name: cwAgentRBACClusterRoleBindingName,
- Namespace: "default",
- Labels: map[string]string{
- "app.kubernetes.io/name": cwAgentAppName,
- },
- },
- RoleRef: rbac_v1.RoleRef{
- APIGroup: "rbac.authorization.k8s.io",
- Kind: "ClusterRole",
- Name: cwAgentRBACRoleName,
- },
- Subjects: []rbac_v1.Subject{
- {
- APIGroup: "",
- Kind: "ServiceAccount",
- Name: cwAgentServiceAccountName,
- Namespace: ts.cfg.Namespace,
- },
- },
- },
- meta_v1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create cw agent RBAC ClusterRoleBinding (%v)", err)
- }
-
- ts.cfg.Logger.Info("created cw agent RBAC ClusterRoleBinding")
- return nil
-}
-
-// ref. https://github.com/kubernetes/client-go/tree/master/examples/in-cluster-client-configuration
-// ref. https://kubernetes.io/docs/reference/access-authn-authz/rbac/
-func (ts *tester) deleteRBACClusterRoleBinding() error {
- ts.cfg.Logger.Info("deleting cw agent RBAC ClusterRoleBinding")
- foreground := meta_v1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.Client.KubernetesClient().
- RbacV1().
- ClusterRoleBindings().
- Delete(
- ctx,
- cwAgentRBACClusterRoleBindingName,
- meta_v1.DeleteOptions{
- GracePeriodSeconds: int64Ref(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !k8s_errors.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return fmt.Errorf("failed to delete cw agent RBAC ClusterRoleBinding (%v)", err)
- }
-
- ts.cfg.Logger.Info("deleted cw agent RBAC ClusterRoleBinding", zap.Error(err))
- return nil
-}
-
-// https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Container-Insights-setup-metrics.html
-const TemplateCWAgentConf = `{
- "agent": {
- "region": "{{.RegionName}}"
- },
- "logs": {
- "metrics_collected": {
- "kubernetes": {
- "cluster_name": "{{.ClusterName}}",
- "metrics_collection_interval": 60
- }
- },
- "force_flush_interval": 5
- }
-}
-`
-
-type templateCWAgentConf struct {
- RegionName string
- ClusterName string
-}
-
-func (ts *tester) createConfigMapConfig() (err error) {
- ts.cfg.Logger.Info("creating cw agent ConfigMap config")
-
- buf := bytes.NewBuffer(nil)
- cwConf := templateCWAgentConf{
- RegionName: ts.cfg.Region,
- ClusterName: ts.cfg.ClusterName,
- }
- cwConfTmpl := template.Must(template.New("TemplateCWAgentConf").Parse(TemplateCWAgentConf))
- if err := cwConfTmpl.Execute(buf, cwConf); err != nil {
- return err
- }
- cwConfBody := buf.String()
- buf.Reset()
-
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err = ts.cfg.Client.KubernetesClient().
- CoreV1().
- ConfigMaps(ts.cfg.Namespace).
- Create(
- ctx,
- &core_v1.ConfigMap{
- TypeMeta: meta_v1.TypeMeta{
- APIVersion: "v1",
- Kind: "ConfigMap",
- },
- ObjectMeta: meta_v1.ObjectMeta{
- Name: cwAgentConfigMapNameConfig,
- Namespace: ts.cfg.Namespace,
- Labels: map[string]string{
- "name": cwAgentConfigMapNameConfig,
- },
- },
- Data: map[string]string{
- cwAgentConfigMapFileNameConfig: cwConfBody,
- },
- },
- meta_v1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return err
- }
-
- ts.cfg.Logger.Info("created cw agent ConfigMap config")
- return nil
-}
-
-func (ts *tester) deleteConfigMapConfig() error {
- ts.cfg.Logger.Info("deleting cw agent ConfigMap config")
- foreground := meta_v1.DeletePropagationForeground
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.Client.KubernetesClient().
- CoreV1().
- ConfigMaps(ts.cfg.Namespace).
- Delete(
- ctx,
- cwAgentConfigMapNameConfig,
- meta_v1.DeleteOptions{
- GracePeriodSeconds: int64Ref(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil {
- return err
- }
- ts.cfg.Logger.Info("deleted cw agent ConfigMap config")
- return nil
-}
-
-// CWAgentImageName is the image name of CloudWatch agent daemon set.
-// ref. https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Container-Insights-setup-logs.html
-// ref. https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Container-Insights-setup-EKS-quickstart.html
-// ref. https://hub.docker.com/r/amazon/cloudwatch-agent
-const CWAgentImageName = "amazon/cloudwatch-agent:1.247347.6b250880"
-
-func (ts *tester) createDaemonSet() (err error) {
- podSpec := core_v1.PodTemplateSpec{
- ObjectMeta: meta_v1.ObjectMeta{
- Labels: map[string]string{
- "app.kubernetes.io/name": cwAgentAppName,
- },
- },
- Spec: core_v1.PodSpec{
- ServiceAccountName: cwAgentServiceAccountName,
- TerminationGracePeriodSeconds: int64Ref(60),
- // Unsupported value: "OnFailure": supported values: "Always"
- RestartPolicy: core_v1.RestartPolicyAlways,
-
- // https://www.eksworkshop.com/intermediate/230_logging/deploy/
- Containers: []core_v1.Container{
- {
- Name: cwAgentAppName,
- Image: CWAgentImageName,
- ImagePullPolicy: core_v1.PullAlways,
-
- Resources: core_v1.ResourceRequirements{
- Limits: core_v1.ResourceList{
- core_v1.ResourceCPU: api_resource.MustParse("200m"),
- core_v1.ResourceMemory: api_resource.MustParse("200Mi"),
- },
- Requests: core_v1.ResourceList{
- core_v1.ResourceCPU: api_resource.MustParse("200m"),
- core_v1.ResourceMemory: api_resource.MustParse("200Mi"),
- },
- },
-
- Env: []core_v1.EnvVar{
- {
- Name: "HOST_IP",
- ValueFrom: &core_v1.EnvVarSource{
- FieldRef: &core_v1.ObjectFieldSelector{
- FieldPath: "status.hostIP",
- },
- },
- },
- {
- Name: "HOST_NAME",
- ValueFrom: &core_v1.EnvVarSource{
- FieldRef: &core_v1.ObjectFieldSelector{
- FieldPath: "spec.nodeName",
- },
- },
- },
- {
- Name: "K8S_NAMESPACE",
- ValueFrom: &core_v1.EnvVarSource{
- FieldRef: &core_v1.ObjectFieldSelector{
- FieldPath: "metadata.namespace",
- },
- },
- },
- {
- Name: "CI_VERSION",
- Value: "k8s/1.1.1",
- },
- },
-
- // ref. https://kubernetes.io/docs/concepts/cluster-administration/logging/
- VolumeMounts: []core_v1.VolumeMount{
- {
- Name: cwAgentConfigMapNameConfig,
- MountPath: "/etc/cwagentconfig",
- },
- {
- Name: "rootfs",
- MountPath: "/rootfs",
- ReadOnly: true,
- },
- {
- Name: "dockersock",
- MountPath: "/var/run/docker.sock",
- ReadOnly: true,
- },
- {
- Name: "varlibdocker",
- MountPath: "/var/lib/docker",
- ReadOnly: true,
- },
- {
- Name: "sys",
- MountPath: "/sys",
- ReadOnly: true,
- },
- {
- Name: "devdisk",
- MountPath: "/dev/disk",
- ReadOnly: true,
- },
- },
- },
- },
-
- // ref. https://kubernetes.io/docs/concepts/cluster-administration/logging/
- Volumes: []core_v1.Volume{
- {
- Name: cwAgentConfigMapNameConfig,
- VolumeSource: core_v1.VolumeSource{
- ConfigMap: &core_v1.ConfigMapVolumeSource{
- LocalObjectReference: core_v1.LocalObjectReference{
- Name: cwAgentConfigMapNameConfig,
- },
- DefaultMode: int32Ref(0666),
- },
- },
- },
- {
- Name: "rootfs",
- VolumeSource: core_v1.VolumeSource{
- HostPath: &core_v1.HostPathVolumeSource{
- Path: "/",
- },
- },
- },
- {
- Name: "dockersock",
- VolumeSource: core_v1.VolumeSource{
- HostPath: &core_v1.HostPathVolumeSource{
- Path: "/var/run/docker.sock",
- },
- },
- },
- {
- Name: "varlibdocker",
- VolumeSource: core_v1.VolumeSource{
- HostPath: &core_v1.HostPathVolumeSource{
- Path: "/var/lib/docker",
- },
- },
- },
- {
- Name: "sys",
- VolumeSource: core_v1.VolumeSource{
- HostPath: &core_v1.HostPathVolumeSource{
- Path: "/sys",
- },
- },
- },
- {
- Name: "devdisk",
- VolumeSource: core_v1.VolumeSource{
- HostPath: &core_v1.HostPathVolumeSource{
- Path: "/dev/disk/",
- },
- },
- },
- },
- },
- }
-
- dsObj := apps_v1.DaemonSet{
- TypeMeta: meta_v1.TypeMeta{
- APIVersion: "apps/v1",
- Kind: "DaemonSet",
- },
- ObjectMeta: meta_v1.ObjectMeta{
- Name: cwAgentDaemonSetName,
- Namespace: ts.cfg.Namespace,
- },
- Spec: apps_v1.DaemonSetSpec{
- Selector: &meta_v1.LabelSelector{
- MatchLabels: map[string]string{
- "app.kubernetes.io/name": cwAgentAppName,
- },
- },
-
- Template: podSpec,
- },
- }
-
- ts.cfg.Logger.Info("creating cw agent DaemonSet", zap.String("name", cwAgentDaemonSetName))
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err = ts.cfg.Client.KubernetesClient().
- AppsV1().
- DaemonSets(ts.cfg.Namespace).
- Create(ctx, &dsObj, meta_v1.CreateOptions{})
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create cw agent DaemonSet (%v)", err)
- }
-
- ts.cfg.Logger.Info("created cw agent DaemonSet")
- return nil
-}
-
-func (ts *tester) deleteDaemonSet() (err error) {
- foreground := meta_v1.DeletePropagationForeground
- ts.cfg.Logger.Info("deleting cw agent DaemonSet", zap.String("name", cwAgentDaemonSetName))
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err = ts.cfg.Client.KubernetesClient().
- AppsV1().
- DaemonSets(ts.cfg.Namespace).
- Delete(
- ctx,
- cwAgentDaemonSetName,
- meta_v1.DeleteOptions{
- GracePeriodSeconds: int64Ref(0),
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !k8s_errors.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete cw agent DaemonSet", zap.Error(err))
- return fmt.Errorf("failed to delete cw agent DaemonSet (%v)", err)
- }
- return nil
-}
-
-func (ts *tester) checkPods() (err error) {
- waitDur := 10 * time.Minute
- retryStart := time.Now()
- for time.Since(retryStart) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- return errors.New("check aborted")
- case <-time.After(15 * time.Second):
- }
- if err = ts._checkPods(); err == nil {
- break
- }
- ts.cfg.Logger.Info("failed to check cw agent pods; retrying", zap.Error(err))
- }
- return err
-}
-
-func (ts *tester) _checkPods() error {
- pods, err := client.ListPods(ts.cfg.Logger, ts.cfg.Client.KubernetesClient(), ts.cfg.Namespace, 1000, 5*time.Second)
- if err != nil {
- ts.cfg.Logger.Warn("listing pods failed", zap.Error(err))
- return err
- }
- if len(pods) > 0 {
- ts.cfg.Logger.Info("pods found", zap.Int("pods", len(pods)))
- fmt.Fprintf(ts.cfg.LogWriter, "\n")
- for _, pod := range pods {
- fmt.Fprintf(ts.cfg.LogWriter, "%q Pod using client-go: %q\n", ts.cfg.Namespace, pod.Name)
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n")
- } else {
- ts.cfg.Logger.Info("no pod found", zap.String("namespace", ts.cfg.Namespace))
- return errors.New("no pod found in " + ts.cfg.Namespace)
- }
-
- nodes, err := client.ListNodes(ts.cfg.Client.KubernetesClient())
- if err != nil {
- return fmt.Errorf("failed to list nodes %v", err)
- }
-
- totalNodes := int64(len(nodes))
- targetPods := int64(1)
- if totalNodes > 1 {
- targetPods = totalNodes / int64(2)
- }
- ts.cfg.Logger.Info("checking cw agent pods",
- zap.Int64("target-ready-pods", targetPods),
- zap.Int64("total-nodes", totalNodes),
- )
- readyPods := int64(0)
- for _, pod := range pods {
- appName, ok := pod.Labels["app.kubernetes.io/name"]
- if !ok || appName != cwAgentAppName {
- ts.cfg.Logger.Info("skipping pod, not cw agent", zap.String("labels", fmt.Sprintf("%+v", pod.Labels)))
- continue
- }
-
- descArgsPods := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "describe",
- "pods/" + pod.Name,
- }
- descCmdPods := strings.Join(descArgsPods, " ")
-
- logArgs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "logs",
- "pods/" + pod.Name,
- "--all-containers=true",
- "--timestamps",
- }
- logsCmd := strings.Join(logArgs, " ")
-
- ts.cfg.Logger.Debug("checking Pod",
- zap.String("pod-name", pod.Name),
- zap.String("app-name", appName),
- zap.String("command-describe", descCmdPods),
- zap.String("command-logs", logsCmd),
- )
-
- ready := false
- statusType, status := "", ""
- for _, cond := range pod.Status.Conditions {
- if cond.Status != core_v1.ConditionTrue {
- continue
- }
- statusType = fmt.Sprintf("%s", cond.Type)
- status = fmt.Sprintf("%s", cond.Status)
- if cond.Type == core_v1.PodInitialized || cond.Type == core_v1.PodReady {
- ready = true
- readyPods++
- }
- break
- }
- if !ready {
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, descArgsPods[0], descArgsPods[1:]...).CombinedOutput()
- cancel()
- outDesc := string(output)
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n'%s' output:\n\n%s\n\n", descCmdPods, outDesc)
- ts.cfg.Logger.Warn("pod is not ready yet",
- zap.Int64("current-ready-pods", readyPods),
- zap.Int64("target-ready-pods", targetPods),
- zap.Int64("total-nodes", totalNodes),
- zap.String("pod-name", pod.Name),
- zap.String("app-name", appName),
- zap.String("status-type", statusType),
- zap.String("status", status),
- )
- continue
- }
-
- if readyPods < 3 { // only first 3 nodes
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, descArgsPods[0], descArgsPods[1:]...).CombinedOutput()
- cancel()
- outDesc := string(output)
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe' failed", zap.Error(err))
- continue
- }
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, logArgs[0], logArgs[1:]...).CombinedOutput()
- cancel()
- outLogs := string(output)
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl logs' failed", zap.Error(err))
- continue
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n'%s' output:\n\n%s\n\n", descCmdPods, outDesc)
- logLines := strings.Split(outLogs, "\n")
- logLinesN := len(logLines)
- if logLinesN > 15 {
- logLines = logLines[logLinesN-15:]
- outLogs = strings.Join(logLines, "\n")
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n'%s' output:\n\n%s\n\n", logsCmd, outLogs)
- }
- if readyPods%100 == 0 {
- ts.cfg.Logger.Info("found a ready pod",
- zap.Int64("current-ready-pods", readyPods),
- zap.Int64("target-ready-pods", targetPods),
- zap.Int64("total-nodes", totalNodes),
- zap.String("pod-name", pod.Name),
- zap.String("app-name", appName),
- zap.String("status-type", statusType),
- zap.String("status", status),
- )
- }
- }
- ts.cfg.Logger.Info("checking cw agent pods",
- zap.Int64("current-ready-pods", readyPods),
- zap.Int64("target-ready-pods", targetPods),
- zap.Int64("total-nodes", totalNodes),
- )
- if readyPods < targetPods {
- return errors.New("not enough cw agent pods ready")
- }
-
- return nil
-}
-
-func int32Ref(v int32) *int32 {
- return &v
-}
-
-func int64Ref(v int64) *int64 {
- return &v
-}
diff --git a/k8s-tester/cloudwatch-agent/vend.sh b/k8s-tester/cloudwatch-agent/vend.sh
deleted file mode 100755
index 7937e2980..000000000
--- a/k8s-tester/cloudwatch-agent/vend.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-< 23.5). Notice
- // that all data items with the same label combination should have the same buckets.
- Data map[string]float64 `json:"data"`
- // Unit is the data unit. Notice that all data items with the same label combination
- // should have the same unit.
- Unit string `json:"unit"`
- // Labels is the labels of the data item.
- Labels map[string]string `json:"labels,omitempty"`
-}
diff --git a/k8s-tester/clusterloader/clusterloader_test.go b/k8s-tester/clusterloader/clusterloader_test.go
deleted file mode 100644
index b920f98db..000000000
--- a/k8s-tester/clusterloader/clusterloader_test.go
+++ /dev/null
@@ -1,52 +0,0 @@
-package clusterloader
-
-import (
- "fmt"
- "os"
- "path/filepath"
- "strings"
- "testing"
-
- "go.uber.org/zap"
-)
-
-func Test_installClusterloader(t *testing.T) {
- t.Skip()
-
- err := installClusterloader(zap.NewExample(), DefaultClusterloaderPath(), DefaultClusterloaderDownloadURL())
- if err != nil {
- t.Skip(err)
- }
-}
-
-func Test_parsePodStartupLatency(t *testing.T) {
- perfDatas := []PerfData{}
- err := filepath.Walk("test-data", func(path string, info os.FileInfo, werr error) error {
- if werr != nil {
- return werr
- }
- if info.IsDir() {
- return nil
- }
- if !strings.HasPrefix(filepath.Base(path), "PodStartupLatency-") {
- return nil
- }
- p, perr := parsePodStartupLatency(path)
- if perr != nil {
- return perr
- }
- perfDatas = append(perfDatas, p)
- return nil
- })
- if err != nil {
- t.Fatal(err)
- }
- if len(perfDatas) != 5 {
- t.Fatalf("expected 5 data, got %d", len(perfDatas))
- }
- for _, v := range perfDatas {
- fmt.Println(v)
- }
-
- fmt.Printf("%+v\n", mergePodStartupLatency(perfDatas...))
-}
diff --git a/k8s-tester/clusterloader/cmd/k8s-tester-clusterloader/main.go b/k8s-tester/clusterloader/cmd/k8s-tester-clusterloader/main.go
deleted file mode 100644
index 30b06ef1e..000000000
--- a/k8s-tester/clusterloader/cmd/k8s-tester-clusterloader/main.go
+++ /dev/null
@@ -1,240 +0,0 @@
-// k8s-tester-clusterloader installs Kubernetes clusterloader tester.
-package main
-
-import (
- "fmt"
- "os"
- "time"
-
- "github.com/aws/aws-k8s-tester/client"
- "github.com/aws/aws-k8s-tester/k8s-tester/clusterloader"
- "github.com/aws/aws-k8s-tester/utils/log"
- "github.com/spf13/cobra"
- "go.uber.org/zap"
-)
-
-var rootCmd = &cobra.Command{
- Use: "k8s-tester-clusterloader",
- Short: "Kubernetes clusterloader tester",
- SuggestFor: []string{"clusterloader"},
-}
-
-func init() {
- cobra.EnablePrefixMatching = true
-}
-
-var (
- prompt bool
- logLevel string
- logOutputs []string
- minimumNodes int
- kubectlDownloadURL string
- kubectlPath string
- kubeconfigPath string
-)
-
-func init() {
- rootCmd.PersistentFlags().BoolVar(&prompt, "prompt", true, "'true' to enable prompt mode")
- rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", log.DefaultLogLevel, "Logging level")
- rootCmd.PersistentFlags().StringSliceVar(&logOutputs, "log-outputs", []string{"stderr"}, "Additional logger outputs")
- rootCmd.PersistentFlags().IntVar(&minimumNodes, "minimum-nodes", clusterloader.DefaultMinimumNodes, "minimum number of Kubernetes nodes required for installing this addon")
- rootCmd.PersistentFlags().StringVar(&kubectlDownloadURL, "kubectl-download-url", client.DefaultKubectlDownloadURL(), "kubectl download URL")
- rootCmd.PersistentFlags().StringVar(&kubectlPath, "kubectl-path", client.DefaultKubectlPath(), "kubectl path")
- rootCmd.PersistentFlags().StringVar(&kubeconfigPath, "kubeconfig-path", "", "KUBECONFIG path")
-
- rootCmd.AddCommand(
- newApply(),
- newDelete(),
- )
-}
-
-func main() {
- if err := rootCmd.Execute(); err != nil {
- fmt.Fprintf(os.Stderr, "k8s-tester-clusterloader failed %v\n", err)
- os.Exit(1)
- }
- os.Exit(0)
-}
-
-var (
- clusterloaderPath string
- clusterloaderDownloadURL string
-
- provider string
-
- runs int
- runTimeout time.Duration
-
- testConfigPath string
-
- runFromCluster bool
- nodes int
- enableExecService bool
-
- nodesPerNamespace int
- podsPerNode int
-
- bigGroupSize int
- mediumGroupSize int
- smallGroupSize int
-
- smallStatefulSetsPerNamespace int
- mediumStatefulSetsPerNamespace int
-
- cl2UseHostNetworkPods bool
- cl2LoadTestThroughput int
- cl2EnablePVS bool
- cl2SchedulerThroughputThreshold int
- prometheusScrapeKubeProxy bool
- enableSystemPodMetrics bool
-)
-
-func newApply() *cobra.Command {
- cmd := &cobra.Command{
- Use: "apply",
- Short: "Apply tests",
- Run: createApplyFunc,
- }
-
- cmd.PersistentFlags().StringVar(&clusterloaderPath, "clusterloader-path", clusterloader.DefaultClusterloaderPath(), "clusterloader path")
- cmd.PersistentFlags().StringVar(&clusterloaderDownloadURL, "clusterloader-download-url", clusterloader.DefaultClusterloaderDownloadURL(), "clusterloader download URL")
- cmd.PersistentFlags().StringVar(&provider, "provider", clusterloader.DefaultProvider, "clusterloader provider")
- cmd.PersistentFlags().IntVar(&runs, "runs", clusterloader.DefaultRuns, "clusterloader runs")
- cmd.PersistentFlags().DurationVar(&runTimeout, "run-timeout", clusterloader.DefaultRunTimeout, "clusterloader run timeout")
- cmd.PersistentFlags().StringVar(&testConfigPath, "test-config-path", "", "clusterloader test config path")
- cmd.PersistentFlags().BoolVar(&runFromCluster, "run-from-cluster", clusterloader.DefaultRunFromCluster, "to run clusterloader2 in cluster")
- cmd.PersistentFlags().IntVar(&nodes, "nodes", clusterloader.DefaultNodes, "clusterloader nodes")
- cmd.PersistentFlags().BoolVar(&enableExecService, "enable-exec-service", clusterloader.DefaultEnableExecService, "clusterloader enable exec service")
- cmd.PersistentFlags().IntVar(&nodesPerNamespace, "nodes-per-namespace", clusterloader.DefaultNodesPerNamespace, "clusterloader nodes per namespace")
- cmd.PersistentFlags().IntVar(&podsPerNode, "pods-per-node", clusterloader.DefaultPodsPerNode, "clusterloader pods per node")
- cmd.PersistentFlags().IntVar(&bigGroupSize, "big-group-size", clusterloader.DefaultBigGroupSize, "clusterloader big group size")
- cmd.PersistentFlags().IntVar(&mediumGroupSize, "medium-group-size", clusterloader.DefaultMediumGroupSize, "clusterloader medium group size")
- cmd.PersistentFlags().IntVar(&smallGroupSize, "small-group-size", clusterloader.DefaultSmallGroupSize, "clusterloader small group size")
- cmd.PersistentFlags().IntVar(&smallStatefulSetsPerNamespace, "small-stateful-sets-per-namespace", clusterloader.DefaultSmallStatefulSetsPerNamespace, "clusterloader small stateful sets per namespace")
- cmd.PersistentFlags().IntVar(&mediumStatefulSetsPerNamespace, "medium-stateful-sets-per-namespace", clusterloader.DefaultMediumStatefulSetsPerNamespace, "clusterloader medium stateful sets per namespace")
- cmd.PersistentFlags().BoolVar(&cl2UseHostNetworkPods, "cl2-use-host-network-pods", clusterloader.DefaultCL2UseHostNetworkPods, "clusterloader CL2 use host network pods")
- cmd.PersistentFlags().IntVar(&cl2LoadTestThroughput, "cl2-load-test-throughput", clusterloader.DefaultCL2LoadTestThroughput, "clusterloader CL2 load test throughput")
- cmd.PersistentFlags().BoolVar(&cl2EnablePVS, "cl2-enable-pvs", clusterloader.DefaultCL2UseHostNetworkPods, "clusterloader CL2 use host network pods")
- cmd.PersistentFlags().IntVar(&cl2SchedulerThroughputThreshold, "cl2-scheduler-throughput-threshold", clusterloader.DefaultCL2SchedulerThroughputThreshold, "clusterloader CL2 scheduler throughput threshold")
- cmd.PersistentFlags().BoolVar(&prometheusScrapeKubeProxy, "prometheus-scrape-kube-proxy", clusterloader.DefaultPrometheusScrapeKubeProxy, "clusterloader prometheus scrape kube-proxy")
- cmd.PersistentFlags().BoolVar(&enableSystemPodMetrics, "enable-system-pod-metrics", clusterloader.DefaultEnableSystemPodMetrics, "clusterloader enable system pod metrics")
-
- return cmd
-}
-
-func createApplyFunc(cmd *cobra.Command, args []string) {
- lg, logWriter, _, err := log.NewWithStderrWriter(logLevel, logOutputs)
- if err != nil {
- panic(err)
- }
- _ = zap.ReplaceGlobals(lg)
-
- cli, err := client.New(&client.Config{
- Logger: lg,
- KubectlDownloadURL: kubectlDownloadURL,
- KubectlPath: kubectlPath,
- KubeconfigPath: kubeconfigPath,
- })
- if err != nil {
- lg.Panic("failed to create client", zap.Error(err))
- }
-
- cfg := &clusterloader.Config{
- Prompt: prompt,
- Logger: lg,
- LogWriter: logWriter,
- MinimumNodes: minimumNodes,
- Client: cli,
-
- ClusterloaderPath: clusterloaderPath,
- ClusterloaderDownloadURL: clusterloaderDownloadURL,
-
- Provider: provider,
-
- Runs: runs,
- RunTimeout: runTimeout,
-
- TestConfigPath: testConfigPath,
-
- RunFromCluster: runFromCluster,
- Nodes: nodes,
- EnableExecService: enableExecService,
-
- TestOverride: &clusterloader.TestOverride{
- Path: clusterloader.DefaultTestOverridePath(),
-
- NodesPerNamespace: nodesPerNamespace,
- PodsPerNode: podsPerNode,
-
- BigGroupSize: bigGroupSize,
- MediumGroupSize: mediumGroupSize,
- SmallGroupSize: smallGroupSize,
-
- SmallStatefulSetsPerNamespace: smallStatefulSetsPerNamespace,
- MediumStatefulSetsPerNamespace: mediumStatefulSetsPerNamespace,
-
- CL2UseHostNetworkPods: cl2UseHostNetworkPods,
- CL2LoadTestThroughput: cl2LoadTestThroughput,
- CL2EnablePVS: cl2EnablePVS,
- CL2SchedulerThroughputThreshold: cl2SchedulerThroughputThreshold,
- PrometheusScrapeKubeProxy: prometheusScrapeKubeProxy,
- EnableSystemPodMetrics: enableSystemPodMetrics,
- },
- }
- if err := cfg.ValidateAndSetDefaults(); err != nil {
- fmt.Fprintf(os.Stderr, "failed to validate (%v)\n", err)
- os.Exit(1)
- }
-
- ts := clusterloader.New(cfg)
- if err := ts.Apply(); err != nil {
- fmt.Fprintf(os.Stderr, "failed to apply (%v)\n", err)
- os.Exit(1)
- }
-
- fmt.Printf("\n*********************************\n")
- fmt.Printf("'k8s-tester-clusterloader apply' success\n")
-}
-
-func newDelete() *cobra.Command {
- cmd := &cobra.Command{
- Use: "delete",
- Short: "Delete resources",
- Run: createDeleteFunc,
- }
- return cmd
-}
-
-func createDeleteFunc(cmd *cobra.Command, args []string) {
- lg, logWriter, _, err := log.NewWithStderrWriter(logLevel, logOutputs)
- if err != nil {
- panic(err)
- }
- _ = zap.ReplaceGlobals(lg)
-
- cli, err := client.New(&client.Config{
- Logger: lg,
- KubectlDownloadURL: kubectlDownloadURL,
- KubectlPath: kubectlPath,
- KubeconfigPath: kubeconfigPath,
- })
- if err != nil {
- lg.Panic("failed to create client", zap.Error(err))
- }
-
- cfg := &clusterloader.Config{
- Prompt: prompt,
- Logger: lg,
- LogWriter: logWriter,
- Client: cli,
- }
-
- ts := clusterloader.New(cfg)
- if err := ts.Delete(); err != nil {
- fmt.Fprintf(os.Stderr, "failed to delete (%v)\n", err)
- os.Exit(1)
- }
-
- fmt.Printf("\n*********************************\n")
- fmt.Printf("'k8s-tester-clusterloader delete' success\n")
-}
diff --git a/k8s-tester/clusterloader/test-data/PodStartupLatency-1.json b/k8s-tester/clusterloader/test-data/PodStartupLatency-1.json
deleted file mode 100644
index 9db859aa1..000000000
--- a/k8s-tester/clusterloader/test-data/PodStartupLatency-1.json
+++ /dev/null
@@ -1,60 +0,0 @@
-{
- "version": "1.0",
- "dataItems": [
- {
- "data": {
- "Perc50": 1000,
- "Perc90": 3000,
- "Perc99": 4000
- },
- "unit": "ms",
- "labels": {
- "Metric": "schedule_to_run"
- }
- },
- {
- "data": {
- "Perc50": 3672.31438,
- "Perc90": 5950.600599,
- "Perc99": 8124.999776
- },
- "unit": "ms",
- "labels": {
- "Metric": "run_to_watch"
- }
- },
- {
- "data": {
- "Perc50": 5419.452999,
- "Perc90": 7950.600599,
- "Perc99": 10124.999776
- },
- "unit": "ms",
- "labels": {
- "Metric": "schedule_to_watch"
- }
- },
- {
- "data": {
- "Perc50": 5419.452999,
- "Perc90": 7950.600599,
- "Perc99": 10124.999776
- },
- "unit": "ms",
- "labels": {
- "Metric": "pod_startup"
- }
- },
- {
- "data": {
- "Perc50": 0,
- "Perc90": 0,
- "Perc99": 0
- },
- "unit": "ms",
- "labels": {
- "Metric": "create_to_schedule"
- }
- }
- ]
-}
diff --git a/k8s-tester/clusterloader/test-data/PodStartupLatency-2.json b/k8s-tester/clusterloader/test-data/PodStartupLatency-2.json
deleted file mode 100644
index 33059f7d2..000000000
--- a/k8s-tester/clusterloader/test-data/PodStartupLatency-2.json
+++ /dev/null
@@ -1,60 +0,0 @@
-{
- "version": "1.0",
- "dataItems": [
- {
- "data": {
- "Perc50": 1000,
- "Perc90": 2000,
- "Perc99": 3000
- },
- "unit": "ms",
- "labels": {
- "Metric": "schedule_to_run"
- }
- },
- {
- "data": {
- "Perc50": 2481.509542,
- "Perc90": 5090.485946,
- "Perc99": 6498.200061
- },
- "unit": "ms",
- "labels": {
- "Metric": "run_to_watch"
- }
- },
- {
- "data": {
- "Perc50": 3902.70332,
- "Perc90": 6691.346533,
- "Perc99": 7898.422761
- },
- "unit": "ms",
- "labels": {
- "Metric": "schedule_to_watch"
- }
- },
- {
- "data": {
- "Perc50": 3902.70332,
- "Perc90": 6691.346533,
- "Perc99": 7898.422761
- },
- "unit": "ms",
- "labels": {
- "Metric": "pod_startup"
- }
- },
- {
- "data": {
- "Perc50": 0,
- "Perc90": 0,
- "Perc99": 0
- },
- "unit": "ms",
- "labels": {
- "Metric": "create_to_schedule"
- }
- }
- ]
-}
diff --git a/k8s-tester/clusterloader/test-data/PodStartupLatency-3.json b/k8s-tester/clusterloader/test-data/PodStartupLatency-3.json
deleted file mode 100644
index 72630d19a..000000000
--- a/k8s-tester/clusterloader/test-data/PodStartupLatency-3.json
+++ /dev/null
@@ -1,60 +0,0 @@
-{
- "version": "1.0",
- "dataItems": [
- {
- "data": {
- "Perc50": 2870.804716,
- "Perc90": 6642.876753,
- "Perc99": 9695.783054
- },
- "unit": "ms",
- "labels": {
- "Metric": "run_to_watch"
- }
- },
- {
- "data": {
- "Perc50": 4842.424464,
- "Perc90": 8896.50057,
- "Perc99": 12095.952701
- },
- "unit": "ms",
- "labels": {
- "Metric": "schedule_to_watch"
- }
- },
- {
- "data": {
- "Perc50": 4842.424464,
- "Perc90": 8896.50057,
- "Perc99": 12095.952701
- },
- "unit": "ms",
- "labels": {
- "Metric": "pod_startup"
- }
- },
- {
- "data": {
- "Perc50": 0,
- "Perc90": 0,
- "Perc99": 0
- },
- "unit": "ms",
- "labels": {
- "Metric": "create_to_schedule"
- }
- },
- {
- "data": {
- "Perc50": 2000,
- "Perc90": 3000,
- "Perc99": 10000
- },
- "unit": "ms",
- "labels": {
- "Metric": "schedule_to_run"
- }
- }
- ]
-}
diff --git a/k8s-tester/clusterloader/test-data/PodStartupLatency-4.json b/k8s-tester/clusterloader/test-data/PodStartupLatency-4.json
deleted file mode 100644
index e18c611b9..000000000
--- a/k8s-tester/clusterloader/test-data/PodStartupLatency-4.json
+++ /dev/null
@@ -1,60 +0,0 @@
-{
- "version": "1.0",
- "dataItems": [
- {
- "data": {
- "Perc50": 0,
- "Perc90": 0,
- "Perc99": 0
- },
- "unit": "ms",
- "labels": {
- "Metric": "create_to_schedule"
- }
- },
- {
- "data": {
- "Perc50": 1000,
- "Perc90": 2000,
- "Perc99": 3000
- },
- "unit": "ms",
- "labels": {
- "Metric": "schedule_to_run"
- }
- },
- {
- "data": {
- "Perc50": 3463.640676,
- "Perc90": 5999.036186,
- "Perc99": 7136.509223
- },
- "unit": "ms",
- "labels": {
- "Metric": "run_to_watch"
- }
- },
- {
- "data": {
- "Perc50": 5108.634699,
- "Perc90": 7737.411797,
- "Perc99": 8536.134803
- },
- "unit": "ms",
- "labels": {
- "Metric": "schedule_to_watch"
- }
- },
- {
- "data": {
- "Perc50": 5108.634699,
- "Perc90": 7737.411797,
- "Perc99": 8536.134803
- },
- "unit": "ms",
- "labels": {
- "Metric": "pod_startup"
- }
- }
- ]
-}
diff --git a/k8s-tester/clusterloader/test-data/PodStartupLatency-5.json b/k8s-tester/clusterloader/test-data/PodStartupLatency-5.json
deleted file mode 100644
index ecf189795..000000000
--- a/k8s-tester/clusterloader/test-data/PodStartupLatency-5.json
+++ /dev/null
@@ -1,60 +0,0 @@
-{
- "version": "1.0",
- "dataItems": [
- {
- "data": {
- "Perc50": 0,
- "Perc90": 0,
- "Perc99": 1000
- },
- "unit": "ms",
- "labels": {
- "Metric": "create_to_schedule"
- }
- },
- {
- "data": {
- "Perc50": 2000,
- "Perc90": 2000,
- "Perc99": 3000
- },
- "unit": "ms",
- "labels": {
- "Metric": "schedule_to_run"
- }
- },
- {
- "data": {
- "Perc50": 2715.294357,
- "Perc90": 5684.517774,
- "Perc99": 8545.86331
- },
- "unit": "ms",
- "labels": {
- "Metric": "run_to_watch"
- }
- },
- {
- "data": {
- "Perc50": 4323.362793,
- "Perc90": 7474.832147,
- "Perc99": 10346.204899
- },
- "unit": "ms",
- "labels": {
- "Metric": "schedule_to_watch"
- }
- },
- {
- "data": {
- "Perc50": 4324.31988,
- "Perc90": 7474.832147,
- "Perc99": 10346.204899
- },
- "unit": "ms",
- "labels": {
- "Metric": "pod_startup"
- }
- }
- ]
-}
diff --git a/k8s-tester/clusterloader/tester.go b/k8s-tester/clusterloader/tester.go
deleted file mode 100644
index e0f510bcf..000000000
--- a/k8s-tester/clusterloader/tester.go
+++ /dev/null
@@ -1,715 +0,0 @@
-// Package clusterloader installs clusterloader.
-// Replace https://github.com/aws/aws-k8s-tester/tree/v1.5.9/eks/cluster-loader.
-package clusterloader
-
-import (
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "io/ioutil"
- "os"
- "os/exec"
- "path"
- "path/filepath"
- "reflect"
- "strings"
- "sync"
- "time"
-
- "github.com/aws/aws-k8s-tester/client"
- k8s_tester "github.com/aws/aws-k8s-tester/k8s-tester/tester"
- "github.com/aws/aws-k8s-tester/utils/file"
- "github.com/dustin/go-humanize"
- "github.com/manifoldco/promptui"
- "github.com/mholt/archiver/v3"
- "go.uber.org/zap"
-)
-
-// TODO: support s3 uploads
-
-// Config defines parameters for Kubernetes clusterloader tests.
-type Config struct {
- Enable bool `json:"enable"`
- Prompt bool `json:"-"`
-
- Stopc chan struct{} `json:"-"`
- Logger *zap.Logger `json:"-"`
- LogWriter io.Writer `json:"-"`
- Client client.Client `json:"-"`
-
- // MinimumNodes is the minimum number of Kubernetes nodes required for installing this addon.
- MinimumNodes int `json:"minimum_nodes"`
-
- // ClusterloaderPath is the path to download the "clusterloader".
- ClusterloaderPath string `json:"clusterloader_path"`
- // ClusterloaderDownloadURL is the download URL to download "clusterloader" binary from.
- ClusterloaderDownloadURL string `json:"clusterloader_download_url"`
-
- // Provider is the provider name for "clusterloader2".
- Provider string `json:"provider"`
-
- // Runs is the number of "clusterloader2" runs back-to-back.
- Runs int `json:"runs"`
- // RunTimeout is the timeout for the total test runs.
- RunTimeout time.Duration `json:"run_timeout"`
- RunTimeoutString string `json:"run_timeout_string" read-only:"true"`
-
- // TestConfigPath is the clusterloader2 test configuration file.
- // Must be located along with other configuration files.
- // For instance, if the clusterloader2 default configuration file is located at
- // ${HOME}/go/src/k8s.io/perf-tests/clusterloader2/testing/load/config.yaml,
- // then run this tester from "${HOME}/go/src/k8s.io/perf-tests/clusterloader2".
- // ref. https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/testing/load/config.yaml
- // Set via "--testconfig" flag.
- TestConfigPath string `json:"test_config_path"`
-
- // RunFromCluster is set 'true' to override KUBECONFIG set in "Client" field.
- // If "false", instead pass Client.Config().KubeconfigPath to "--kubeconfig" flag.
- // Set via "--run-from-cluster" flag.
- // ref. https://github.com/kubernetes/perf-tests/pull/1295
- RunFromCluster bool `json:"run_from_cluster"`
- // Nodes is the number of nodes.
- // Set via "--nodes" flag.
- Nodes int `json:"nodes"`
- // EnableExecService is set to "true" to allow executing arbitrary commands from a pod running in the cluster.
- // Set via "--enable-exec-service" flag.
- // ref. https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/cmd/clusterloader.go#L120
- EnableExecService bool `json:"enable_exec_service"`
-
- // TestOverride defines "testoverrides" flag values.
- // Set via "--testoverrides" flag.
- // See https://github.com/kubernetes/perf-tests/tree/master/clusterloader2/testing/overrides for more.
- // ref. https://github.com/kubernetes/perf-tests/pull/1345
- TestOverride *TestOverride `json:"test_override"`
-
- // TestReportDir is the clusterloader2 test report output directory.
- // Set via "--report-dir" flag.
- TestReportDir string `json:"test_report_dir" read-only:"true"`
- // TestReportDirTarGzPath is the test report .tar.gz file path.
- TestReportDirTarGzPath string `json:"test_report_dir_tar_gz_path" read-only:"true"`
- // TestLogPath is the "clusterloader2" test log file path.
- TestLogPath string `json:"test_log_path" read-only:"true"`
- // PodStartupLatency is the result of clusterloader runs.
- PodStartupLatency PerfData `json:"pod_startup_latency" read-only:"true"`
- // PodStartupLatencyPath is the JSON file path to store pod startup latency.
- PodStartupLatencyPath string `json:"pod_startup_latency_path" read-only:"true"`
-}
-
-func (cfg *Config) ValidateAndSetDefaults() error {
- if cfg.ClusterloaderPath == "" {
- cfg.ClusterloaderPath = DefaultClusterloaderPath()
- }
- if cfg.ClusterloaderDownloadURL == "" {
- cfg.ClusterloaderDownloadURL = DefaultClusterloaderDownloadURL()
- }
-
- if cfg.Runs == 0 {
- return fmt.Errorf("invalid Runs %d", cfg.Runs)
- }
- if cfg.RunTimeout == time.Duration(0) {
- cfg.RunTimeout = DefaultRunTimeout
- }
- cfg.RunTimeoutString = cfg.RunTimeout.String()
-
- if !file.Exist(cfg.TestConfigPath) {
- return fmt.Errorf("TestConfigPath %q does not exist", cfg.TestConfigPath)
- }
-
- if cfg.Nodes == 0 {
- cfg.Nodes = cfg.MinimumNodes
- }
-
- if cfg.TestReportDir == "" {
- cfg.TestReportDir = DefaultTestReportDir()
- }
- if cfg.TestReportDirTarGzPath == "" {
- cfg.TestReportDirTarGzPath = DefaultTestReportDirTarGzPath()
- }
- if !strings.HasSuffix(cfg.TestReportDirTarGzPath, ".tar.gz") {
- return fmt.Errorf("TestReportDirTarGzPath %q requires .tar.gz suffix", cfg.TestReportDirTarGzPath)
- }
- if cfg.TestLogPath == "" {
- cfg.TestLogPath = DefaultTestLogPath()
- }
- if cfg.PodStartupLatencyPath == "" {
- cfg.PodStartupLatencyPath = DefaultPodStartupLatencyPath()
- }
-
- return nil
-}
-
-var (
- unixNano = time.Now().UnixNano()
- defaultTestReportDir = filepath.Join(os.TempDir(), fmt.Sprintf("clusterloader-test-report-dir-%x", unixNano))
- defaultTestReportDirTarGzPath = filepath.Join(os.TempDir(), fmt.Sprintf("clusterloader-test-report-dir-%x.tar.gz", unixNano))
- defaultTestOverridePath = filepath.Join(defaultTestReportDir, fmt.Sprintf("clusterloader-test-overrides-%x.yaml", unixNano))
- defaultTestLogPath = filepath.Join(defaultTestReportDir, fmt.Sprintf("clusterloader-test-log-%x.log", unixNano))
- defaultPodStartupLatencyPath = filepath.Join(defaultTestReportDir, fmt.Sprintf("clusterloader-pod-startup-latency-%x.json", unixNano))
-)
-
-func DefaultTestOverridePath() string {
- return defaultTestOverridePath
-}
-
-func DefaultTestReportDir() string {
- return defaultTestReportDir
-}
-
-func DefaultTestReportDirTarGzPath() string {
- return defaultTestReportDirTarGzPath
-}
-
-func DefaultTestLogPath() string {
- return defaultTestLogPath
-}
-
-func DefaultPodStartupLatencyPath() string {
- return defaultPodStartupLatencyPath
-}
-
-const (
- DefaultMinimumNodes int = 1
-
- DefaultRuns = 2
- DefaultRunTimeout = 30 * time.Minute
-
- DefaultRunFromCluster = false
- DefaultNodes = 10
- DefaultEnableExecService = false
-)
-
-func NewDefault() *Config {
- return &Config{
- Enable: false,
- Prompt: false,
- MinimumNodes: DefaultMinimumNodes,
-
- ClusterloaderPath: DefaultClusterloaderPath(),
- ClusterloaderDownloadURL: DefaultClusterloaderDownloadURL(),
-
- Provider: DefaultProvider,
-
- Runs: DefaultRuns,
- RunTimeout: DefaultRunTimeout,
-
- RunFromCluster: DefaultRunFromCluster,
- Nodes: DefaultNodes,
- EnableExecService: DefaultEnableExecService,
-
- TestOverride: newDefaultTestOverride(),
-
- TestReportDir: DefaultTestReportDir(),
- TestReportDirTarGzPath: DefaultTestReportDirTarGzPath(),
- TestLogPath: DefaultTestLogPath(),
- PodStartupLatencyPath: DefaultPodStartupLatencyPath(),
- }
-}
-
-func New(cfg *Config) k8s_tester.Tester {
- return &tester{
- cfg: cfg,
-
- donec: make(chan struct{}),
- donecCloseOnce: new(sync.Once),
- }
-}
-
-type tester struct {
- cfg *Config
- testLogFile *os.File
-
- donec chan struct{}
- donecCloseOnce *sync.Once
-
- rootCtx context.Context
- rootCancel context.CancelFunc
-}
-
-var pkgName = path.Base(reflect.TypeOf(tester{}).PkgPath())
-
-func Env() string {
- return "ADD_ON_" + strings.ToUpper(strings.Replace(pkgName, "-", "_", -1))
-}
-
-func EnvTestOverride() string {
- return Env() + "_TEST_OVERRIDE"
-}
-
-func (ts *tester) Name() string { return pkgName }
-
-func (ts *tester) Enabled() bool { return ts.cfg.Enable }
-
-func (ts *tester) Apply() (err error) {
- if ok := ts.runPrompt("apply"); !ok {
- return errors.New("cancelled")
- }
-
- if ts.cfg.MinimumNodes > 0 {
- if nodes, err := client.ListNodes(ts.cfg.Client.KubernetesClient()); len(nodes) < ts.cfg.MinimumNodes || err != nil {
- return fmt.Errorf("failed to validate minimum nodes requirement %d (nodes %v, error %v)", ts.cfg.MinimumNodes, len(nodes), err)
- }
- }
-
- if err = installClusterloader(ts.cfg.Logger, ts.cfg.ClusterloaderPath, ts.cfg.ClusterloaderDownloadURL); err != nil {
- return err
- }
-
- if err = os.MkdirAll(ts.cfg.TestReportDir, 0700); err != nil {
- return err
- }
- if err = file.IsDirWriteable(ts.cfg.TestReportDir); err != nil {
- return err
- }
- ts.cfg.Logger.Info("mkdir report dir", zap.String("dir", ts.cfg.TestReportDir))
-
- if err := ts.cfg.TestOverride.Sync(ts.cfg.Logger); err != nil {
- return err
- }
-
- ts.testLogFile, err = os.OpenFile(ts.cfg.TestLogPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
- if err != nil {
- return err
- }
- defer func() {
- ts.testLogFile.Sync()
- ts.testLogFile.Close()
- }()
-
- checkDonec := ts.streamTestLogs()
- runErr := ts.runCL2s(checkDonec)
-
- testFinishedCount, err := ts.countTestFinishes()
- if err != nil {
- return err
- }
-
- podStartupLats, err := ts.appendResultsToTestLogPath()
- if err != nil {
- return err
- }
- if err = ts.collectPodStartupLatency(podStartupLats); err != nil {
- return err
- }
-
- if err = ts.compressReports(); err != nil {
- return err
- }
-
- if testFinishedCount == ts.cfg.Runs {
- ts.cfg.Logger.Info("completed expected test runs; overriding error",
- zap.Int("finished-count", testFinishedCount),
- zap.Int("expected-runs", ts.cfg.Runs),
- zap.Error(runErr),
- )
- runErr = nil
- } else {
- ts.cfg.Logger.Warn("failed to complete expected test runs",
- zap.Int("finished-count", testFinishedCount),
- zap.Int("expected-runs", ts.cfg.Runs),
- zap.Error(runErr),
- )
- completeErr := fmt.Errorf("failed to complete expected test runs [expected %d, completed %d]", ts.cfg.Runs, testFinishedCount)
- if runErr == nil {
- runErr = completeErr
- } else {
- runErr = fmt.Errorf("%v (run error: %v)", completeErr, runErr)
- }
- }
- return runErr
-}
-
-func (ts *tester) Delete() error {
- if ok := ts.runPrompt("delete"); !ok {
- return errors.New("cancelled")
- }
-
- var errs []string
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- return nil
-}
-
-func (ts *tester) runPrompt(action string) (ok bool) {
- if ts.cfg.Prompt {
- msg := fmt.Sprintf("Ready to %q resources, should we continue?", action)
- prompt := promptui.Select{
- Label: msg,
- Items: []string{
- "No, cancel it!",
- fmt.Sprintf("Yes, let's %q!", action),
- },
- }
- idx, answer, err := prompt.Run()
- if err != nil {
- panic(err)
- }
- if idx != 1 {
- fmt.Printf("cancelled %q [index %d, answer %q]\n", action, idx, answer)
- return false
- }
- }
- return true
-}
-
-// stream command run outputs for debugging purposes
-func (ts *tester) streamTestLogs() (checkDonec chan struct{}) {
- checkDonec = make(chan struct{})
- go func() {
- defer func() {
- close(checkDonec)
- }()
- for {
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Info("exiting cluster loader command output checks")
- return
- case <-ts.rootCtx.Done():
- ts.cfg.Logger.Info("exiting cluster loader command output checks")
- return
- case <-time.After(10 * time.Second):
- }
-
- if ts.testLogFile != nil {
- ts.testLogFile.Sync()
- }
- b, lerr := ioutil.ReadFile(ts.cfg.TestLogPath)
- if lerr != nil {
- ts.cfg.Logger.Warn("failed to read cluster loader command output from logs file", zap.Error(lerr))
- continue
- }
- output := tailLogs(b)
- linesN := strings.Count(output, "\n")
-
- ts.cfg.Logger.Info("checked cluster loader command output from logs file", zap.Int("total-lines", linesN))
- fmt.Fprintf(ts.cfg.LogWriter, "\n%q output:\n%s\n\n", ts.cfg.TestLogPath, output)
- }
- }()
- return checkDonec
-}
-
-func (ts *tester) getCL2Args() (args []string) {
- args = []string{
- ts.cfg.ClusterloaderPath,
- "--logtostderr", // log to standard error instead of files (default true)
- "--alsologtostderr", // log to standard error as well as files
- fmt.Sprintf("--enable-exec-service=%v", ts.cfg.EnableExecService),
- "--testconfig=" + ts.cfg.TestConfigPath,
- "--testoverrides=" + ts.cfg.TestOverride.Path,
- "--report-dir=" + ts.cfg.TestReportDir,
- "--nodes=" + fmt.Sprintf("%d", ts.cfg.Nodes),
- "--provider=" + ts.cfg.Provider,
- }
- if ts.cfg.RunFromCluster {
- // ref. https://github.com/kubernetes/perf-tests/pull/1295
- args = append(args, "--run-from-cluster=true")
- } else {
- args = append(args, "--kubeconfig="+ts.cfg.Client.Config().KubeconfigPath)
- }
- return args
-}
-
-/*
-E0610 03:20:23.917606 16894 simple_test_executor.go:391] Resource cleanup error: [timed out waiting for the condition
-timed out waiting for the condition]
-I0610 03:20:23.917636 16894 clusterloader.go:227] --------------------------------------------------------------------------------
-I0610 03:20:23.917650 16894 clusterloader.go:228] Test Finished
-I0610 03:20:23.917666 16894 clusterloader.go:229] Test: ./testing/load/config.yaml
-I0610 03:20:23.917681 16894 clusterloader.go:230] Status: Success
-I0610 03:20:23.917693 16894 clusterloader.go:234] --------------------------------------------------------------------------------
-
-/tmp/kubectl-v1.21.1 --kubeconfig=/tmp/eks-2021060922-sunahvg04tie.kubeconfig.yaml get ns
-{
- "lastTransitionTime": "2021-06-10T03:10:28Z",
- "message": "Discovery failed for some groups, 1 failing: unable to retrieve the complete list of server APIs: metrics.k8s.io/v1beta1: the server is currently unable to handle the request",
- "reason": "DiscoveryFailed",
- "status": "True",
- "type": "NamespaceDeletionDiscoveryFailure"
-},
-*/
-// each clusterloader2 run takes about 2-minute
-// but may stuck with test namespace deletion
-func (ts *tester) runCL2(idx int, args []string) (err error) {
- ts.cfg.Logger.Info("running clusterloader2", zap.Int("index", idx), zap.String("command", strings.Join(args, " ")))
- timeout := 3 * time.Minute
- ctx, cancel := context.WithTimeout(ts.rootCtx, 2*timeout)
- cmd := exec.CommandContext(ctx, args[0], args[1:]...)
- cmd.Stderr = ts.testLogFile
- cmd.Stdout = ts.testLogFile
- err = cmd.Start()
- if err != nil {
- return err
- }
- errc := make(chan error)
- go func() {
- errc <- cmd.Wait()
- }()
- select {
- case <-ctx.Done():
- return fmt.Errorf("command context timeout %v", err)
- case <-time.After(timeout):
- cancel()
- ts.cfg.Logger.Warn("command timeout, gracefully interrupting", zap.String("timeout", timeout.String()))
- iterr := cmd.Process.Signal(os.Interrupt)
- time.Sleep(5 * time.Second)
- ts.cfg.Logger.Warn("interrupted command", zap.Error(iterr))
- err = fmt.Errorf("command timeout after %v", timeout)
- case err = <-errc:
- cancel()
- return fmt.Errorf("command failed %v", err)
- }
- return err
-}
-
-func (ts *tester) runCL2s(checkDonec chan struct{}) (runErr error) {
- args := ts.getCL2Args()
- now := time.Now()
- errc := make(chan error)
- ts.rootCtx, ts.rootCancel = context.WithTimeout(context.Background(), ts.cfg.RunTimeout)
- go func() {
- for i := 0; i < ts.cfg.Runs; i++ {
- select {
- case <-ts.rootCtx.Done():
- return
- case <-time.After(5 * time.Second):
- }
-
- rerr := ts.runCL2(i, args)
- if rerr == nil {
- ts.cfg.Logger.Info("completed cluster loader", zap.Int("current-run", i), zap.Int("total-runs", ts.cfg.Runs))
- continue
- }
-
- ts.cfg.Logger.Warn("checking cluster loader error from log file", zap.Error(rerr))
- b, lerr := ioutil.ReadFile(ts.cfg.TestLogPath)
- if lerr != nil {
- ts.cfg.Logger.Warn("failed to read cluster loader error from logs file", zap.Error(lerr))
- errc <- rerr
- return
- }
- output := tailLogs(b)
-
- if strings.Contains(output, `Status: Success`) {
- // e.g., "Resource cleanup error: [timed out)"...
- ts.cfg.Logger.Warn("cluster loader command exited but continue for its success status")
- continue
- }
- if strings.Contains(output, `PodStartupLatency: perc`) {
- ts.cfg.Logger.Warn("cluster loader command exited but continue for its success report")
- continue
- }
- if strings.Contains(output, skipErr) {
- ts.cfg.Logger.Warn("cluster loader failed but continue", zap.String("skip-error-message", skipErr))
- continue
- }
-
- errc <- rerr
- return
- }
- errc <- nil
- }()
- select {
- case <-ts.donec:
- ts.cfg.Logger.Info("done cluster loader")
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Info("stopping cluster loader")
- case <-ts.rootCtx.Done():
- ts.cfg.Logger.Info("timed out cluster loader")
- case runErr = <-errc:
- if runErr == nil {
- ts.cfg.Logger.Info("successfully ran cluster loader",
- zap.String("took", time.Since(now).String()),
- zap.Int("total-runs", ts.cfg.Runs),
- )
- } else {
- ts.cfg.Logger.Warn("failed to run cluster loader",
- zap.String("took", time.Since(now).String()),
- zap.Int("total-runs", ts.cfg.Runs),
- zap.Error(runErr),
- )
- }
- }
- ts.rootCancel()
- select {
- case <-checkDonec:
- ts.cfg.Logger.Info("confirmed exit cluster loader command output checks")
- case <-time.After(3 * time.Minute):
- ts.cfg.Logger.Warn("took too long to confirm exit cluster loader command output checks")
- }
- if runErr != nil {
- ts.cfg.Logger.Warn("failed to run cluster loader", zap.Error(runErr))
- } else {
- ts.cfg.Logger.Info("successfully ran cluster loader")
- }
- return runErr
-}
-
-func (ts *tester) countTestFinishes() (testFinishedCount int, err error) {
- ts.cfg.Logger.Info("counting test finishes", zap.String("test-log-path", ts.cfg.TestLogPath))
- lout, err := ioutil.ReadFile(ts.cfg.TestLogPath)
- if err != nil {
- return 0, err
- }
- logOutput := string(lout)
- testFinishedCount = strings.Count(logOutput, `] Test Finished`)
- return testFinishedCount, nil
-}
-
-func (ts *tester) appendResultsToTestLogPath() (podStartupLats []PerfData, err error) {
- // append results in "TestLogPath"
- // "0777" to fix "scp: /var/log/cluster-loader-remote.log: Permission denied"
- logFile, cerr := os.OpenFile(ts.cfg.TestLogPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0777)
- if cerr != nil {
- return nil, fmt.Errorf("open(%q): %v", ts.cfg.TestLogPath, cerr)
- }
- defer logFile.Close()
-
- podStartupLats = make([]PerfData, 0)
- cerr = filepath.Walk(ts.cfg.TestReportDir, func(path string, info os.FileInfo, ferr error) error {
- if ferr != nil {
- return ferr
- }
- if info.IsDir() {
- return nil
- }
- ts.cfg.Logger.Info("found report", zap.String("path", path))
-
- if strings.HasPrefix(filepath.Base(path), "PodStartupLatency_") {
- ts.cfg.Logger.Info("parsing PodStartupLatency", zap.String("path", path))
- p, perr := parsePodStartupLatency(path)
- if perr != nil {
- ts.cfg.Logger.Warn("failed to parse PodStartupLatency", zap.String("path", path))
- return perr
- }
- ts.cfg.Logger.Info("parsed PodStartupLatency", zap.String("path", path))
- podStartupLats = append(podStartupLats, p)
- }
-
- if _, werr := logFile.WriteString(fmt.Sprintf("\n\n\nreport output from %q:\n\n", path)); werr != nil {
- ts.cfg.Logger.Warn("failed to write report to log file", zap.Error(werr))
- return nil
- }
-
- b, lerr := ioutil.ReadFile(path)
- if lerr != nil {
- ts.cfg.Logger.Warn("failed to read cluster loader command output from logs file", zap.Error(lerr))
- if _, werr := logFile.WriteString(fmt.Sprintf("failed to write %v", lerr)); werr != nil {
- ts.cfg.Logger.Warn("failed to write report to log file", zap.Error(werr))
- return nil
- }
- } else {
- if _, werr := logFile.Write(b); werr != nil {
- ts.cfg.Logger.Warn("failed to write report to log file", zap.Error(werr))
- return nil
- }
- }
- return nil
- })
- return podStartupLats, cerr
-}
-
-func (ts *tester) collectPodStartupLatency(podStartupLats []PerfData) error {
- ts.cfg.PodStartupLatency = mergePodStartupLatency(podStartupLats...)
- podStartupLatData, err := json.Marshal(ts.cfg.PodStartupLatency)
- if err != nil {
- return err
- }
- if err := ioutil.WriteFile(ts.cfg.PodStartupLatencyPath, podStartupLatData, 0600); err != nil {
- return err
- }
- return nil
-}
-
-func (ts *tester) compressReports() error {
- ts.cfg.Logger.Info("tar-gzipping report dir", zap.String("report-dir", ts.cfg.TestReportDir), zap.String("file-path", ts.cfg.TestReportDirTarGzPath))
- if err := os.RemoveAll(ts.cfg.TestReportDirTarGzPath); err != nil {
- ts.cfg.Logger.Warn("failed to remove temp file", zap.Error(err))
- return err
- }
-
- if err := archiver.Archive([]string{ts.cfg.TestReportDir}, ts.cfg.TestReportDirTarGzPath); err != nil {
- ts.cfg.Logger.Warn("archive failed", zap.Error(err))
- return err
- }
- stat, err := os.Stat(ts.cfg.TestReportDirTarGzPath)
- if err != nil {
- ts.cfg.Logger.Warn("failed to os stat", zap.Error(err))
- return err
- }
-
- sz := humanize.Bytes(uint64(stat.Size()))
- ts.cfg.Logger.Info("tar-gzipped report dir", zap.String("report-dir", ts.cfg.TestReportDir), zap.String("file-path", ts.cfg.TestReportDirTarGzPath), zap.String("file-size", sz))
- return nil
-}
-
-/*
-DO NOT FAIL THE TEST JUST BECAUSE IT CANNOT GET METRICS
-
-I0620 10:48:09.278149 256 simple_test_executor.go:384] Resources cleanup time: 15.009539312s
-E0620 10:48:09.278189 256 clusterloader.go:219] --------------------------------------------------------------------------------
-E0620 10:48:09.278193 256 clusterloader.go:220] Test Finished
-E0620 10:48:09.278196 256 clusterloader.go:221] Test: /clusterloader2-test-config.yaml
-E0620 10:48:09.278199 256 clusterloader.go:222] Status: Fail
-E0620 10:48:09.278202 256 clusterloader.go:224] Errors: [measurement call TestMetrics - TestMetrics error: [action start failed for SchedulingMetrics measurement: unexpected error (code: 0) in ssh connection to master: &errors.errorString{s:"error getting signer for provider : 'GetSigner(...) not implemented for '"}]
-measurement call TestMetrics - TestMetrics error: [action gather failed for SchedulingMetrics measurement: unexpected error (code: 0) in ssh connection to master: &errors.errorString{s:"error getting signer for provider : 'GetSigner(...) not implemented for '"}]]
-E0620 10:48:09.278206 256 clusterloader.go:226] --------------------------------------------------------------------------------
-
-JUnit report was created: /data/eks-2020062010-exclusiver66-cluster-loader-local-report/junit.xml
-F0620 10:48:09.278371 256 clusterloader.go:329] 1 tests have failed!
-
-
-E0621 01:15:53.003734 415 test_metrics.go:226] TestMetrics: [action gather failed for SchedulingMetrics measurement: unexpected error (code: 0) in ssh connection to master: &errors.errorString{s:"error getting signer for provider : 'GetSigner(...) not implemented for '"}]
-I0621 01:15:53.003760 415 simple_test_executor.go:162] Step "Collecting measurements" ended
-W0621 01:15:53.003766 415 simple_test_executor.go:165] Got errors during step execution: [measurement call TestMetrics - TestMetrics error: [action gather failed for SchedulingMetrics measurement: unexpected error (code: 0) in ssh connection to master: &errors.errorString{s:"error getting signer for provider : 'GetSigner(...) not implemented for '"}]]
-I0621 01:15:53.003789 415 simple_test_executor.go:72] Waiting for the chaos monkey subroutine to end...
-I0621 01:15:53.003795 415 simple_test_executor.go:74] Chaos monkey ended.
-I0621 01:15:53.007928 415 simple_test_executor.go:94]
-{"level":"info","ts":"2020-06-21T01:16:20.231-0700","caller":"cluster-loader/cluster-loader.go:201","msg":"checked cluster loader command output from logs file","total-lines":153}
-I0621 01:15:53.007938 415 probes.go:131] Probe DnsLookupLatency wasn't started, skipping the Dispose() step
-I0621 01:15:53.007977 415 probes.go:131] Probe InClusterNetworkLatency wasn't started, skipping the Dispose() step
-*/
-
-/*
-DO NOT FAIL THE TEST JUST BECAUSE IT CANNOT TEAR DOWN EXEC
-
-I0610 01:33:32.780753 7596 clusterloader.go:228] Test Finished
-I0610 01:33:32.780758 7596 clusterloader.go:229] Test: ./testing/load/config.yaml
-I0610 01:33:32.780763 7596 clusterloader.go:230] Status: Success
-I0610 01:33:32.780768 7596 clusterloader.go:234] --------------------------------------------------------------------------------
-
-JUnit report was created: /tmp/clusterloader-test-report-dir-168713f4aacf6138/junit.xml
-E0610 01:43:32.811103 7596 clusterloader.go:359] Error while tearing down exec service: timed out waiting for the condition
-*/
-
-/*
-I0610 07:56:44.169615 27280 phase_latency.go:146] PodStartupLatency: perc50: 6.183378663s, perc90: 11.146006032s, perc99: 12.55199317s
-I0610 07:56:44.169689 27280 phase_latency.go:146] PodStartupLatency: perc50: 8.048347299s, perc90: 14.55199317s, perc99: 16.545840721s
-I0610 07:56:44.169761 27280 phase_latency.go:146] PodStartupLatency: perc50: 8.048347299s, perc90: 14.980628148s, perc99: 16.545840721s; threshold 1h0m0s
-*/
-
-const skipErr = `action gather failed for SchedulingMetrics`
-
-func tailLogs(b []byte) string {
- output := strings.TrimSpace(string(b))
- lines := strings.Split(output, "\n")
- newLines := make([]string, 0, len(lines))
- for _, line := range lines {
- // remove "fetching profile data from is not possible from provider: eks"
- if strings.Contains(line, "profile.go") &&
- strings.Contains(line, "fetching profile") &&
- strings.Contains(line, "eks") {
- continue
- }
- newLines = append(newLines, line)
- }
- newLinesN := len(newLines)
- if newLinesN > 15 {
- newLines = newLines[newLinesN-15:]
- }
- output = strings.Join(newLines, "\n")
- return output
-}
diff --git a/k8s-tester/clusterloader/vend.sh b/k8s-tester/clusterloader/vend.sh
deleted file mode 100755
index 7937e2980..000000000
--- a/k8s-tester/clusterloader/vend.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-< 0 {
- return errors.New(strings.Join(errs, ", "))
- }
- return nil
-}
-
-func (ts *tester) runPrompt(action string) (ok bool) {
- if ts.cfg.Prompt {
- msg := fmt.Sprintf("Ready to %q resources for the namespace %q, should we continue?", action, ts.cfg.Namespace)
- prompt := promptui.Select{
- Label: msg,
- Items: []string{
- "No, cancel it!",
- fmt.Sprintf("Yes, let's %q!", action),
- },
- }
- idx, answer, err := prompt.Run()
- if err != nil {
- panic(err)
- }
- if idx != 1 {
- fmt.Printf("cancelled %q [index %d, answer %q]\n", action, idx, answer)
- return false
- }
- }
- return true
-}
-
-func (ts *tester) checkCNI() error {
- // List the pods in the namespace
- daemonsetlist, err := client.ListDaemonSets(
- ts.cfg.Logger,
- ts.cfg.Client.KubernetesClient(),
- ts.cfg.CNINamespace,
- 5,
- 5*time.Second,
- )
- if err != nil {
- ts.cfg.Logger.Warn("'daemonset list' failed", zap.Error(err))
- }
- for _, daemonset := range daemonsetlist {
- switch daemonset.ObjectMeta.Name {
- case "aws-node":
- ts.cfg.Logger.Info("Using CNI:", zap.String("CNI", daemonset.ObjectMeta.Name))
-
- case "cillium":
- ts.cfg.Logger.Info("Using CNI:", zap.String("CNI", daemonset.ObjectMeta.Name))
- }
- }
- return nil
-}
-
-func (ts *tester) testPodtoPod() error {
- //Create Server Pod
- ts.cfg.Logger.Info("Creating ServerPod:", zap.String("ServerPod", ServerPod))
- serverPod := client.NewBusyBoxPod(ServerPod, "sleep 120")
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- serverPod, err := ts.cfg.Client.KubernetesClient().CoreV1().Pods(ts.cfg.Namespace).Create(ctx, serverPod, meta_v1.CreateOptions{})
- if err != nil {
- return fmt.Errorf("failed to create CNI server pod (%v)", err)
- }
- cancel()
- ts.cfg.Logger.Info("Checking for ServerPod completed", zap.String("ServerPod", ServerPod))
- err = client.WaitForPodRunningInNamespace(ts.cfg.Client.KubernetesClient(), serverPod)
- if err != nil {
- return fmt.Errorf("failed to wait for CNI server pod to become healthy (%v)", err)
- }
- serverPod, err = ts.cfg.Client.KubernetesClient().CoreV1().Pods(ts.cfg.Namespace).Get(context.TODO(), ServerPod, meta_v1.GetOptions{})
- //Create Ping Pod
- ts.cfg.Logger.Info("Creating PingPod:", zap.String("PingPod", PingPod))
- pingPod := client.NewBusyBoxPod(PingPod, "ping -c 3 -w 2 -w 30 "+serverPod.Status.PodIP)
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- pingPod, err = ts.cfg.Client.KubernetesClient().CoreV1().Pods(ts.cfg.Namespace).Create(ctx, pingPod, meta_v1.CreateOptions{})
- cancel()
- ts.cfg.Logger.Info("Checking for PingPod completed", zap.String("ServerPod", ServerPod))
- err = client.WaitForPodRunningInNamespace(ts.cfg.Client.KubernetesClient(), pingPod)
- if err != nil {
- return fmt.Errorf("failed to wait for CNI Ping pod to become healthy (%v)", err)
- }
- //Check for Ping Pod Success
- err = client.WaitForPodSuccessInNamespaceTimeout(ts.cfg.Logger, ts.cfg.Client.KubernetesClient(), pingPod.Name, ts.cfg.Namespace, PodTimeout)
- if err != nil {
- return fmt.Errorf("failed to wait for CNI Ping pod to become healthy (%v)", err)
- }
- ts.cfg.Logger.Info("Pod to Pod communication SUCCESS")
- return nil
-}
-
-func (ts *tester) deletePodtoPod() error {
- //Delete Server Pod
- ts.cfg.Logger.Info("Deleting ServerPod", zap.String("ServerPod", ServerPod))
- foreground := meta_v1.DeletePropagationForeground
- err := ts.cfg.Client.KubernetesClient().CoreV1().Pods(ts.cfg.Namespace).Delete(context.TODO(), ServerPod,
- meta_v1.DeleteOptions{
- GracePeriodSeconds: int64Ref(0),
- PropagationPolicy: &foreground,
- },
- )
- if err != nil {
- return fmt.Errorf("failed to Delete CNI Server pod (%v)", err)
- }
- return nil
-}
-
-func (ts *tester) testPodtoNode() error {
- //Find random schedule-able node and it's IP
- node, err := client.GetRandomReadySchedulableNode(ts.cfg.Client.KubernetesClient())
- if err != nil {
- return fmt.Errorf("failed getting random ready schedulable node (%v)", err)
- }
- internalIP, err := client.GetInternalIP(node)
- ts.cfg.Logger.Info("Found Random schedulabe Node:", zap.String("Node IP", internalIP))
- if err != nil {
- return fmt.Errorf("failed IP of schedulable node (%v)", err)
- }
- //Create Node Pod
- ts.cfg.Logger.Info("Creating NodePod:", zap.String("NodePod", NodePod))
- nodePod := client.NewBusyBoxPod(NodePod, "ping -c 3 -w 2 -w 30 "+internalIP)
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- nodePod, err = ts.cfg.Client.KubernetesClient().CoreV1().Pods(ts.cfg.Namespace).Create(ctx, nodePod, meta_v1.CreateOptions{})
- cancel()
- ts.cfg.Logger.Info("Checking for NodePod completed", zap.String("NodePod", NodePod))
- err = client.WaitForPodRunningInNamespace(ts.cfg.Client.KubernetesClient(), nodePod)
- if err != nil {
- return fmt.Errorf("failed to wait for CNI Node pod to become healthy (%v)", err)
- }
- //Check for Node Pod Success
- ts.cfg.Logger.Info("Checking for NodePod success")
- err = client.WaitForPodSuccessInNamespaceTimeout(ts.cfg.Logger, ts.cfg.Client.KubernetesClient(), nodePod.Name, ts.cfg.Namespace, PodTimeout)
- if err != nil {
- return fmt.Errorf("failed to wait for CNI Node pod to become healthy (%v)", err)
- }
- ts.cfg.Logger.Info("Pod to Node communication SUCCESS")
- return nil
-}
-
-func (ts *tester) deletePodtoNode() error {
- //Delete Server Pod
- foreground := meta_v1.DeletePropagationForeground
- ts.cfg.Logger.Info("Deleting NodePod", zap.String("NodePod", NodePod))
- err := ts.cfg.Client.KubernetesClient().CoreV1().Pods(ts.cfg.Namespace).Delete(context.TODO(), NodePod,
- meta_v1.DeleteOptions{
- GracePeriodSeconds: int64Ref(0),
- PropagationPolicy: &foreground,
- },
- )
- if err != nil {
- return fmt.Errorf("failed to Delete CNI Node pod (%v)", err)
- }
- return nil
-}
-
-func int64Ref(v int64) *int64 {
- return &v
-}
diff --git a/k8s-tester/cni/vend.sh b/k8s-tester/cni/vend.sh
deleted file mode 100755
index 7937e2980..000000000
--- a/k8s-tester/cni/vend.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-< 0 {
- if nodes, err := client.ListNodes(ts.cfg.Client.KubernetesClient()); len(nodes) < ts.cfg.MinimumNodes || err != nil {
- return fmt.Errorf("failed to validate minimum nodes requirement %d (nodes %v, error %v)", ts.cfg.MinimumNodes, len(nodes), err)
- }
- }
-
- if err := client.CreateNamespace(ts.cfg.Logger, ts.cfg.Client.KubernetesClient(), ts.cfg.Namespace); err != nil {
- return err
- }
-
- latencies := ts.startWrites()
- if len(latencies) == 0 {
- ts.cfg.Logger.Warn("no latency collected")
- return nil
- }
-
- ts.cfg.Logger.Info("sorting write latency results", zap.Int("total-data-points", latencies.Len()))
- now := time.Now()
- sort.Sort(latencies)
- ts.cfg.Logger.Info("sorted write latency results", zap.Int("total-data-points", latencies.Len()), zap.String("took", time.Since(now).String()))
- ts.cfg.LatencySummary.TestID = time.Now().UTC().Format(time.RFC3339Nano)
- ts.cfg.LatencySummary.P50 = latencies.PickP50()
- ts.cfg.LatencySummary.P90 = latencies.PickP90()
- ts.cfg.LatencySummary.P99 = latencies.PickP99()
- ts.cfg.LatencySummary.P999 = latencies.PickP999()
- ts.cfg.LatencySummary.P9999 = latencies.PickP9999()
-
- // https://pkg.go.dev/github.com/prometheus/client_golang/prometheus?tab=doc#Gatherer
- mfs, err := prometheus.DefaultGatherer.Gather()
- if err != nil {
- ts.cfg.Logger.Warn("failed to gather prometheus metrics", zap.Error(err))
- return err
- }
- for _, mf := range mfs {
- if mf == nil {
- continue
- }
- switch *mf.Name {
- case "configmaps_client_write_requests_success_total":
- gg := mf.Metric[0].GetGauge()
- ts.cfg.LatencySummary.SuccessTotal = gg.GetValue()
- case "configmaps_client_write_requests_failure_total":
- gg := mf.Metric[0].GetGauge()
- ts.cfg.LatencySummary.FailureTotal = gg.GetValue()
- case "configmaps_client_write_request_latency_milliseconds":
- ts.cfg.LatencySummary.Histogram, err = latency.ParseHistogram("milliseconds", mf.Metric[0].GetHistogram())
- if err != nil {
- return err
- }
- }
- }
-
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nLatencySummary:\n%s\n", ts.cfg.LatencySummary.Table())
- return nil
-}
-
-func (ts *tester) Delete() error {
- if ok := ts.runPrompt("delete"); !ok {
- return errors.New("cancelled")
- }
-
- ts.donecCloseOnce.Do(func() {
- close(ts.donec)
- })
-
- var errs []string
-
- if err := client.DeleteNamespaceAndWait(
- ts.cfg.Logger,
- ts.cfg.Client.KubernetesClient(),
- ts.cfg.Namespace,
- client.DefaultNamespaceDeletionInterval,
- client.DefaultNamespaceDeletionTimeout,
- client.WithForceDelete(true),
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete namespace (%v)", err))
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- return nil
-}
-
-func (ts *tester) runPrompt(action string) (ok bool) {
- if ts.cfg.Prompt {
- msg := fmt.Sprintf("Ready to %q resources for the namespace %q, should we continue?", action, ts.cfg.Namespace)
- prompt := promptui.Select{
- Label: msg,
- Items: []string{
- "No, cancel it!",
- fmt.Sprintf("Yes, let's %q!", action),
- },
- }
- idx, answer, err := prompt.Run()
- if err != nil {
- panic(err)
- }
- if idx != 1 {
- fmt.Printf("cancelled %q [index %d, answer %q]\n", action, idx, answer)
- return false
- }
- }
- return true
-}
-
-func (ts *tester) startWrites() (latencies latency.Durations) {
- ts.cfg.Logger.Info("writing", zap.Int("objects", ts.cfg.Objects), zap.Int("object-size", ts.cfg.Objects))
- latencies = make(latency.Durations, 0, 20000)
-
- val := rand.String(ts.cfg.ObjectSize)
- for i := 0; i < ts.cfg.Objects; i++ {
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("writes stopped")
- return
- case <-ts.donec:
- ts.cfg.Logger.Info("writes done")
- return
- default:
- }
-
- key := fmt.Sprintf("configmap%d%s", i, rand.String(7))
-
- start := time.Now()
- ctx, cancel := context.WithTimeout(context.Background(), ts.cfg.Client.Config().ClientTimeout)
- _, err := ts.cfg.Client.KubernetesClient().
- CoreV1().
- ConfigMaps(ts.cfg.Namespace).
- Create(ctx, &core_v1.ConfigMap{
- TypeMeta: meta_v1.TypeMeta{
- APIVersion: "v1",
- Kind: "ConfigMap",
- },
- ObjectMeta: meta_v1.ObjectMeta{
- Name: key,
- Namespace: ts.cfg.Namespace,
- Labels: map[string]string{
- "name": key,
- },
- },
- Data: map[string]string{key: val},
- }, meta_v1.CreateOptions{})
- cancel()
- took := time.Since(start)
- tookMS := float64(took / time.Millisecond)
- writeRequestLatencyMs.Observe(tookMS)
- latencies = append(latencies, took)
- if err != nil {
- writeRequestsFailureTotal.Inc()
- ts.cfg.Logger.Warn("write configmap failed", zap.String("namespace", ts.cfg.Namespace), zap.Error(err))
- } else {
- writeRequestsSuccessTotal.Inc()
- if i%20 == 0 {
- ts.cfg.Logger.Info("wrote configmap", zap.Int("iteration", i), zap.String("namespace", ts.cfg.Namespace))
- }
- }
- }
- return latencies
-}
diff --git a/k8s-tester/configmaps/vend.sh b/k8s-tester/configmaps/vend.sh
deleted file mode 100755
index 7937e2980..000000000
--- a/k8s-tester/configmaps/vend.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-< 0 {
- if nodes, err := client.ListNodes(ts.cfg.Client.KubernetesClient()); len(nodes) < ts.cfg.MinimumNodes || err != nil {
- return fmt.Errorf("failed to validate minimum nodes requirement %d (nodes %v, error %v)", ts.cfg.MinimumNodes, len(nodes), err)
- }
- }
-
- if err := client.CreateNamespace(ts.cfg.Logger, ts.cfg.Client.KubernetesClient(), ts.cfg.Namespace); err != nil {
- return err
- }
-
- if err := installSonobuoy(ts.cfg.Logger, ts.cfg.SonobuoyPath, ts.cfg.SonobuoyDownloadURL); err != nil {
- return err
- }
- if err := ts.deleteSonobuoy(); err != nil {
- return err
- }
- if err := ts.runSonobuoy(); err != nil {
- return err
- }
- if err := ts.checkSonobuoy(); err != nil {
- return err
- }
- if err := ts.checkResults(); err != nil {
- return err
- }
-
- return nil
-}
-
-func (ts *tester) Delete() error {
- if ok := ts.runPrompt("delete"); !ok {
- return errors.New("cancelled")
- }
-
- var errs []string
-
- if err := installSonobuoy(ts.cfg.Logger, ts.cfg.SonobuoyPath, ts.cfg.SonobuoyDownloadURL); err != nil {
- return err
- }
- if err := ts.deleteSonobuoy(); err != nil {
- return err
- }
-
- if err := client.DeleteNamespaceAndWait(
- ts.cfg.Logger,
- ts.cfg.Client.KubernetesClient(),
- ts.cfg.Namespace,
- client.DefaultNamespaceDeletionInterval,
- client.DefaultNamespaceDeletionTimeout,
- client.WithForceDelete(true),
- ); err != nil {
- errs = append(errs, fmt.Sprintf("failed to delete namespace (%v)", err))
- }
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- return nil
-}
-
-func (ts *tester) runPrompt(action string) (ok bool) {
- if ts.cfg.Prompt {
- msg := fmt.Sprintf("Ready to %q resources for the namespace %q, should we continue?", action, ts.cfg.Namespace)
- prompt := promptui.Select{
- Label: msg,
- Items: []string{
- "No, cancel it!",
- fmt.Sprintf("Yes, let's %q!", action),
- },
- }
- idx, answer, err := prompt.Run()
- if err != nil {
- panic(err)
- }
- if idx != 1 {
- fmt.Printf("cancelled %q [index %d, answer %q]\n", action, idx, answer)
- return false
- }
- }
- return true
-}
-
-func (ts *tester) deleteSonobuoy() (err error) {
- args := []string{
- ts.cfg.SonobuoyPath,
- "--logtostderr",
- "--alsologtostderr",
- "--v=3",
- "delete",
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "--wait",
- }
- cmd := strings.Join(args, " ")
-
- ts.cfg.Logger.Info("deleting sonobuoy",
- zap.String("command", cmd),
- zap.String("timeout", ts.cfg.SonobuoyDeleteTimeoutString),
- )
-
- var output []byte
- ctx, cancel := context.WithTimeout(context.Background(), ts.cfg.SonobuoyDeleteTimeout)
- output, err = exec.New().CommandContext(ctx, args[0], args[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- if err != nil {
- // TODO: check error
- ts.cfg.Logger.Warn("failed to delete sonobuoy", zap.String("command", cmd), zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n'%s' output:\n\n%s\n\n", cmd, out)
-
- ts.cfg.Logger.Info("deleted sonobuoy", zap.String("command", cmd))
- return nil
-}
-
-func (ts *tester) runSonobuoy() (err error) {
- timeoutSeconds := int64(ts.cfg.SonobuoyRunTimeout.Seconds())
- args := []string{
- ts.cfg.SonobuoyPath,
- "--logtostderr",
- "--alsologtostderr",
- "--v=3",
- "run",
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "--mode=" + ts.cfg.SonobuoyRunMode,
- "--kube-conformance-image=" + ts.cfg.SonobuoyRunKubeConformanceImage,
- "--show-default-podspec=true",
- fmt.Sprintf("--timeout=%d", timeoutSeconds), // default "10800", 3-hour
- }
- if ts.cfg.SonobuoyRunE2ERepoConfig != "" {
- args = append(args, "--e2e-repo-config="+ts.cfg.SonobuoyRunE2ERepoConfig)
- }
- if ts.cfg.SonobuoyRunImage != "" {
- args = append(args, "--sonobuoy-image="+ts.cfg.SonobuoyRunImage)
- }
- if ts.cfg.SonobuoyRunSystemdLogsImage != "" {
- args = append(args, "--systemd-logs-image="+ts.cfg.SonobuoyRunSystemdLogsImage)
- }
- if ts.cfg.SonobuoyRunE2EFocus != "" {
- args = append(args, "--e2e-focus="+ts.cfg.SonobuoyRunE2EFocus)
- }
- if ts.cfg.SonobuoyRunE2ESkip != "" {
- args = append(args, "--e2e-skip="+ts.cfg.SonobuoyRunE2ESkip)
- }
- cmd := strings.Join(args, " ")
-
- ts.cfg.Logger.Info("running sonobuoy",
- zap.Duration("timeout", ts.cfg.SonobuoyRunTimeout),
- zap.Int64("timeout-seconds", timeoutSeconds),
- zap.String("mode", ts.cfg.SonobuoyRunMode),
- zap.String("command", cmd),
- )
-
- var output []byte
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute) // do not wait, so just set timeout for launching tests
- output, err = exec.New().CommandContext(ctx, args[0], args[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- if err != nil {
- // TODO: check error
- ts.cfg.Logger.Warn("failed to run sonobuoy", zap.String("command", cmd), zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n'%s' output:\n\n%s\n\n", cmd, out)
-
- ts.cfg.Logger.Info("ran sonobuoy", zap.String("mode", ts.cfg.SonobuoyRunMode), zap.String("command", cmd))
- return nil
-}
-
-func (ts *tester) checkSonobuoy() (err error) {
- ts.cfg.Logger.Info("checking pod/sonobuoy-e2e-job")
- sonobuoyE2EJobPod := ""
- retryStart := time.Now()
- for time.Since(retryStart) < 10*time.Minute {
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("sonobuoy check stopped")
- return nil
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("sonobuoy check timeout")
- return fmt.Errorf("sonobuoy run took too long (exceeded %v)", ts.cfg.SonobuoyRunTimeout)
- case <-time.After(10 * time.Second):
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- var pods *v1.PodList
- pods, err = ts.cfg.Client.KubernetesClient().
- CoreV1().
- Pods(ts.cfg.Namespace).
- List(ctx, meta_v1.ListOptions{})
- cancel()
- if err != nil {
- ts.cfg.Logger.Warn("failed to list pods", zap.Error(err))
- continue
- }
-
- for _, pv := range pods.Items {
- ts.cfg.Logger.Info("found pod", zap.String("name", pv.GetName()))
- if strings.HasPrefix(pv.GetName(), "sonobuoy-e2e-job-") {
- sonobuoyE2EJobPod = pv.GetName()
- break
- }
- }
- if sonobuoyE2EJobPod != "" {
- break
- }
- }
- if sonobuoyE2EJobPod == "" {
- return fmt.Errorf("failed to find pod/sonobuoy-e2e-job in %q", ts.cfg.Namespace)
- }
- ts.cfg.Logger.Info("found pod/sonobuoy-e2e-job", zap.String("name", sonobuoyE2EJobPod))
-
- argsLogsSonobuoy := []string{
- ts.cfg.SonobuoyPath,
- "--logtostderr",
- "--alsologtostderr",
- "--v=3",
- "logs",
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- }
- cmdLogsSonobuoy := strings.Join(argsLogsSonobuoy, " ")
-
- argsLogsPod := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "logs",
- fmt.Sprintf("pod/%s", sonobuoyE2EJobPod),
- "e2e",
- "--tail=30",
- }
- cmdLogsPod := strings.Join(argsLogsPod, " ")
-
- argsStatus := []string{
- ts.cfg.SonobuoyPath,
- "--logtostderr",
- "--alsologtostderr",
- "--v=3",
- "status",
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "--show-all",
- }
- cmdStatus := strings.Join(argsStatus, " ")
-
- ts.cfg.Logger.Info("running sonobuoy",
- zap.String("logs-command-sonobuoy", cmdLogsSonobuoy),
- zap.String("logs-command-pod", cmdLogsPod),
- zap.String("status-command", cmdStatus),
- )
-
- deadline := time.Now().Add(ts.cfg.SonobuoyRunTimeout)
- donec := time.After(ts.cfg.SonobuoyRunTimeout)
- start, waitDur := time.Now(), ts.cfg.SonobuoyRunTimeout
-
- interval := 15 * time.Minute
- cnt := 0
- for time.Since(start) < waitDur {
- cnt++
- ts.cfg.Logger.Info(
- "waiting for sonobuoy run",
- zap.Duration("interval", interval),
- zap.String("time-left", deadline.Sub(time.Now()).String()),
- zap.Duration("timeout", ts.cfg.SonobuoyRunTimeout),
- )
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("sonobuoy check stopped")
- return nil
- case <-donec:
- ts.cfg.Logger.Warn("sonobuoy check timeout")
- return fmt.Errorf("sonobuoy run took too long (exceeded %v)", ts.cfg.SonobuoyRunTimeout)
- case <-time.After(interval):
- }
-
- argsLogs, cmdLogs := argsLogsSonobuoy, cmdLogsSonobuoy
- if cnt%2 == 0 {
- argsLogs, cmdLogs = argsLogsPod, cmdLogsPod
- }
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- output, err := exec.New().CommandContext(ctx, argsLogs[0], argsLogs[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("failed to fetch sonobuoy logs", zap.String("command", cmdLogs), zap.Error(err))
- }
- lines := strings.Split(out, "\n")
- linesN := len(lines)
- if linesN > 30 { // tail 30 lines
- out = strings.Join(lines[linesN-30:], "\n")
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n'%s' output (total lines %d, last 30 lines):\n\n%s\n\n", cmdLogs, linesN, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
- output, err = exec.New().CommandContext(ctx, argsStatus[0], argsStatus[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("failed to run sonobuoy status", zap.String("command", cmdStatus), zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n'%s' output:\n\n%s\n\n", cmdStatus, out)
-
- // ref. https://github.com/vmware-tanzu/sonobuoy/blob/master/cmd/sonobuoy/app/status.go
- if strings.Contains(out, "Sonobuoy has completed. ") ||
- strings.Contains(out, "Sonobuoy plugins have completed. ") {
- break
- }
- if strings.Contains(out, "Sonobuoy has failed. ") ||
- strings.Contains(out, "Sonobuoy is in unknown state") {
- return errors.New("sonobuoy run failed")
- }
-
- interval = time.Duration(float64(interval) * 0.7)
- if interval < 2*time.Minute {
- interval = 2 * time.Minute
- }
- }
-
- return nil
-}
-
-func (ts *tester) checkResults() (err error) {
- argsRetrieve := []string{
- ts.cfg.SonobuoyPath,
- "retrieve",
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- os.TempDir(),
- }
- cmdRetrieve := strings.Join(argsRetrieve, " ")
-
- ts.cfg.Logger.Info("running sonobuoy", zap.String("retrieve-command", cmdRetrieve))
-
- os.RemoveAll(ts.cfg.SonobuoyResultsTarGzPath)
- start, waitDur := time.Now(), 3*time.Minute
- for time.Since(start) < waitDur {
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("sonobuoy retrieve stopped")
- return nil
- default:
- }
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- output, err := exec.New().CommandContext(ctx, argsRetrieve[0], argsRetrieve[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("failed to run sonobuoy retrieve", zap.String("command", cmdRetrieve), zap.Error(err))
- time.Sleep(10 * time.Second)
- continue
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n'%s' output:\n\n%s\n\n", cmdRetrieve, out)
-
- if err = file.Copy(out, ts.cfg.SonobuoyResultsTarGzPath); err != nil {
- ts.cfg.Logger.Warn("failed to copy sonobuoy retrieve results", zap.Error(err))
- return err
- }
-
- ts.cfg.Logger.Info("retrieved sonobuoy results", zap.String("path", ts.cfg.SonobuoyResultsTarGzPath))
- break
- }
-
- err = readResults(
- ts.cfg.Logger,
- ts.cfg.LogWriter,
- ts.cfg.SonobuoyPath,
- ts.cfg.SonobuoyResultsTarGzPath,
- )
- if err != nil {
- ts.cfg.Logger.Warn("read results failed", zap.Error(err))
- }
-
- logPath, xmlPath, terr := untarResults(
- ts.cfg.Logger,
- ts.cfg.SonobuoyResultsTarGzPath,
- ts.cfg.SonobuoyResultsOutputDir,
- )
- if terr != nil {
- ts.cfg.Logger.Warn("failed to untar results", zap.Error(terr))
- if err == nil {
- err = terr
- } else {
- err = fmt.Errorf("read results error [%v], untar error [%v]", err, terr)
- }
- }
- if err != nil {
- return err
- }
- if err = file.Copy(logPath, ts.cfg.SonobuoyResultsE2ELogPath); err != nil {
- return err
- }
- if err = file.Copy(xmlPath, ts.cfg.SonobuoyResultsJunitXMLPath); err != nil {
- return err
- }
-
- return nil
-}
diff --git a/k8s-tester/conformance/vend.sh b/k8s-tester/conformance/vend.sh
deleted file mode 100755
index 7937e2980..000000000
--- a/k8s-tester/conformance/vend.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-< 0 {
- return errors.New(strings.Join(errs, ", "))
- }
- return nil
-}
-
-func (ts *tester) runPrompt(action string) (ok bool) {
- if ts.cfg.Prompt {
- msg := fmt.Sprintf("Ready to %q resources for the namespace %q, should we continue?", action, ts.cfg.Namespace)
- prompt := promptui.Select{
- Label: msg,
- Items: []string{
- "No, cancel it!",
- fmt.Sprintf("Yes, let's %q!", action),
- },
- }
- idx, answer, err := prompt.Run()
- if err != nil {
- panic(err)
- }
- if idx != 1 {
- fmt.Printf("cancelled %q [index %d, answer %q]\n", action, idx, answer)
- return false
- }
- }
- return true
-}
-
-func (ts *tester) installEBSHelmChart() error {
- getAllArgs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "get",
- "all",
- }
- getAllCmd := strings.Join(getAllArgs, " ")
-
- descArgsDs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "describe",
- "daemonset.apps/ebs-csi-node",
- }
- descCmdDs := strings.Join(descArgsDs, " ")
-
- descArgsPods := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "describe",
- "pods",
- "--selector=app=ebs-csi-controller",
- }
- descCmdPods := strings.Join(descArgsPods, " ")
-
- logArgs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "logs",
- "--selector=app=ebs-csi-controller",
- "--all-containers=true",
- "--timestamps",
- }
- logsCmd := strings.Join(logArgs, " ")
-
- return helm.Install(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Stopc: ts.cfg.Stopc,
- Timeout: 10 * time.Minute,
- KubeconfigPath: ts.cfg.Client.Config().KubeconfigPath,
- Namespace: ts.cfg.Namespace,
- ChartRepoURL: ts.cfg.HelmChartRepoURL,
- ChartName: chartName,
- ReleaseName: chartName,
- Values: values,
- LogFunc: func(format string, v ...interface{}) {
- ts.cfg.Logger.Info(fmt.Sprintf("[install] "+format, v...))
- },
- QueryFunc: func() {
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, getAllArgs[0], getAllArgs[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl get all' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getAllCmd, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, descArgsDs[0], descArgsDs[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe daemonset' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmdDs, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, descArgsPods[0], descArgsPods[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe pods' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmdPods, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, logArgs[0], logArgs[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl logs' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", logsCmd, out)
- },
- QueryInterval: 30 * time.Second,
- })
-}
-
-func (ts *tester) deleteEBSHelmChart() error {
- ts.cfg.Logger.Info("deleting %s: %s", zap.String("helm-chart-name", chartName))
- err := helm.Uninstall(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Timeout: 3 * time.Minute,
- KubeconfigPath: ts.cfg.Client.Config().KubeconfigPath,
- Namespace: "kube-system",
- ChartName: chartName,
- ReleaseName: chartName,
- })
- if err == nil {
- ts.cfg.Logger.Info("deleted helm chart", zap.String("namespace", ts.cfg.Namespace), zap.String("name", chartName))
- return nil
- }
- // requires "k8s_errors.IsNotFound"
- // ref. https://github.com/aws/aws-k8s-tester/issues/79
- if k8s_errors.IsNotFound(err) || k8s_errors.IsGone(err) {
- ts.cfg.Logger.Info("helm chart already deleted", zap.String("namespace", ts.cfg.Namespace), zap.String("name", chartName), zap.Error(err))
- return nil
- }
- ts.cfg.Logger.Warn("failed to delete helm chart", zap.String("namespace", ts.cfg.Namespace), zap.String("name", chartName), zap.Error(err))
- return err
-}
-
-func (ts *tester) createEBSStorageClass() (err error) {
- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
- firstConsumerBinding := storage_v1.VolumeBindingWaitForFirstConsumer
- allowVolumeExpansion := true
- _, err = ts.cfg.Client.KubernetesClient().StorageV1().StorageClasses().Create(
- ctx,
- &storage_v1.StorageClass{
- TypeMeta: meta_v1.TypeMeta{
- APIVersion: "storage.k8s.io/v1",
- Kind: "StorageClass",
- },
- ObjectMeta: meta_v1.ObjectMeta{
- Name: storageClassName,
- },
- Provisioner: provisioner,
- AllowVolumeExpansion: &allowVolumeExpansion,
- VolumeBindingMode: &firstConsumerBinding,
- Parameters: map[string]string{
- "type": "gp2",
- },
- },
- meta_v1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create StorageClasses (%v)", err)
- }
- ts.cfg.Logger.Info("created a StorageClasses for EBS")
- return nil
-}
-
-func (ts *tester) deleteEBSStorageClass() (err error) {
- ts.cfg.Logger.Info("deleting storageClass for EBS")
- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
- err = ts.cfg.Client.KubernetesClient().StorageV1().StorageClasses().Delete(
- ctx,
- storageClassName,
- meta_v1.DeleteOptions{
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !k8s_errors.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return err
- }
- ts.cfg.Logger.Info("delete StorageClasses for EBS")
- return nil
-}
-
-func (ts *tester) createPersistentVolumeClaim(storageClass string) error {
- ts.cfg.Logger.Info("creating PersistentVolumeClaim for EBS, Provisioning test")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.Client.KubernetesClient().CoreV1().PersistentVolumeClaims(ts.cfg.Namespace).Create(
- ctx,
- &core_v1.PersistentVolumeClaim{
- TypeMeta: meta_v1.TypeMeta{
- APIVersion: "v1",
- Kind: "PersistentVolumeClaim",
- },
- ObjectMeta: meta_v1.ObjectMeta{
- Name: pvcProvisionName,
- },
- Spec: core_v1.PersistentVolumeClaimSpec{
- AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
- StorageClassName: &storageClass,
- Resources: core_v1.VolumeResourceRequirements{
- Requests: core_v1.ResourceList{
- core_v1.ResourceStorage: api_resource.MustParse("4Gi"),
- },
- },
- },
- },
- meta_v1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create PersistentVolumeClaims (%v)", err)
- }
- ts.cfg.Logger.Info("created a PersistentVolumeClaims for EBS")
- return nil
-}
-
-var foreground = meta_v1.DeletePropagationForeground
-
-func (ts *tester) deletePersistentVolumeClaim() error {
- ts.cfg.Logger.Info("deleting PersistentVolumeClaim for EBS, Provisioning test")
- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
- err := ts.cfg.Client.KubernetesClient().CoreV1().PersistentVolumeClaims(ts.cfg.Namespace).Delete(
- ctx,
- pvcProvisionName,
- meta_v1.DeleteOptions{
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to delete PersistentVolumeClaim (%v)", err)
- }
- ts.cfg.Logger.Info("Deleted a PersistentVolumeClaim for EBS")
- return nil
-}
-
-// dynamically provision a volume from the PVC without pod mount/startup failure
-func (ts *tester) provisionPVC() error {
- var gracePeriod int64 = 1
- ts.cfg.Logger.Info("creating Pod to test volume provisioning", zap.String("Pod", provisionPodName))
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.Client.KubernetesClient().
- CoreV1().
- Pods(ts.cfg.Namespace).
- Create(
- ctx,
- &core_v1.Pod{
- TypeMeta: meta_v1.TypeMeta{
- Kind: "Pod",
- APIVersion: "v1",
- },
- ObjectMeta: meta_v1.ObjectMeta{
- Name: provisionPodName,
- },
- Spec: core_v1.PodSpec{
- Containers: []v1.Container{
- {
- Name: provisionPodName,
- Image: "public.ecr.aws/hudsonbay/busybox:latest",
- WorkingDir: "/opt",
- // An imperative and easily debuggable container which reads/writes vol contents for
- // us to scan in the tests or by eye.
- // We expect that /opt is empty in the minimal containers which we use in this test.
- Command: []string{"/bin/sh", "-c", "while true ; do sleep 2; done "},
- VolumeMounts: []core_v1.VolumeMount{
- {
- Name: provisionVolumeName,
- MountPath: "/opt/1",
- },
- },
- },
- },
- TerminationGracePeriodSeconds: &gracePeriod,
- Volumes: []core_v1.Volume{
- {
- Name: provisionVolumeName,
- VolumeSource: core_v1.VolumeSource{
- PersistentVolumeClaim: &core_v1.PersistentVolumeClaimVolumeSource{
- ClaimName: pvcProvisionName,
- },
- },
- },
- },
- },
- },
- meta_v1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create VolumeProvision Pod for provisionPVC test: (%v)", err)
- }
-
- // wait for Pod to spawn
- time.Sleep(20 * time.Second)
-
- ts.cfg.Logger.Info("retrieving Dynamic Provisioed Claim on Pod", zap.String("claim", pvcProvisionName))
- ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
- claim, err := ts.cfg.Client.KubernetesClient().
- CoreV1().
- PersistentVolumeClaims(ts.cfg.Namespace).
- Get(
- ctx,
- pvcProvisionName,
- meta_v1.GetOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to GET PersistentVolumeClaims pvcProvisionName (%v)", err)
- }
-
- pv, err := ts.getBoundPV(claim)
- ts.cfg.Logger.Info("got PV from Pod", zap.String("pv", pv.ObjectMeta.Name))
- if err != nil {
- return fmt.Errorf("failed to getBoundPV (%v)", err)
- }
-
- expectedCapacity := resource.MustParse("4Gi")
- pvCapacity := pv.Spec.Capacity[core_v1.ResourceName(core_v1.ResourceStorage)]
- ts.cfg.Logger.Info("checking Desired Capacity vs actual PV Capacity")
- if expectedCapacity.Value() != pvCapacity.Value() {
- return fmt.Errorf("capacity did not equal volume Capacity (%v)", err)
- }
-
- ts.cfg.Logger.Info("[PASSED] expectedCapacity did equal volume Capacity", zap.String(expectedCapacity.String(), pvCapacity.String()))
- return nil
-}
-
-//It should handle resizing on running, and stopped pods
-func (ts *tester) resizePVC() error {
- // resize testing
- ts.cfg.Logger.Info("starting PVC Resizing Tests")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- pvc, err := ts.cfg.Client.KubernetesClient().
- CoreV1().
- PersistentVolumeClaims(ts.cfg.Namespace).
- Get(
- ctx,
- pvcProvisionName,
- meta_v1.GetOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to GET PersistentVolumeClaims pvcProvisionName (%v)", err)
- }
- ts.cfg.Logger.Info("found PVC for resizing tests", zap.String("pvc", pvcProvisionName))
-
- // make Deepcopy of PVC with new size, and apply to current PVC
- ts.cfg.Logger.Info("chaning PVC Size of running pod from 4GI to", zap.String("Size", "6Gi"))
- newSize := resource.MustParse("6Gi")
- newPVC, err := ts.expandPVCSize(pvc, newSize)
- if newPVC == nil {
- return fmt.Errorf("failed to create Resize of PVC (%v)", err)
- }
-
- // check if PVC is being updated
- pvcSize := newPVC.Spec.Resources.Requests[v1.ResourceStorage]
- if pvcSize.Cmp(newSize) != 0 {
- return fmt.Errorf("error updating pvc size %v", err)
- }
-
- // wait for PVC to come back healthy
- ts.cfg.Logger.Info("waiting on PVC ReSize for max timeout of 8 minutes...")
- err = ts.waitForControllerVolumeResize(newPVC, 8*time.Minute)
- if err != nil {
- return fmt.Errorf("VolumeResize resize timeout occured due to error (%v)", err)
- }
-
- ts.cfg.Logger.Info("[PASSED] PVC ReSize on running Pod", zap.String("New Size", "6Gi"))
- return nil
-}
-
-// cleanup testing Pods
-func (ts *tester) deletevolumePods() error {
- ts.cfg.Logger.Info("deleting Pods for EBS CSI tests")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- err := ts.cfg.Client.KubernetesClient().CoreV1().Pods(ts.cfg.Namespace).Delete(
- ctx,
- provisionPodName,
- meta_v1.DeleteOptions{
- GracePeriodSeconds: &graceperiod,
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to delete Pod (%v)", err)
- }
- ts.cfg.Logger.Info("deleted a Pod for EBS tests")
- return nil
-}
-
-// getBoundPV returns a PV details.
-func (ts *tester) getBoundPV(pvc *v1.PersistentVolumeClaim) (*v1.PersistentVolume, error) {
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- // Get new copy of the claim
- claim, err := ts.cfg.Client.KubernetesClient().CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(ctx, pvc.Name, meta_v1.GetOptions{})
- if err != nil {
- return nil, err
- }
- // Get the bound PV
- pv, err := ts.cfg.Client.KubernetesClient().CoreV1().PersistentVolumes().Get(ctx, claim.Spec.VolumeName, meta_v1.GetOptions{})
- cancel()
- return pv, err
-}
-
-// expandPVCSize expands PVC size
-func (ts *tester) expandPVCSize(origPVC *v1.PersistentVolumeClaim, size resource.Quantity) (*v1.PersistentVolumeClaim, error) {
- pvcName := origPVC.Name
- updatedPVC := origPVC.DeepCopy()
- var resizePollInterval = 2 * time.Second
- // Retry the update on error, until we hit a timeout.
- // TODO: Determine whether "retry with timeout" is appropriate here. Maybe we should only retry on version conflict.
- var lastUpdateError error
- waitErr := wait.PollImmediate(resizePollInterval, 30*time.Second, func() (bool, error) {
- var err error
- updatedPVC, err = ts.cfg.Client.KubernetesClient().CoreV1().PersistentVolumeClaims(origPVC.Namespace).Get(context.TODO(), pvcName, meta_v1.GetOptions{})
- if err != nil {
- return false, fmt.Errorf("error fetching pvc %q for resizing: %v", pvcName, err)
- }
- updatedPVC.Spec.Resources.Requests[v1.ResourceStorage] = size
- updatedPVC, err = ts.cfg.Client.KubernetesClient().CoreV1().PersistentVolumeClaims(origPVC.Namespace).Update(context.TODO(), updatedPVC, meta_v1.UpdateOptions{})
- if err != nil {
- return false, fmt.Errorf("error fetching pvc %q for resizing: %v", updatedPVC, err)
- }
- return true, nil
- })
- if waitErr == wait.ErrWaitTimeout {
- return nil, fmt.Errorf("timed out attempting to update PVC size. last update error: %v", lastUpdateError)
- }
- if waitErr != nil {
- return nil, fmt.Errorf("failed to expand PVC size (check logs for error): %v", waitErr)
- }
- return updatedPVC, nil
-}
-
-func (ts *tester) waitForControllerVolumeResize(pvc *v1.PersistentVolumeClaim, timeout time.Duration) error {
- pvName := pvc.Spec.VolumeName
- var resizePollInterval = 2 * time.Second
- waitErr := wait.PollImmediate(resizePollInterval, timeout, func() (bool, error) {
- pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage]
- pv, err := ts.cfg.Client.KubernetesClient().CoreV1().PersistentVolumes().Get(context.TODO(), pvName, meta_v1.GetOptions{})
- if err != nil {
- return false, fmt.Errorf("error fetching pv %q for resizing %v", pvName, err)
- }
-
- pvSize := pv.Spec.Capacity[v1.ResourceStorage]
-
- // If pv size is greater or equal to requested size that means controller resize is finished.
- if pvSize.Cmp(pvcSize) >= 0 {
- return true, nil
- }
- return false, nil
- })
- if waitErr != nil {
- return fmt.Errorf("error while waiting for controller resize to finish: %v", waitErr)
- }
- return nil
-}
diff --git a/k8s-tester/csi-ebs/vend.sh b/k8s-tester/csi-ebs/vend.sh
deleted file mode 100755
index 7937e2980..000000000
--- a/k8s-tester/csi-ebs/vend.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-< 0 {
- return errors.New(strings.Join(errs, ", "))
- }
- return nil
-}
-
-func (ts *tester) runPrompt(action string) (ok bool) {
- if ts.cfg.Prompt {
- msg := fmt.Sprintf("Ready to %q resources for the namespace %q, should we continue?", action, ts.cfg.Namespace)
- prompt := promptui.Select{
- Label: msg,
- Items: []string{
- "No, cancel it!",
- fmt.Sprintf("Yes, let's %q!", action),
- },
- }
- idx, answer, err := prompt.Run()
- if err != nil {
- panic(err)
- }
- if idx != 1 {
- fmt.Printf("cancelled %q [index %d, answer %q]\n", action, idx, answer)
- return false
- }
- }
- return true
-}
-
-func (ts *tester) installChart() error {
- getAllArgs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "get",
- "all",
- }
- getAllCmd := strings.Join(getAllArgs, " ")
-
- descArgsDs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "describe",
- "daemonset.apps/efs-csi-node",
- }
- descCmdDs := strings.Join(descArgsDs, " ")
-
- descArgsPods := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "describe",
- "pods",
- "--selector=app=efs-csi-controller",
- }
- descCmdPods := strings.Join(descArgsPods, " ")
-
- logArgs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "logs",
- "--selector=app=efs-csi-controller",
- "--all-containers=true",
- "--timestamps",
- }
- logsCmd := strings.Join(logArgs, " ")
-
- return helm.Install(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Stopc: ts.cfg.Stopc,
- Timeout: 10 * time.Minute,
- KubeconfigPath: ts.cfg.Client.Config().KubeconfigPath,
- Namespace: ts.cfg.Namespace,
- ChartRepoURL: ts.cfg.HelmChartRepoURL,
- ChartName: chartName,
- ReleaseName: chartName,
- Values: values,
- LogFunc: func(format string, v ...interface{}) {
- ts.cfg.Logger.Info(fmt.Sprintf("[install] "+format, v...))
- },
- QueryFunc: func() {
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, getAllArgs[0], getAllArgs[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl get all' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getAllCmd, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, descArgsDs[0], descArgsDs[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe daemonset' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmdDs, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, descArgsPods[0], descArgsPods[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe pods' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmdPods, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, logArgs[0], logArgs[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl logs' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", logsCmd, out)
- },
- QueryInterval: 30 * time.Second,
- })
-}
-
-func (ts *tester) deleteChart() error {
- ts.cfg.Logger.Info("deleting %s: %s", zap.String("helm-chart-name", chartName))
- err := helm.Uninstall(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Timeout: 3 * time.Minute,
- KubeconfigPath: ts.cfg.Client.Config().KubeconfigPath,
- Namespace: "kube-system",
- ChartName: chartName,
- ReleaseName: chartName,
- })
- if err == nil {
- ts.cfg.Logger.Info("deleted helm chart", zap.String("namespace", ts.cfg.Namespace), zap.String("name", chartName))
- return nil
- }
- // requires "k8s_errors.IsNotFound"
- // ref. https://github.com/aws/aws-k8s-tester/issues/79
- if k8s_errors.IsNotFound(err) || k8s_errors.IsGone(err) {
- ts.cfg.Logger.Info("helm chart already deleted", zap.String("namespace", ts.cfg.Namespace), zap.String("name", chartName), zap.Error(err))
- return nil
- }
- ts.cfg.Logger.Warn("failed to delete helm chart", zap.String("namespace", ts.cfg.Namespace), zap.String("name", chartName), zap.Error(err))
- return err
-}
-
-func (ts *tester) createSC() (err error) {
- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
- firstConsumerBinding := storage_v1.VolumeBindingWaitForFirstConsumer
- allowVolumeExpansion := true
- _, err = ts.cfg.Client.KubernetesClient().StorageV1().StorageClasses().Create(
- ctx,
- &storage_v1.StorageClass{
- TypeMeta: meta_v1.TypeMeta{
- APIVersion: "storage.k8s.io/v1",
- Kind: "StorageClass",
- },
- ObjectMeta: meta_v1.ObjectMeta{
- Name: storageClassName,
- },
- Provisioner: provisioner,
- AllowVolumeExpansion: &allowVolumeExpansion,
- VolumeBindingMode: &firstConsumerBinding,
- Parameters: map[string]string{
- "basePath": "/dynamic_provisioning",
- },
- },
- meta_v1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create StorageClasses (%v)", err)
- }
- ts.cfg.Logger.Info("created a StorageClasses for EFS")
- return nil
-}
-
-func (ts *tester) deleteSC() (err error) {
- ts.cfg.Logger.Info("deleting storageClass for EFS")
- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
- err = ts.cfg.Client.KubernetesClient().StorageV1().StorageClasses().Delete(
- ctx,
- storageClassName,
- meta_v1.DeleteOptions{
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil && !k8s_errors.IsNotFound(err) && !strings.Contains(err.Error(), "not found") {
- ts.cfg.Logger.Warn("failed to delete", zap.Error(err))
- return err
- }
- ts.cfg.Logger.Info("delete StorageClasses for EFS")
- return nil
-}
-
-func (ts *tester) createPVC() error {
- scName := storageClassName
- ts.cfg.Logger.Info("creating PersistentVolumeClaim for EFS, provisioning test")
- ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
- _, err := ts.cfg.Client.KubernetesClient().CoreV1().PersistentVolumeClaims(ts.cfg.Namespace).Create(
- ctx,
- &core_v1.PersistentVolumeClaim{
- TypeMeta: meta_v1.TypeMeta{
- APIVersion: "v1",
- Kind: "PersistentVolumeClaim",
- },
- ObjectMeta: meta_v1.ObjectMeta{
- Name: pvcName,
- },
- Spec: core_v1.PersistentVolumeClaimSpec{
- AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteMany},
- StorageClassName: &scName,
- Resources: core_v1.VolumeResourceRequirements{
- Requests: core_v1.ResourceList{
- core_v1.ResourceStorage: api_resource.MustParse("5Gi"),
- },
- },
- },
- },
- meta_v1.CreateOptions{},
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to create PersistentVolumeClaims (%v)", err)
- }
- ts.cfg.Logger.Info("created a PersistentVolumeClaims for EFS")
- return nil
-}
-
-var foreground = meta_v1.DeletePropagationForeground
-
-func (ts *tester) deletePVC() error {
- ts.cfg.Logger.Info("deleting PersistentVolumeClaim for EFS, Provisioning test")
- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
- err := ts.cfg.Client.KubernetesClient().CoreV1().PersistentVolumeClaims(ts.cfg.Namespace).Delete(
- ctx,
- pvcName,
- meta_v1.DeleteOptions{
- PropagationPolicy: &foreground,
- },
- )
- cancel()
- if err != nil {
- return fmt.Errorf("failed to delete PersistentVolumeClaim (%v)", err)
- }
- ts.cfg.Logger.Info("Deleted a PersistentVolumeClaim for EFS")
- return nil
-}
diff --git a/k8s-tester/csi-efs/vend.sh b/k8s-tester/csi-efs/vend.sh
deleted file mode 100755
index 7937e2980..000000000
--- a/k8s-tester/csi-efs/vend.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-< 0 {
- if nodes, err := client.ListNodes(ts.cfg.Client.KubernetesClient()); len(nodes) < ts.cfg.MinimumNodes || err != nil {
- return fmt.Errorf("failed to validate minimum nodes requirement %d (nodes %v, error %v)", ts.cfg.MinimumNodes, len(nodes), err)
- }
- }
-
- latencies := ts.startWrites()
- if len(latencies) == 0 {
- ts.cfg.Logger.Warn("no latency collected")
- return nil
- }
-
- ts.cfg.Logger.Info("sorting write latency results", zap.Int("total-data-points", latencies.Len()))
- now := time.Now()
- sort.Sort(latencies)
- ts.cfg.Logger.Info("sorted write latency results", zap.Int("total-data-points", latencies.Len()), zap.String("took", time.Since(now).String()))
- ts.cfg.LatencySummary.TestID = time.Now().UTC().Format(time.RFC3339Nano)
- ts.cfg.LatencySummary.P50 = latencies.PickP50()
- ts.cfg.LatencySummary.P90 = latencies.PickP90()
- ts.cfg.LatencySummary.P99 = latencies.PickP99()
- ts.cfg.LatencySummary.P999 = latencies.PickP999()
- ts.cfg.LatencySummary.P9999 = latencies.PickP9999()
-
- // https://pkg.go.dev/github.com/prometheus/client_golang/prometheus?tab=doc#Gatherer
- mfs, err := prometheus.DefaultGatherer.Gather()
- if err != nil {
- ts.cfg.Logger.Warn("failed to gather prometheus metrics", zap.Error(err))
- return err
- }
- for _, mf := range mfs {
- if mf == nil {
- continue
- }
- switch *mf.Name {
- case "csrs_client_write_requests_success_total":
- gg := mf.Metric[0].GetGauge()
- ts.cfg.LatencySummary.SuccessTotal = gg.GetValue()
- case "csrs_client_write_requests_failure_total":
- gg := mf.Metric[0].GetGauge()
- ts.cfg.LatencySummary.FailureTotal = gg.GetValue()
- case "csrs_client_write_request_latency_milliseconds":
- ts.cfg.LatencySummary.Histogram, err = latency.ParseHistogram("milliseconds", mf.Metric[0].GetHistogram())
- if err != nil {
- return err
- }
- }
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\nLatencySummary:\n%s\n", ts.cfg.LatencySummary.Table())
-
- return nil
-}
-
-func (ts *tester) Delete() error {
- if ok := ts.runPrompt("delete"); !ok {
- return errors.New("cancelled")
- }
-
- ts.donecCloseOnce.Do(func() {
- close(ts.donec)
- })
-
- var errs []string
-
- if len(errs) > 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- return nil
-}
-
-func (ts *tester) runPrompt(action string) (ok bool) {
- if ts.cfg.Prompt {
- msg := fmt.Sprintf("Ready to %q resources, should we continue?", action)
- prompt := promptui.Select{
- Label: msg,
- Items: []string{
- "No, cancel it!",
- fmt.Sprintf("Yes, let's %q!", action),
- },
- }
- idx, answer, err := prompt.Run()
- if err != nil {
- panic(err)
- }
- if idx != 1 {
- fmt.Printf("cancelled %q [index %d, answer %q]\n", action, idx, answer)
- return false
- }
- }
- return true
-}
-
-func (ts *tester) startWrites() (latencies latency.Durations) {
- ts.cfg.Logger.Info("writing", zap.Int("objects", ts.cfg.Objects), zap.Int("object-size", ts.cfg.Objects))
- latencies = make(latency.Durations, 0, 20000)
-
- for i := 0; i < ts.cfg.Objects; i++ {
- select {
- case <-ts.cfg.Stopc:
- ts.cfg.Logger.Warn("writes stopped")
- return
- case <-ts.donec:
- ts.cfg.Logger.Info("writes done")
- return
- default:
- }
-
- key := fmt.Sprintf("csr%d%s", i, rand.String(7))
- cd := createCond(i, "test via "+key, ts.cfg.InitialRequestConditionType)
-
- start := time.Now()
- ctx, cancel := context.WithTimeout(context.Background(), ts.cfg.Client.Config().ClientTimeout)
- _, err := ts.cfg.Client.KubernetesClient().
- CertificatesV1beta1().
- CertificateSigningRequests().
- Create(ctx, &certificates_v1beta1.CertificateSigningRequest{
- TypeMeta: meta_v1.TypeMeta{
- APIVersion: "certificates.k8s.io/v1beta1",
- Kind: "CertificateSigningRequest",
- },
- ObjectMeta: meta_v1.ObjectMeta{
- Name: key,
- GenerateName: key,
- CreationTimestamp: meta_v1.Time{Time: time.Now().Add(-20 * time.Minute)},
- },
- Spec: certificates_v1beta1.CertificateSigningRequestSpec{
- Groups: []string{"system:bootstrappers", "system:nodes", "system:authenticated"},
- Request: reqData,
- UID: "heptio-authenticator-aws:280347406217:AROAUCRQB56EUYTYXXJKV",
- Usages: []certificates_v1beta1.KeyUsage{
- certificates_v1beta1.UsageDigitalSignature,
- certificates_v1beta1.UsageKeyEncipherment,
- certificates_v1beta1.UsageServerAuth,
- },
- Username: "system:node:ip-172-20-32-89.us-west-2.compute.internal",
- },
- Status: certificates_v1beta1.CertificateSigningRequestStatus{
- Certificate: nil,
- Conditions: cd,
- },
- }, meta_v1.CreateOptions{})
- cancel()
- took := time.Since(start)
- tookMS := float64(took / time.Millisecond)
- writeRequestLatencyMs.Observe(tookMS)
- latencies = append(latencies, took)
- if err != nil {
- writeRequestsFailureTotal.Inc()
- ts.cfg.Logger.Warn("write csr failed", zap.Error(err))
- } else {
- writeRequestsSuccessTotal.Inc()
- if i%20 == 0 {
- ts.cfg.Logger.Info("wrote csr", zap.Int("iteration", i))
- }
- }
- }
- return latencies
-}
-
-var conds = []certificates_v1beta1.RequestConditionType{
- certificates_v1beta1.CertificateApproved,
- certificates_v1beta1.CertificateDenied,
- certificates_v1beta1.RequestConditionType(""),
-}
-
-func createCond(idx int, msg string, tp string) (cs []certificates_v1beta1.CertificateSigningRequestCondition) {
- cs = []certificates_v1beta1.CertificateSigningRequestCondition{
- {
- Reason: "Test",
- Message: msg,
- LastUpdateTime: meta_v1.NewTime(time.Now().Add(-time.Hour)),
- },
- }
- switch tp {
- case string(certificates_v1beta1.CertificateApproved):
- cs[0].Type = certificates_v1beta1.CertificateApproved
- case string(certificates_v1beta1.CertificateDenied):
- cs[0].Type = certificates_v1beta1.CertificateDenied
- case "Pending", "":
- cs = make([]certificates_v1beta1.CertificateSigningRequestCondition, 0)
- case "Random":
- cs[0].Type = conds[idx%3]
- }
- return cs
-}
-
-var reqData, _ = base64.StdEncoding.DecodeString("LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0KTUlJQnJEQ0NBVk1DQVFBd1dERVZNQk1HQTFVRUNoTU1jM2x6ZEdWdE9tNXZaR1Z6TVQ4d1BRWURWUVFERXpaegplWE4wWlcwNmJtOWtaVHBwY0MweE56SXRNakF0TXpJdE9Ea3VkWE10ZDJWemRDMHlMbU52YlhCMWRHVXVhVzUwClpYSnVZV3d3V1RBVEJnY3Foa2pPUFFJQkJnZ3Foa2pPUFFNQkJ3TkNBQVJGSzI3L2w4U2NtMXF1K2xXbEs5WFoKUUtVM0grSnFENTZuSEFYOXBUQ25YVWRQaUppemRzc01QaSs2emtCU1I2MXVJcVRsdnNIcjkwbFNyU2tQeDd1aQpvSUdZTUlHVkJna3Foa2lHOXcwQkNRNHhnWWN3Z1lRd2dZRUdBMVVkRVFSNk1IaUNNbVZqTWkwMU5DMHhPRFV0Ck1qUTJMVEV5T0M1MWN5MTNaWE4wTFRJdVkyOXRjSFYwWlM1aGJXRjZiMjVoZDNNdVkyOXRod1NzRkNCWmh3UTIKdWZhQWhqWnplWE4wWlcwNmJtOWtaVHBwY0MweE56SXRNakF0TXpJdE9Ea3VkWE10ZDJWemRDMHlMbU52YlhCMQpkR1V1YVc1MFpYSnVZV3d3Q2dZSUtvWkl6ajBFQXdJRFJ3QXdSQUlnVTUrNEFkWVcvRm9kdDExMmgvRjV4RHFQClFJS1BJemk4TUJMSTBBaVE2cGtDSUdqOHZPNDlTQldJVlo2SnhJL1lENldrRVhXdlZEbFp4cjFlZmVMM0NIeEgKLS0tLS1FTkQgQ0VSVElGSUNBVEUgUkVRVUVTVC0tLS0tCg==")
-
-/*
-https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/
-
-$ cat < /tmp/csr.out && openssl req -text -noout -in /tmp/csr.out
-
-*/
diff --git a/k8s-tester/csrs/vend.sh b/k8s-tester/csrs/vend.sh
deleted file mode 100755
index 7937e2980..000000000
--- a/k8s-tester/csrs/vend.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-< 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- return nil
-}
-
-func (ts *tester) runPrompt(action string) (ok bool) {
- if ts.cfg.Prompt {
- msg := fmt.Sprintf("Ready to %q resources, should we continue?", action)
- prompt := promptui.Select{
- Label: msg,
- Items: []string{
- "No, cancel it!",
- fmt.Sprintf("Yes, let's %q!", action),
- },
- }
- idx, answer, err := prompt.Run()
- if err != nil {
- panic(err)
- }
- if idx != 1 {
- fmt.Printf("cancelled %q [index %d, answer %q]\n", action, idx, answer)
- return false
- }
- }
- return true
-}
-
-func (ts *tester) checkForStorageClass() (err error) {
- storageclass, err := client.ListStorageClass(
- ts.cfg.Logger,
- ts.cfg.Client.KubernetesClient(),
- 5,
- 5*time.Second,
- )
- for _, class := range storageclass {
- if class.ObjectMeta.Annotations["storageclass.kubernetes.io/is-default-class"] == "true" {
- ts.cfg.Logger.Info("FOUND DEFAULT STORAGE CLASS")
- return nil
- } else {
- return errors.New("No Default StroageClass")
- }
- }
- return nil
-}
-
-// https://github.com/epsagon/helm-charts
-func (ts *tester) createHelmEpsagon() error {
- values := map[string]interface{}{
- "metrics": map[string]interface{}{
- "enabled": true,
- },
- "metrics-agent": map[string]interface{}{
- "server": map[string]interface{}{
- "remoteWrite[0]": map[string]interface{}{
- "url": ts.cfg.CollectorEndpoint,
- },
- },
- },
- "epsagonToken": ts.cfg.APIToken,
- "clusterName": ts.cfg.ClusterName,
- }
- getAllArgs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "get",
- "all",
- }
- getAllCmd := strings.Join(getAllArgs, " ")
-
- descArgsDs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "describe",
- "deployment.apps/epsagon-agent-cluster-agent",
- }
- descCmdDs := strings.Join(descArgsDs, " ")
-
- descArgsPods := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "describe",
- "pods",
- "--selector=app.kubernetes.io/instance=epsagon-agent",
- }
- descCmdPods := strings.Join(descArgsPods, " ")
-
- logArgs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "logs",
- "--selector=app.kubernetes.io/instance=epsagon-agent",
- "--all-containers=true",
- "--timestamps",
- }
- logsCmd := strings.Join(logArgs, " ")
-
- return helm.Install(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Stopc: ts.cfg.Stopc,
- Timeout: 10 * time.Minute,
- KubeconfigPath: ts.cfg.Client.Config().KubeconfigPath,
- Namespace: ts.cfg.Namespace,
- ChartRepoURL: ts.cfg.HelmChartRepoURL,
- ChartName: chartName,
- ReleaseName: chartName,
- Values: values,
- LogFunc: func(format string, v ...interface{}) {
- ts.cfg.Logger.Info(fmt.Sprintf("[install] "+format, v...))
- },
- QueryFunc: func() {
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, getAllArgs[0], getAllArgs[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl get all' failed", zap.Error(err))
- ts.cfg.Logger.Warn("Epsagon Tests::", zap.String("TEST", "FAILED"))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getAllCmd, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, descArgsDs[0], descArgsDs[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe daemonset' failed", zap.Error(err))
- ts.cfg.Logger.Warn("Epsagon Tests::", zap.String("TEST", "FAILED"))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmdDs, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, descArgsPods[0], descArgsPods[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe pods' failed", zap.Error(err))
- ts.cfg.Logger.Warn("Epsagon Tests::", zap.String("TEST", "FAILED"))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmdPods, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, logArgs[0], logArgs[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl logs' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", logsCmd, out)
- },
- QueryInterval: 30 * time.Second,
- })
-}
-
-func (ts *tester) deleteHelmEpsagon() error {
- return helm.Uninstall(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Timeout: 15 * time.Minute,
- KubeconfigPath: ts.cfg.Client.Config().KubeconfigPath,
- Namespace: ts.cfg.Namespace,
- ChartName: chartName,
- ReleaseName: chartName,
- })
-}
diff --git a/k8s-tester/epsagon/vend.sh b/k8s-tester/epsagon/vend.sh
deleted file mode 100755
index 7937e2980..000000000
--- a/k8s-tester/epsagon/vend.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-< 0 {
- return errors.New(strings.Join(errs, ", "))
- }
-
- return nil
-}
-
-func (ts *tester) runPrompt(action string) (ok bool) {
- if ts.cfg.Prompt {
- msg := fmt.Sprintf("Ready to %q resources, should we continue?", action)
- prompt := promptui.Select{
- Label: msg,
- Items: []string{
- "No, cancel it!",
- fmt.Sprintf("Yes, let's %q!", action),
- },
- }
- idx, answer, err := prompt.Run()
- if err != nil {
- panic(err)
- }
- if idx != 1 {
- fmt.Printf("cancelled %q [index %d, answer %q]\n", action, idx, answer)
- return false
- }
- }
- return true
-}
-
-// https://github.com/falcosecurity/charts/blob/master/falco/values.yaml
-func (ts *tester) createHelmFalco() error {
- values := map[string]interface{}{
- "image": map[string]interface{}{
- "tag": "0.28.1",
- },
- }
-
- getAllArgs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "get",
- "all",
- }
- getAllCmd := strings.Join(getAllArgs, " ")
-
- descArgsDs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "describe",
- "daemonset.apps/falco",
- }
- descCmdDs := strings.Join(descArgsDs, " ")
-
- descArgsPods := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "describe",
- "pods",
- "--selector=app=falco",
- }
- descCmdPods := strings.Join(descArgsPods, " ")
-
- logArgs := []string{
- ts.cfg.Client.Config().KubectlPath,
- "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath,
- "--namespace=" + ts.cfg.Namespace,
- "logs",
- "--selector=app=falco",
- "--all-containers=true",
- "--timestamps",
- }
- logsCmd := strings.Join(logArgs, " ")
-
- return helm.Install(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Stopc: ts.cfg.Stopc,
- Timeout: 10 * time.Minute,
- KubeconfigPath: ts.cfg.Client.Config().KubeconfigPath,
- Namespace: ts.cfg.Namespace,
- ChartRepoURL: ts.cfg.HelmChartRepoURL,
- ChartName: chartName,
- ReleaseName: chartName,
- Values: values,
- LogFunc: func(format string, v ...interface{}) {
- ts.cfg.Logger.Info(fmt.Sprintf("[install] "+format, v...))
- },
- QueryFunc: func() {
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
- output, err := exec.New().CommandContext(ctx, getAllArgs[0], getAllArgs[1:]...).CombinedOutput()
- cancel()
- out := strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl get all' failed", zap.Error(err))
- ts.cfg.Logger.Warn("Falco Tests::", zap.String("TEST", "FAILED"))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getAllCmd, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, descArgsDs[0], descArgsDs[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe daemonset' failed", zap.Error(err))
- ts.cfg.Logger.Warn("Falco Tests::", zap.String("TEST", "FAILED"))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmdDs, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, descArgsPods[0], descArgsPods[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl describe pods' failed", zap.Error(err))
- ts.cfg.Logger.Warn("Falco Tests::", zap.String("TEST", "FAILED"))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmdPods, out)
-
- ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
- output, err = exec.New().CommandContext(ctx, logArgs[0], logArgs[1:]...).CombinedOutput()
- cancel()
- out = strings.TrimSpace(string(output))
- if err != nil {
- ts.cfg.Logger.Warn("'kubectl logs' failed", zap.Error(err))
- }
- fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", logsCmd, out)
- },
- QueryInterval: 30 * time.Second,
- })
-}
-
-func (ts *tester) deleteHelmFalco() error {
- return helm.Uninstall(helm.InstallConfig{
- Logger: ts.cfg.Logger,
- LogWriter: ts.cfg.LogWriter,
- Timeout: 15 * time.Minute,
- KubeconfigPath: ts.cfg.Client.Config().KubeconfigPath,
- Namespace: ts.cfg.Namespace,
- ChartName: chartName,
- ReleaseName: chartName,
- })
-}
diff --git a/k8s-tester/falco/vend.sh b/k8s-tester/falco/vend.sh
deleted file mode 100755
index 7937e2980..000000000
--- a/k8s-tester/falco/vend.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-<[^ ]*) (?[^ ]*) (?[^ ]*) \[(?