diff --git a/test/e2e-framework/common/config/environment.go b/test/e2e-framework/common/config/environment.go index cd22f763b35749..344202fde92096 100644 --- a/test/e2e-framework/common/config/environment.go +++ b/test/e2e-framework/common/config/environment.go @@ -235,7 +235,7 @@ func (e *CommonEnvironment) KubernetesVersion() string { } func (e *CommonEnvironment) KindVersion() string { - return e.GetStringWithDefault(e.InfraConfig, DDInfraKindVersion, "v0.30.0") + return e.GetStringWithDefault(e.InfraConfig, DDInfraKindVersion, "v0.31.0") } func (e *CommonEnvironment) KubeNodeURL() string { diff --git a/test/e2e-framework/components/kubernetes/cilium/hosts.toml b/test/e2e-framework/components/kubernetes/cilium/hosts.toml new file mode 100644 index 00000000000000..7e76cab395ec27 --- /dev/null +++ b/test/e2e-framework/components/kubernetes/cilium/hosts.toml @@ -0,0 +1,8 @@ +server = "https://docker.io" + +[host."https://mirror.gcr.io"] + capabilities = ["pull", "resolve"] +[host."https://registry-1.docker.io"] + capabilities = ["pull", "resolve"] +[host."https://docker.io"] + capabilities = ["pull", "resolve"] diff --git a/test/e2e-framework/components/kubernetes/cilium/kind-cluster-v1.35+.yaml b/test/e2e-framework/components/kubernetes/cilium/kind-cluster-v1.35+.yaml new file mode 100644 index 00000000000000..ac615b98dd5514 --- /dev/null +++ b/test/e2e-framework/components/kubernetes/cilium/kind-cluster-v1.35+.yaml @@ -0,0 +1,24 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraMounts: + - hostPath: /proc + containerPath: /host/proc + - hostPath: ./certs.d + containerPath: "/etc/containerd/certs.d" +- role: worker + extraMounts: + - hostPath: /proc + containerPath: /host/proc +containerdConfigPatches: + - |- + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" +networking: + apiServerAddress: "0.0.0.0" + apiServerPort: 8443 + disableDefaultCNI: true +{{if .KubeProxyReplacement}} + kubeProxyMode: "none" +{{end}} diff --git a/test/e2e-framework/components/kubernetes/cilium/kind.go b/test/e2e-framework/components/kubernetes/cilium/kind.go index 02e69bcbd82e9b..bd86c5dfb0a1a6 100644 --- a/test/e2e-framework/components/kubernetes/cilium/kind.go +++ b/test/e2e-framework/components/kubernetes/cilium/kind.go @@ -12,6 +12,7 @@ import ( "strings" "text/template" + "github.com/Masterminds/semver/v3" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" "gopkg.in/yaml.v3" @@ -25,14 +26,29 @@ import ( //go:embed kind-cilium-cluster.yaml var kindCilumClusterFS embed.FS -func kindKubeClusterConfigFromCiliumParams(params *Params) (string, error) { +//go:embed kind-cluster-v1.35+.yaml +var kindCiliumV135ClusterFS embed.FS + +//go:embed hosts.toml +var containerdDockerioHostConfig string + +func kindKubeClusterConfigFromCiliumParams(params *Params, kubeVersion string) (string, error) { o := struct { KubeProxyReplacement bool }{ KubeProxyReplacement: params.hasKubeProxyReplacement(), } - kindCiliumClusterTemplate, err := template.ParseFS(kindCilumClusterFS, "kind-cilium-cluster.yaml") + var kindCiliumCluster embed.FS = kindCilumClusterFS + if index := strings.Index(kubeVersion, "@"); index != -1 { + kubeVersion = kubeVersion[:index] + } + + if semver.MustParse(kubeVersion).GreaterThanEqual(semver.MustParse("v1.35.0")) { + kindCiliumCluster = kindCiliumV135ClusterFS + } + + kindCiliumClusterTemplate, err := template.ParseFS(kindCiliumCluster, "kind-cilium-cluster.yaml") if err != nil { return "", err } @@ -51,12 +67,12 @@ func NewKindCluster(env config.Env, vm *remote.Host, name string, kubeVersion st return nil, fmt.Errorf("could not create cilium params from opts: %w", err) } - clusterConfig, err := kindKubeClusterConfigFromCiliumParams(params) + clusterConfig, err := kindKubeClusterConfigFromCiliumParams(params, kubeVersion) if err != nil { return nil, err } - cluster, err := kubernetes.NewKindClusterWithConfig(env, vm, name, kubeVersion, clusterConfig, opts...) + cluster, err := kubernetes.NewKindClusterWithConfig(env, vm, name, kubeVersion, clusterConfig, containerdDockerioHostConfig, opts...) if err != nil { return nil, err } diff --git a/test/e2e-framework/components/kubernetes/hosts.toml b/test/e2e-framework/components/kubernetes/hosts.toml new file mode 100644 index 00000000000000..7e76cab395ec27 --- /dev/null +++ b/test/e2e-framework/components/kubernetes/hosts.toml @@ -0,0 +1,8 @@ +server = "https://docker.io" + +[host."https://mirror.gcr.io"] + capabilities = ["pull", "resolve"] +[host."https://registry-1.docker.io"] + capabilities = ["pull", "resolve"] +[host."https://docker.io"] + capabilities = ["pull", "resolve"] diff --git a/test/e2e-framework/components/kubernetes/kind-cluster-v1.35+.yaml b/test/e2e-framework/components/kubernetes/kind-cluster-v1.35+.yaml new file mode 100644 index 00000000000000..95b788ddffed21 --- /dev/null +++ b/test/e2e-framework/components/kubernetes/kind-cluster-v1.35+.yaml @@ -0,0 +1,16 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraMounts: + - hostPath: /proc + containerPath: /host/proc + - hostPath: ./certs.d + containerPath: "/etc/containerd/certs.d" +containerdConfigPatches: + - |- + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" +networking: + apiServerAddress: "0.0.0.0" + apiServerPort: 8443 diff --git a/test/e2e-framework/components/kubernetes/kind.go b/test/e2e-framework/components/kubernetes/kind.go index 31ee84d6d52c08..c7ccdc6de97357 100644 --- a/test/e2e-framework/components/kubernetes/kind.go +++ b/test/e2e-framework/components/kubernetes/kind.go @@ -12,6 +12,7 @@ import ( "regexp" "strings" + "github.com/Masterminds/semver/v3" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" "github.com/DataDog/datadog-agent/test/e2e-framework/common/config" @@ -29,11 +30,29 @@ const ( ) //go:embed kind-cluster.yaml -var kindClusterConfig string +var kindClusterConfigV131 string + +//go:embed kind-cluster-v1.35+.yaml +var kindClusterConfigV132Plus string + +//go:embed hosts.toml +var containerdDockerioHostConfig string + +func GetKindConfig(kubeVersion string) string { + if index := strings.Index(kubeVersion, "@"); index != -1 { + kubeVersion = kubeVersion[:index] + } + + if semver.MustParse(kubeVersion).GreaterThanEqual(semver.MustParse("v1.32.0")) { + return kindClusterConfigV132Plus + } + + return kindClusterConfigV131 +} // Install Kind on a Linux virtual machine. func NewKindCluster(env config.Env, vm *remote.Host, name string, kubeVersion string, opts ...pulumi.ResourceOption) (*Cluster, error) { - return NewKindClusterWithConfig(env, vm, name, kubeVersion, kindClusterConfig, opts...) + return NewKindClusterWithConfig(env, vm, name, kubeVersion, GetKindConfig(kubeVersion), containerdDockerioHostConfig, opts...) } func validateKubeVersionFormat(kubeVersion string) error { @@ -50,7 +69,7 @@ func validateKubeVersionFormat(kubeVersion string) error { return nil } -func NewKindClusterWithConfig(env config.Env, vm *remote.Host, name string, kubeVersion, kindConfig string, opts ...pulumi.ResourceOption) (*Cluster, error) { +func NewKindClusterWithConfig(env config.Env, vm *remote.Host, name string, kubeVersion, kindConfig, containerdOverrideConfig string, opts ...pulumi.ResourceOption) (*Cluster, error) { return components.NewComponent(env, name, func(clusterComp *Cluster) error { kindClusterName := env.CommonNamer().DisplayName(49) // We can have some issues if the name is longer than 50 characters opts = utils.MergeOptions[pulumi.ResourceOption](opts, pulumi.Parent(clusterComp)) @@ -84,8 +103,9 @@ func NewKindClusterWithConfig(env config.Env, vm *remote.Host, name string, kube } kindVersionConfig = &KindConfig{ - KindVersion: env.KindVersion(), - NodeImageVersion: kubeVersion, + KindVersion: env.KindVersion(), + NodeImageVersion: kubeVersion, + UseNewContainerdConfig: true, } } @@ -102,6 +122,32 @@ func NewKindClusterWithConfig(env config.Env, vm *remote.Host, name string, kube return err } + // pre-create the folder architecture for containerd overrides + // The kind cluster configuration has a mount point + // - host: /tmp/certs.d/ + // - cluster: /etc/containerd/certs.d/ + // this way we can place configuration override files in that 'certs.d' folder + + containerdOverrideDir := "/tmp/certs.d/docker.io/" + containerdDir, err := vm.OS.FileManager().CreateDirectory(containerdOverrideDir, false, opts...) + if err != nil { + return err + } + + if kindVersionConfig.UseNewContainerdConfig { + // for kind versions using containerd > 2.x we need a different config format + // kind started using containerd 2.x in kind v0.27.0 + containerdConfigFilePath := containerdOverrideDir + "hosts.toml" + _, err = vm.OS.FileManager().CopyInlineFile( + pulumi.String(containerdOverrideConfig), + containerdConfigFilePath, + utils.MergeOptions(opts, utils.PulumiDependsOn(containerdDir))..., + ) + if err != nil { + return err + } + } + /* The internal mirror should be able to pull arbitrary kubernetes images but the sha is required with the tag. We also support the user supplying the url (in case we want to host @@ -170,7 +216,7 @@ func NewLocalKindCluster(env config.Env, name string, kubeVersion string, opts . clusterConfig, err := runner.Command("kind-config", &command.Args{ Create: pulumi.Sprintf("cat - | tee %s > /dev/null", clusterConfigFilePath), Delete: pulumi.Sprintf("rm -f %s", clusterConfigFilePath), - Stdin: pulumi.String(kindClusterConfig), + Stdin: pulumi.String(GetKindConfig(kubeVersion)), }, opts...) if err != nil { return err @@ -182,7 +228,7 @@ func NewLocalKindCluster(env config.Env, name string, kubeVersion string, opts . &command.Args{ Create: pulumi.Sprintf("kind create cluster --name %s --config %s --image %s --wait %s", kindClusterName, clusterConfigFilePath, nodeImage, kindReadinessWait), Delete: pulumi.Sprintf("kind delete cluster --name %s", kindClusterName), - Triggers: pulumi.Array{pulumi.String(kindClusterConfig)}, + Triggers: pulumi.Array{pulumi.String(GetKindConfig(kubeVersion))}, }, utils.MergeOptions(opts, utils.PulumiDependsOn(clusterConfig), pulumi.DeleteBeforeReplace(true))..., ) diff --git a/test/e2e-framework/components/kubernetes/kind_versions.go b/test/e2e-framework/components/kubernetes/kind_versions.go index 5a37843c05895f..c820b5189f7c13 100644 --- a/test/e2e-framework/components/kubernetes/kind_versions.go +++ b/test/e2e-framework/components/kubernetes/kind_versions.go @@ -14,24 +14,30 @@ import ( // KindConfig contains the kind version and the kind node image to use type KindConfig struct { - KindVersion string - NodeImageVersion string + KindVersion string + NodeImageVersion string + UseNewContainerdConfig bool } // Source: https://github.com/kubernetes-sigs/kind/releases var kubeToKindVersion = map[string]KindConfig{ "1.34": { - KindVersion: "v0.30.0", - NodeImageVersion: "v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a", + KindVersion: "v0.30.0", + NodeImageVersion: "v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a", + UseNewContainerdConfig: true, }, "1.33": { - KindVersion: "v0.28.0", - NodeImageVersion: "v1.33.0@sha256:91e9ed777db80279c22d1d1068c091b899b2078506e4a0f797fbf6e397c0b0b2", + KindVersion: "v0.28.0", + NodeImageVersion: "v1.33.0@sha256:91e9ed777db80279c22d1d1068c091b899b2078506e4a0f797fbf6e397c0b0b2", + UseNewContainerdConfig: true, }, "1.32": { - KindVersion: "v0.26.0", - NodeImageVersion: "v1.32.0@sha256:c48c62eac5da28cdadcf560d1d8616cfa6783b58f0d94cf63ad1bf49600cb027", + KindVersion: "v0.27.0", + NodeImageVersion: "v1.32.0@sha256:c48c62eac5da28cdadcf560d1d8616cfa6783b58f0d94cf63ad1bf49600cb027", + UseNewContainerdConfig: true, }, + // kind versions bellow this line will use containerd <= 2.x and so will need a different containerd docker.io registry mirror config format + // we can leave the UseNewContainerdConfig to false as its default value is false "1.31": { KindVersion: "v0.26.0", NodeImageVersion: "v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30", diff --git a/test/e2e-framework/go.mod b/test/e2e-framework/go.mod index f0fa3bd3e867a1..4fa032150abb4d 100644 --- a/test/e2e-framework/go.mod +++ b/test/e2e-framework/go.mod @@ -11,6 +11,7 @@ require ( github.com/DataDog/datadog-agent/test/new-e2e v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-api-client-go/v2 v2.52.0 github.com/Masterminds/semver v1.5.0 + github.com/Masterminds/semver/v3 v3.4.0 github.com/alessio/shellescape v1.4.2 github.com/aws/aws-sdk-go-v2 v1.41.0 github.com/aws/aws-sdk-go-v2/config v1.32.6