Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion test/e2e-framework/common/config/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ func (e *CommonEnvironment) KubernetesVersion() string {
}

func (e *CommonEnvironment) KindVersion() string {
return e.GetStringWithDefault(e.InfraConfig, DDInfraKindVersion, "v0.30.0")
return e.GetStringWithDefault(e.InfraConfig, DDInfraKindVersion, "v0.31.0")
}

func (e *CommonEnvironment) KubeNodeURL() string {
Expand Down
8 changes: 8 additions & 0 deletions test/e2e-framework/components/kubernetes/cilium/hosts.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
server = "https://docker.io"

[host."https://mirror.gcr.io"]
capabilities = ["pull", "resolve"]
[host."https://registry-1.docker.io"]
capabilities = ["pull", "resolve"]
[host."https://docker.io"]
capabilities = ["pull", "resolve"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
extraMounts:
- hostPath: /proc
containerPath: /host/proc
- hostPath: ./certs.d
containerPath: "/etc/containerd/certs.d"
- role: worker
extraMounts:
- hostPath: /proc
containerPath: /host/proc
containerdConfigPatches:
- |-
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
networking:
apiServerAddress: "0.0.0.0"
apiServerPort: 8443
disableDefaultCNI: true
{{if .KubeProxyReplacement}}
kubeProxyMode: "none"
{{end}}
24 changes: 20 additions & 4 deletions test/e2e-framework/components/kubernetes/cilium/kind.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"strings"
"text/template"

"github.com/Masterminds/semver/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
"gopkg.in/yaml.v3"

Expand All @@ -25,14 +26,29 @@ import (
//go:embed kind-cilium-cluster.yaml
var kindCilumClusterFS embed.FS

func kindKubeClusterConfigFromCiliumParams(params *Params) (string, error) {
//go:embed kind-cluster-v1.35+.yaml
var kindCiliumV135ClusterFS embed.FS

//go:embed hosts.toml
var containerdDockerioHostConfig string

func kindKubeClusterConfigFromCiliumParams(params *Params, kubeVersion string) (string, error) {
o := struct {
KubeProxyReplacement bool
}{
KubeProxyReplacement: params.hasKubeProxyReplacement(),
}

kindCiliumClusterTemplate, err := template.ParseFS(kindCilumClusterFS, "kind-cilium-cluster.yaml")
var kindCiliumCluster embed.FS = kindCilumClusterFS
if index := strings.Index(kubeVersion, "@"); index != -1 {
kubeVersion = kubeVersion[:index]
}

if semver.MustParse(kubeVersion).GreaterThanEqual(semver.MustParse("v1.35.0")) {
kindCiliumCluster = kindCiliumV135ClusterFS
}

kindCiliumClusterTemplate, err := template.ParseFS(kindCiliumCluster, "kind-cilium-cluster.yaml")
if err != nil {
return "", err
}
Expand All @@ -51,12 +67,12 @@ func NewKindCluster(env config.Env, vm *remote.Host, name string, kubeVersion st
return nil, fmt.Errorf("could not create cilium params from opts: %w", err)
}

clusterConfig, err := kindKubeClusterConfigFromCiliumParams(params)
clusterConfig, err := kindKubeClusterConfigFromCiliumParams(params, kubeVersion)
if err != nil {
return nil, err
}

cluster, err := kubernetes.NewKindClusterWithConfig(env, vm, name, kubeVersion, clusterConfig, opts...)
cluster, err := kubernetes.NewKindClusterWithConfig(env, vm, name, kubeVersion, clusterConfig, containerdDockerioHostConfig, opts...)
if err != nil {
return nil, err
}
Expand Down
8 changes: 8 additions & 0 deletions test/e2e-framework/components/kubernetes/hosts.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
server = "https://docker.io"

[host."https://mirror.gcr.io"]
capabilities = ["pull", "resolve"]
[host."https://registry-1.docker.io"]
capabilities = ["pull", "resolve"]
[host."https://docker.io"]
capabilities = ["pull", "resolve"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
extraMounts:
- hostPath: /proc
containerPath: /host/proc
- hostPath: ./certs.d
containerPath: "/etc/containerd/certs.d"
containerdConfigPatches:
- |-
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
networking:
apiServerAddress: "0.0.0.0"
apiServerPort: 8443
60 changes: 53 additions & 7 deletions test/e2e-framework/components/kubernetes/kind.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"regexp"
"strings"

"github.com/Masterminds/semver/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"

"github.com/DataDog/datadog-agent/test/e2e-framework/common/config"
Expand All @@ -29,11 +30,29 @@ const (
)

//go:embed kind-cluster.yaml
var kindClusterConfig string
var kindClusterConfigV131 string

//go:embed kind-cluster-v1.35+.yaml
var kindClusterConfigV132Plus string

//go:embed hosts.toml
var containerdDockerioHostConfig string

func GetKindConfig(kubeVersion string) string {
if index := strings.Index(kubeVersion, "@"); index != -1 {
kubeVersion = kubeVersion[:index]
}

if semver.MustParse(kubeVersion).GreaterThanEqual(semver.MustParse("v1.32.0")) {
return kindClusterConfigV132Plus
}

return kindClusterConfigV131
}

// Install Kind on a Linux virtual machine.
func NewKindCluster(env config.Env, vm *remote.Host, name string, kubeVersion string, opts ...pulumi.ResourceOption) (*Cluster, error) {
return NewKindClusterWithConfig(env, vm, name, kubeVersion, kindClusterConfig, opts...)
return NewKindClusterWithConfig(env, vm, name, kubeVersion, GetKindConfig(kubeVersion), containerdDockerioHostConfig, opts...)
}

func validateKubeVersionFormat(kubeVersion string) error {
Expand All @@ -50,7 +69,7 @@ func validateKubeVersionFormat(kubeVersion string) error {
return nil
}

func NewKindClusterWithConfig(env config.Env, vm *remote.Host, name string, kubeVersion, kindConfig string, opts ...pulumi.ResourceOption) (*Cluster, error) {
func NewKindClusterWithConfig(env config.Env, vm *remote.Host, name string, kubeVersion, kindConfig, containerdOverrideConfig string, opts ...pulumi.ResourceOption) (*Cluster, error) {
return components.NewComponent(env, name, func(clusterComp *Cluster) error {
kindClusterName := env.CommonNamer().DisplayName(49) // We can have some issues if the name is longer than 50 characters
opts = utils.MergeOptions[pulumi.ResourceOption](opts, pulumi.Parent(clusterComp))
Expand Down Expand Up @@ -84,8 +103,9 @@ func NewKindClusterWithConfig(env config.Env, vm *remote.Host, name string, kube
}

kindVersionConfig = &KindConfig{
KindVersion: env.KindVersion(),
NodeImageVersion: kubeVersion,
KindVersion: env.KindVersion(),
NodeImageVersion: kubeVersion,
UseNewContainerdConfig: true,
}
}

Expand All @@ -102,6 +122,32 @@ func NewKindClusterWithConfig(env config.Env, vm *remote.Host, name string, kube
return err
}

// pre-create the folder architecture for containerd overrides
// The kind cluster configuration has a mount point
// - host: /tmp/certs.d/
// - cluster: /etc/containerd/certs.d/
// this way we can place configuration override files in that 'certs.d' folder

containerdOverrideDir := "/tmp/certs.d/docker.io/"
containerdDir, err := vm.OS.FileManager().CreateDirectory(containerdOverrideDir, false, opts...)
if err != nil {
return err
}

if kindVersionConfig.UseNewContainerdConfig {
// for kind versions using containerd > 2.x we need a different config format
// kind started using containerd 2.x in kind v0.27.0
containerdConfigFilePath := containerdOverrideDir + "hosts.toml"
_, err = vm.OS.FileManager().CopyInlineFile(
pulumi.String(containerdOverrideConfig),
containerdConfigFilePath,
utils.MergeOptions(opts, utils.PulumiDependsOn(containerdDir))...,
)
if err != nil {
return err
}
}

/*
The internal mirror should be able to pull arbitrary kubernetes images but the sha is required
with the tag. We also support the user supplying the url (in case we want to host
Expand Down Expand Up @@ -170,7 +216,7 @@ func NewLocalKindCluster(env config.Env, name string, kubeVersion string, opts .
clusterConfig, err := runner.Command("kind-config", &command.Args{
Create: pulumi.Sprintf("cat - | tee %s > /dev/null", clusterConfigFilePath),
Delete: pulumi.Sprintf("rm -f %s", clusterConfigFilePath),
Stdin: pulumi.String(kindClusterConfig),
Stdin: pulumi.String(GetKindConfig(kubeVersion)),
}, opts...)
if err != nil {
return err
Expand All @@ -182,7 +228,7 @@ func NewLocalKindCluster(env config.Env, name string, kubeVersion string, opts .
&command.Args{
Create: pulumi.Sprintf("kind create cluster --name %s --config %s --image %s --wait %s", kindClusterName, clusterConfigFilePath, nodeImage, kindReadinessWait),
Delete: pulumi.Sprintf("kind delete cluster --name %s", kindClusterName),
Triggers: pulumi.Array{pulumi.String(kindClusterConfig)},
Triggers: pulumi.Array{pulumi.String(GetKindConfig(kubeVersion))},
},
utils.MergeOptions(opts, utils.PulumiDependsOn(clusterConfig), pulumi.DeleteBeforeReplace(true))...,
)
Expand Down
22 changes: 14 additions & 8 deletions test/e2e-framework/components/kubernetes/kind_versions.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,30 @@ import (

// KindConfig contains the kind version and the kind node image to use
type KindConfig struct {
KindVersion string
NodeImageVersion string
KindVersion string
NodeImageVersion string
UseNewContainerdConfig bool
}

// Source: https://github.com/kubernetes-sigs/kind/releases
var kubeToKindVersion = map[string]KindConfig{
"1.34": {
KindVersion: "v0.30.0",
NodeImageVersion: "v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a",
KindVersion: "v0.30.0",
NodeImageVersion: "v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a",
UseNewContainerdConfig: true,
},
"1.33": {
KindVersion: "v0.28.0",
NodeImageVersion: "v1.33.0@sha256:91e9ed777db80279c22d1d1068c091b899b2078506e4a0f797fbf6e397c0b0b2",
KindVersion: "v0.28.0",
NodeImageVersion: "v1.33.0@sha256:91e9ed777db80279c22d1d1068c091b899b2078506e4a0f797fbf6e397c0b0b2",
UseNewContainerdConfig: true,
},
"1.32": {
KindVersion: "v0.26.0",
NodeImageVersion: "v1.32.0@sha256:c48c62eac5da28cdadcf560d1d8616cfa6783b58f0d94cf63ad1bf49600cb027",
KindVersion: "v0.27.0",
NodeImageVersion: "v1.32.0@sha256:c48c62eac5da28cdadcf560d1d8616cfa6783b58f0d94cf63ad1bf49600cb027",
UseNewContainerdConfig: true,
},
// kind versions bellow this line will use containerd <= 2.x and so will need a different containerd docker.io registry mirror config format
// we can leave the UseNewContainerdConfig to false as its default value is false
"1.31": {
KindVersion: "v0.26.0",
NodeImageVersion: "v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30",
Expand Down
1 change: 1 addition & 0 deletions test/e2e-framework/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ require (
github.com/DataDog/datadog-agent/test/new-e2e v0.0.0-00010101000000-000000000000
github.com/DataDog/datadog-api-client-go/v2 v2.52.0
github.com/Masterminds/semver v1.5.0
github.com/Masterminds/semver/v3 v3.4.0
github.com/alessio/shellescape v1.4.2
github.com/aws/aws-sdk-go-v2 v1.41.0
github.com/aws/aws-sdk-go-v2/config v1.32.6
Expand Down
Loading