Add support for etcd encryption in vSphere (aws#6831)

* Add support for etcd encryption in vSphere * Add e2e test for vsphere etcd encryption
ahreehong · Oct 17, 2023 · 3d5bbb1 · 3d5bbb1
1 parent 51636f0
commit 3d5bbb1
Show file tree

Hide file tree

Showing 8 changed files with 1,038 additions and 1 deletion.
diff --git a/cmd/integration_test/build/buildspecs/vsphere-test-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/vsphere-test-eks-a-cli.yml
@@ -77,6 +77,13 @@ env:
     T_PRIVATE_REGISTRY_MIRROR_CA_CERT: "harbor-registry-data:authenticated_caCert"
     T_NTP_SERVERS: "ntp:servers"
     T_AWS_IAM_ROLE_ARN: "aws-iam-auth-role:ec2_role_arn"
+    T_IRSA_S3_BUCKET: "etcd-encryption:irsa_s3_bucket"
+    T_KMS_IAM_ROLE: "etcd-encryption:kms_iam_role_arn"
+    T_KMS_IMAGE: "etcd-encryption:kms_image"
+    T_KMS_KEY_ARN: "etcd-encryption:kms_key_arn"
+    T_KMS_KEY_REGION: "etcd-encryption:region"
+    T_KMS_SOCKET: "etcd-encryption:socket"
+    T_SSH_PRIVATE_KEY: "vsphere_ci_beta_connection:base64_encoded_ssh_private_key"
 phases:
   pre_build:
     commands:

diff --git a/pkg/providers/vsphere/config/template-cp.yaml b/pkg/providers/vsphere/config/template-cp.yaml
@@ -182,6 +182,16 @@ spec:
           name: awsiamcert
           readOnly: false
 {{- end}}
+{{- if .encryptionProviderConfig }}
+        - hostPath: /etc/kubernetes/enc
+          mountPath: /etc/kubernetes/enc
+          name: encryption-config
+          readOnly: false
+        - hostPath: /var/run/kmsplugin/
+          mountPath: /var/run/kmsplugin/
+          name: kms-plugin
+          readOnly: false
+{{- end }}
       controllerManager:
         extraArgs:
           cloud-provider: external
@@ -213,6 +223,12 @@ spec:
       certificatesDir: /var/lib/kubeadm/pki
 {{- end }}
     files:
+{{- if .encryptionProviderConfig }}
+    - content: |
+{{ .encryptionProviderConfig | indent 8}}
+      owner: root:root
+      path: /etc/kubernetes/enc/encryption-config.yaml
+{{- end }}
     - content: |
         apiVersion: v1
         kind: Pod

diff --git a/pkg/providers/vsphere/template.go b/pkg/providers/vsphere/template.go
@@ -134,6 +134,7 @@ func buildTemplateMapCP(
 	apiServerExtraArgs := clusterapi.OIDCToExtraArgs(clusterSpec.OIDCConfig).
 		Append(clusterapi.AwsIamAuthExtraArgs(clusterSpec.AWSIamConfig)).
 		Append(clusterapi.PodIAMAuthExtraArgs(clusterSpec.Cluster.Spec.PodIAMConfig)).
+		Append(clusterapi.EtcdEncryptionExtraArgs(clusterSpec.Cluster.Spec.EtcdEncryption)).
 		Append(sharedExtraArgs)
 	controllerManagerExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs().
 		Append(clusterapi.NodeCIDRMaskExtraArgs(&clusterSpec.Cluster.Spec.ClusterNetwork))
@@ -324,6 +325,14 @@ func buildTemplateMapCP(
 		values["bottlerocketSettings"] = brSettings
 	}
 
+	if clusterSpec.Cluster.Spec.EtcdEncryption != nil && len(*clusterSpec.Cluster.Spec.EtcdEncryption) != 0 {
+		conf, err := common.GenerateKMSEncryptionConfiguration(clusterSpec.Cluster.Spec.EtcdEncryption)
+		if err != nil {
+			return nil, err
+		}
+		values["encryptionProviderConfig"] = conf
+	}
+
 	return values, nil
 }
 

diff --git a/pkg/providers/vsphere/testdata/cluster_ubuntu_etcd_encryption.yaml b/pkg/providers/vsphere/testdata/cluster_ubuntu_etcd_encryption.yaml
@@ -0,0 +1,132 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: Cluster
+metadata:
+  name: test
+  namespace: test-namespace
+spec:
+  controlPlaneConfiguration:
+    count: 3
+    endpoint:
+      host: 1.2.3.4
+    machineGroupRef:
+      name: test-cp
+      kind: VSphereMachineConfig
+  kubernetesVersion: "1.21"
+  etcdEncryption:
+  - providers:
+    - kms:
+        name: config1
+        socketListenAddress: unix:///var/run/kmsplugin/socket1-new.sock
+    - kms:
+        name: config2
+        socketListenAddress: unix:///var/run/kmsplugin/socket1-old.sock
+    resources:
+    - secrets
+    - resource1.anywhere.eks.amazonsaws.com
+  - providers:
+    - kms:
+        name: config3
+        socketListenAddress: unix:///var/run/kmsplugin/socket2-new.sock
+    - kms:
+        name: config4
+        socketListenAddress: unix:///var/run/kmsplugin/socket2-old.sock
+    resources:
+    - configmaps
+    - resource2.anywhere.eks.amazonsaws.com
+  workerNodeGroupConfigurations:
+    - count: 3
+      machineGroupRef:
+        name: test-wn
+        kind: VSphereMachineConfig
+      name: md-0
+  externalEtcdConfiguration:
+    count: 3
+    machineGroupRef:
+      name: test-etcd
+      kind: VSphereMachineConfig
+  datacenterRef:
+    kind: VSphereDatacenterConfig
+    name: test
+  clusterNetwork:
+    cni: "cilium"
+    pods:
+      cidrBlocks:
+        - 192.168.0.0/16
+    services:
+      cidrBlocks:
+        - 10.96.0.0/12
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: VSphereMachineConfig
+metadata:
+  name: test-cp
+  namespace: test-namespace
+spec:
+  diskGiB: 25
+  cloneMode: linkedClone
+  datastore: "/SDDC-Datacenter/datastore/WorkloadDatastore"
+  folder: "/SDDC-Datacenter/vm"
+  memoryMiB: 8192
+  numCPUs: 2
+  osFamily: ubuntu
+  resourcePool: "*/Resources"
+  storagePolicyName: "vSAN Default Storage Policy"
+  template: "/SDDC-Datacenter/vm/Templates/ubuntu-2004-kube-v1.21.2"
+  users:
+    - name: capv
+      sshAuthorizedKeys:
+        - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ== [email protected]"
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: VSphereMachineConfig
+metadata:
+  name: test-wn
+  namespace: test-namespace
+spec:
+  diskGiB: 25
+  cloneMode: linkedClone
+  datastore: "/SDDC-Datacenter/datastore/WorkloadDatastore"
+  folder: "/SDDC-Datacenter/vm"
+  memoryMiB: 4096
+  numCPUs: 3
+  osFamily: ubuntu
+  resourcePool: "*/Resources"
+  storagePolicyName: "vSAN Default Storage Policy"
+  template: "/SDDC-Datacenter/vm/Templates/ubuntu-2004-kube-v1.21.2"
+  users:
+    - name: capv
+      sshAuthorizedKeys:
+        - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ== [email protected]"
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: VSphereMachineConfig
+metadata:
+  name: test-etcd
+  namespace: test-namespace
+spec:
+  diskGiB: 25
+  cloneMode: linkedClone
+  datastore: "/SDDC-Datacenter/datastore/WorkloadDatastore"
+  folder: "/SDDC-Datacenter/vm"
+  memoryMiB: 4096
+  numCPUs: 3
+  osFamily: ubuntu
+  resourcePool: "*/Resources"
+  storagePolicyName: "vSAN Default Storage Policy"
+  template: "/SDDC-Datacenter/vm/Templates/ubuntu-2004-kube-v1.21.2"
+  users:
+    - name: capv
+      sshAuthorizedKeys:
+        - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ== [email protected]"
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: VSphereDatacenterConfig
+metadata:
+  name: test
+  namespace: test-namespace
+spec:
+  datacenter: "SDDC-Datacenter"
+  network: "/SDDC-Datacenter/network/sddc-cgw-network-1"
+  server: "vsphere_server"
+  thumbprint: "ABCDEFG"
+  insecure: false