SecureComms: E2e test SecureComms without KBS

Add support for e2e testing SecureComms without KBS

Signed-off-by: David Hadas <[email protected]>

Author: davidhadas

src/cloud-api-adaptor/libvirt/config_libvirt.sh

@@ -92,7 +92,7 @@ installK8sclis() {
     fi
 }
 
-TEST_E2E_SECURE_COMMS=${TEST_E2E_SECURE_COMMS:-none}.
+TEST_E2E_SECURE_COMMS=${TEST_E2E_SECURE_COMMS:-none}
 echo "SECURE_COMMS is ${TEST_E2E_SECURE_COMMS}"
 
 echo "Installing Go..."
@@ -123,10 +123,20 @@ echo "libvirt_ssh_key_file=\"id_rsa\"" >> libvirt.properties
 echo "CLUSTER_NAME=\"peer-pods\"" >> libvirt.properties
 
 # switch to the appropriate e2e test and add configs to libvirt.properties as needed
-case $TEST_E2E_SECURE_COMMS in
+case ${TEST_E2E_SECURE_COMMS} in
+
+  withoutKbs)
+    echo "processing withoutKbs"
+    echo "SECURE_COMMS=\"true\"" >> libvirt.properties
+    echo "SECURE_COMMS_NO_TRUSTEE=\"true\"" >> libvirt.properties
+    echo "INITDATA=\"\"" >> libvirt.properties
+    ;;
 
   *)
     echo "processing none"
     echo "SECURE_COMMS=\"false\"" >> libvirt.properties
     ;;
 esac
+
+echo "Content of libvirt.properties"
+cat libvirt.properties

src/cloud-api-adaptor/libvirt/e2e_matrix_libvirt.json

@@ -1,6 +1,6 @@
 {
     "container_runtime": ["containerd", "crio"],
-    "secure_comms": ["none"],
+    "secure_comms": ["none", "withoutKbs"],
     "os": ["ubuntu"],
     "provider": ["generic"],
     "arch": ["amd64"]

src/cloud-api-adaptor/test/e2e/assessment_helpers.go

@@ -292,6 +292,21 @@ func VerifyAlternateImage(ctx context.Context, t *testing.T, client klient.Clien
 	return nil
 }
 
+func VerifySecureCommsActivated(ctx context.Context, t *testing.T, client klient.Client, pod *v1.Pod) error {
+	nodeName, err := GetNodeNameFromPod(ctx, client, pod)
+	if err != nil {
+		return fmt.Errorf("VerifySecureCommsConnected: GetNodeNameFromPod failed with %v", err)
+	}
+
+	expectedSuccessMessage := "Using PP SecureComms"
+	err = VerifyCaaPodLogContains(ctx, t, client, nodeName, expectedSuccessMessage)
+	if err != nil {
+		return fmt.Errorf("VerifySecureCommsConnected: failed: %v", err)
+	}
+	t.Logf("PodVM was brought up using SecureComms")
+	return nil
+}
+
 func VerifyCaaPodLogContains(ctx context.Context, t *testing.T, client klient.Client, nodeName, expected string) error {
 	caaPod, err := getCaaPod(ctx, client, t, nodeName)
 	if err != nil {

src/cloud-api-adaptor/test/e2e/assessment_runner.go

@@ -48,6 +48,8 @@ type ExtraPod struct {
 	testCommands         []TestCommand
 }
 
+var testCase_secureComms_isActive bool
+
 type TestCase struct {
 	testing                     *testing.T
 	testEnv                     env.Environment
@@ -420,6 +422,13 @@ func (tc *TestCase) Run() {
 						t.Errorf("VerifyAlternateImage failed: %v", err)
 					}
 				}
+
+				if testCase_secureComms_isActive {
+					err := VerifySecureCommsActivated(ctx, t, client, tc.pod)
+					if err != nil {
+						t.Errorf("VerifySecureCommsActivated failed: %v", err)
+					}
+				}
 			}
 
 			if tc.extraPods != nil {

src/cloud-api-adaptor/test/e2e/main_test.go

@@ -123,6 +123,10 @@ func TestMain(m *testing.M) {
 
 		// Get properties
 		props := provisioner.GetProperties(ctx, cfg)
+		if props["SECURE_COMMS"] == "true" {
+			testCase_secureComms_isActive = true
+			log.Info("Do setup secureComms is active")
+		}
 
 		// Set CONTAINER_RUNTIME env variable if present in the properties
 		// Default value is containerd.

src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go

@@ -23,17 +23,21 @@ const AlternateVolumeName = "another-podvm-base.qcow2"
 
 // LibvirtProvisioner implements the CloudProvisioner interface for Libvirt.
 type LibvirtProvisioner struct {
-	conn             *libvirt.Connect // Libvirt connection
-	containerRuntime string           // Name of the container runtime
-	network          string           // Network name
-	ssh_key_file     string           // SSH key file used to connect to Libvirt
-	storage          string           // Storage pool name
-	uri              string           // Libvirt URI
-	wd               string           // libvirt's directory path on this repository
-	volumeName       string           // Podvm volume name
-	clusterName      string           // Cluster name
-	tunnelType       string           // Tunnel Type
-	vxlanPort        string           // VXLAN port number
+	conn                    *libvirt.Connect // Libvirt connection
+	containerRuntime        string           // Name of the container runtime
+	network                 string           // Network name
+	ssh_key_file            string           // SSH key file used to connect to Libvirt
+	storage                 string           // Storage pool name
+	uri                     string           // Libvirt URI
+	wd                      string           // libvirt's directory path on this repository
+	volumeName              string           // Podvm volume name
+	clusterName             string           // Cluster name
+	tunnelType              string           // Tunnel Type
+	vxlanPort               string           // VXLAN port number
+	secure_comms            string           // Activate CAA SECURE_COMMS
+	secure_comms_no_trustee string           // Deactivate Trustee mode in SECURE_COMMS
+	secure_comms_kbs_addr   string           // KBS URL
+	initdata                string           // InitData
 }
 
 // LibvirtInstallOverlay implements the InstallOverlay interface
@@ -95,19 +99,47 @@ func NewLibvirtProvisioner(properties map[string]string) (pv.CloudProvisioner, e
 		vxlanPort = properties["vxlan_port"]
 	}
 
+	secure_comms := "false"
+	if properties["SECURE_COMMS"] != "" {
+		secure_comms = properties["SECURE_COMMS"]
+	}
+
+	log.Tracef("NewLibvirtProvisioner properties %v", properties)
+
+	log.Tracef("NewLibvirtProvisioner secure_comms %s", secure_comms)
+
+	secure_comms_kbs_addr := ""
+	if properties["SECURE_COMMS_KBS_ADDR"] != "" {
+		secure_comms_kbs_addr = properties["SECURE_COMMS_KBS_ADDR"]
+	}
+
+	secure_comms_no_trustee := "false"
+	if properties["SECURE_COMMS_NO_TRUSTEE"] != "" {
+		secure_comms_no_trustee = properties["SECURE_COMMS_NO_TRUSTEE"]
+	}
+
+	initdata := ""
+	if properties["INITDATA"] != "" {
+		initdata = properties["INITDATA"]
+	}
+
 	// TODO: Check network and storage are not nil?
 	return &LibvirtProvisioner{
-		conn:             conn,
-		containerRuntime: properties["container_runtime"],
-		network:          network,
-		ssh_key_file:     ssh_key_file,
-		storage:          storage,
-		uri:              uri,
-		wd:               wd,
-		volumeName:       vol_name,
-		clusterName:      clusterName,
-		tunnelType:       tunnelType,
-		vxlanPort:        vxlanPort,
+		conn:                    conn,
+		containerRuntime:        properties["container_runtime"],
+		network:                 network,
+		ssh_key_file:            ssh_key_file,
+		storage:                 storage,
+		uri:                     uri,
+		wd:                      wd,
+		volumeName:              vol_name,
+		clusterName:             clusterName,
+		tunnelType:              tunnelType,
+		vxlanPort:               vxlanPort,
+		secure_comms:            secure_comms,
+		secure_comms_kbs_addr:   secure_comms_kbs_addr,
+		secure_comms_no_trustee: secure_comms_no_trustee,
+		initdata:                initdata,
 	}, nil
 }
 
@@ -212,14 +244,18 @@ func (l *LibvirtProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config)
 
 func (l *LibvirtProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string {
 	return map[string]string{
-		"CONTAINER_RUNTIME": l.containerRuntime,
-		"network":           l.network,
-		"podvm_volume":      l.volumeName,
-		"ssh_key_file":      l.ssh_key_file,
-		"storage":           l.storage,
-		"uri":               l.uri,
-		"tunnel_type":       l.tunnelType,
-		"vxlan_port":        l.vxlanPort,
+		"CONTAINER_RUNTIME":       l.containerRuntime,
+		"network":                 l.network,
+		"podvm_volume":            l.volumeName,
+		"ssh_key_file":            l.ssh_key_file,
+		"storage":                 l.storage,
+		"uri":                     l.uri,
+		"tunnel_type":             l.tunnelType,
+		"vxlan_port":              l.vxlanPort,
+		"SECURE_COMMS":            l.secure_comms,
+		"SECURE_COMMS_KBS_ADDR":   l.secure_comms_kbs_addr,
+		"SECURE_COMMS_NO_TRUSTEE": l.secure_comms_no_trustee,
+		"INITDATA":                l.initdata,
 	}
 }
 
@@ -326,14 +362,17 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config,
 
 	// Mapping the internal properties to ConfigMapGenerator properties and their default values.
 	mapProps := map[string][2]string{
-		"network":      {"default", "LIBVIRT_NET"},
-		"storage":      {"default", "LIBVIRT_POOL"},
-		"pause_image":  {"", "PAUSE_IMAGE"},
-		"podvm_volume": {"", "LIBVIRT_VOL_NAME"},
-		"uri":          {"qemu+ssh://[email protected]/system?no_verify=1", "LIBVIRT_URI"},
-		"tunnel_type":  {"", "TUNNEL_TYPE"},
-		"vxlan_port":   {"", "VXLAN_PORT"},
-		"INITDATA":     {"", "INITDATA"},
+		"network":                 {"default", "LIBVIRT_NET"},
+		"storage":                 {"default", "LIBVIRT_POOL"},
+		"pause_image":             {"", "PAUSE_IMAGE"},
+		"podvm_volume":            {"", "LIBVIRT_VOL_NAME"},
+		"uri":                     {"qemu+ssh://[email protected]/system?no_verify=1", "LIBVIRT_URI"},
+		"tunnel_type":             {"", "TUNNEL_TYPE"},
+		"vxlan_port":              {"", "VXLAN_PORT"},
+		"INITDATA":                {"", "INITDATA"},
+		"SECURE_COMMS":            {"", "SECURE_COMMS"},
+		"SECURE_COMMS_NO_TRUSTEE": {"", "SECURE_COMMS_NO_TRUSTEE"},
+		"SECURE_COMMS_KBS_ADDR":   {"", "SECURE_COMMS_KBS_ADDR"},
 	}
 
 	for k, v := range mapProps {

src/cloud-api-adaptor/test/provisioner/provision.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/BurntSushi/toml"
@@ -220,7 +221,6 @@ func (p *CloudAPIAdaptor) Delete(ctx context.Context, cfg *envconf.Config) error
 		wait.WithTimeout(time.Minute*1)); err != nil {
 		return err
 	}
-
 	return nil
 }
 
@@ -295,7 +295,17 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props
 		}
 	}
 
-	fmt.Printf("Wait for the %s runtimeclass be created\n", p.runtimeClass.GetName())
+	log.Trace("CAA ConfigMap:\n")
+	caaConfigMap := exec.Command("kubectl", "get", "cm", "peer-pods-cm", "-n", "confidential-containers-system", "-o", "yaml")
+	caaConfigMap.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile()))
+	caaConfigMapOut := new(strings.Builder)
+	caaConfigMap.Stdout = caaConfigMapOut
+	if err = caaConfigMap.Run(); err != nil {
+		return err
+	}
+	log.Tracef("%v, CAA ConfigMap: \n%s", caaConfigMap, caaConfigMapOut.String())
+
+	log.Infof("Wait for the %s runtimeclass be created\n", p.runtimeClass.GetName())
 	if err = wait.For(conditions.New(resources).ResourcesFound(&nodev1.RuntimeClassList{Items: []nodev1.RuntimeClass{*p.runtimeClass}}),
 		wait.WithTimeout(time.Second*60)); err != nil {
 		return err