diff --git a/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go b/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go index 47178c7b80..e4314c6f0d 100644 --- a/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go +++ b/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go @@ -93,7 +93,9 @@ func (cfg *Config) Setup() (cmd.Starter, error) { return nil, err } - if secureComms { + if secureComms || cfg.daemonConfig.SecureComms { + var inbounds, outbounds []string + ppssh.Singleton() host, port, err := net.SplitHostPort(cfg.listenAddr) if err != nil { @@ -103,15 +105,33 @@ func (cfg *Config) Setup() (cmd.Starter, error) { logger.Printf("Address %s is changed to 127.0.0.1:%s since secure-comms is enabled.", cfg.listenAddr, port) cfg.listenAddr = "127.0.0.1:" + port } - inbounds := append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...) - outbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...) // Create a Client that will approach the api-server-rest service at the podns // To obtain secrets from KBS, we approach the api-server-rest service which then approaches the CDH asking for a secret resource // the CDH than contact the KBS (possibly after approaching Attestation Agent for a token) and the KBS serves the requested key // The communication between the CDH (and Attestation Agent) and the KBS is performed via an SSH tunnel named "KBS" apic := apic.NewApiClient(API_SERVER_REST_PORT, cfg.podNamespace) - services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppssh.GetSecret(apic.GetKey), sshutil.SSHPORT)) + + ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(apic.GetKey)) + + if secureComms { + // CoCo in production + ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY) + ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY) + } else { + // Never here under CoCo in production + // Set secureComms using daemonConfig for testing + ppSecrets.SetKey(ppssh.WN_PUBLIC_KEY, cfg.daemonConfig.WnPublicKey) + ppSecrets.SetKey(ppssh.PP_PRIVATE_KEY, cfg.daemonConfig.PpPrivateKey) + } + + inbounds = append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...) + inbounds = append(inbounds, strings.Split(cfg.daemonConfig.SecureCommsInbounds, ",")...) + + outbounds = append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...) + outbounds = append(outbounds, strings.Split(cfg.daemonConfig.SecureCommsOutbounds, ",")...) + + services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppSecrets, sshutil.SSHPORT)) } else { if !disableTLS { cfg.tlsConfig = &tlsConfig diff --git a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go index 6bdbadd630..d2c760265e 100644 --- a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go +++ b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go @@ -9,9 +9,11 @@ import ( "fmt" "io" "os" + "strings" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/cmd" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/proxy" daemon "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork/tunneler/vxlan" @@ -28,7 +30,7 @@ const ( ) type daemonConfig struct { - serverConfig adaptor.ServerConfig + serverConfig cloud.ServerConfig networkConfig } @@ -86,12 +88,14 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) { } var ( - disableTLS bool - tlsConfig tlsutil.TLSConfig - secureComms bool - secureCommsInbounds string - secureCommsOutbounds string - secureCommsKbsAddr string + disableTLS bool + tlsConfig tlsutil.TLSConfig + secureComms bool + secureCommsInbounds string + secureCommsOutbounds string + secureCommsPpInbounds string + secureCommsPpOutbounds string + secureCommsKbsAddr string ) cmd.Parse(programName, os.Args[1:], func(flags *flag.FlagSet) { @@ -112,8 +116,10 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) { flags.BoolVar(&tlsConfig.SkipVerify, "tls-skip-verify", false, "Skip TLS certificate verification - use it only for testing") flags.BoolVar(&disableTLS, "disable-tls", false, "Disable TLS encryption - use it only for testing") flags.BoolVar(&secureComms, "secure-comms", false, "Use SSH to secure communication between cluster and peer pods") - flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "Inbound tags for secure communication tunnels") - flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "Outbound tags for secure communication tunnels") + flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "WN Inbound tags for secure communication tunnels") + flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "WN Outbound tags for secure communication tunnels") + flags.StringVar(&secureCommsPpInbounds, "secure-comms-pp-inbounds", "", "PP Inbound tags for secure communication tunnels") + flags.StringVar(&secureCommsPpOutbounds, "secure-comms-pp-outbounds", "", "PP Outbound tags for secure communication tunnels") flags.StringVar(&secureCommsKbsAddr, "secure-comms-kbs", "kbs-service.trustee-operator-system:8080", "Address of a Trustee Service for Secure-Comms") flags.DurationVar(&cfg.serverConfig.ProxyTimeout, "proxy-timeout", proxy.DefaultProxyTimeout, "Maximum timeout in minutes for establishing agent proxy connection") @@ -137,10 +143,15 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) { if err != nil { return nil, fmt.Errorf("secure comms failed to initialize KubeMgr: %w", err) } + if strings.EqualFold(secureCommsKbsAddr, "false") { + secureCommsKbsAddr = "" + } cfg.serverConfig.SecureComms = true cfg.serverConfig.SecureCommsInbounds = secureCommsInbounds cfg.serverConfig.SecureCommsOutbounds = secureCommsOutbounds + cfg.serverConfig.SecureCommsPpInbounds = secureCommsPpInbounds + cfg.serverConfig.SecureCommsPpOutbounds = secureCommsPpOutbounds cfg.serverConfig.SecureCommsKbsAddress = secureCommsKbsAddr } else { if !disableTLS { diff --git a/src/cloud-api-adaptor/docs/SecureComms.md b/src/cloud-api-adaptor/docs/SecureComms.md index be320eda61..e3eb296e11 100644 --- a/src/cloud-api-adaptor/docs/SecureComms.md +++ b/src/cloud-api-adaptor/docs/SecureComms.md @@ -27,6 +27,34 @@ Once the "Kubernetes Phase" SSH channel is established, Secure Comms connects th See [Secure Comms Architecture Slides](./SecureComms.pdf) for more details. +## Setup for for testing without Trustee (and for non-CoCo peerpods) + +### Deploy CAA +Use any of the option for installing CAA depending on the cloud driver used. + + +### Activate Secure-Comms feature from CAA side +Activate Secure-Comms from CAA side by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`. + +```sh +kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f - +``` + +You may also include additional Inbounds and Outbounds configurations to the Adaptor side using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. +You may also add Inbounds and Outbounds configurations to the Forwarder side using the `SECURE_COMMS_PP_INBOUNDS` and `SECURE_COMMS_PP_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel) + +Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example: +```sh +apiVersion: v1 +data: + ... + SECURE_COMMS: "true" + SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777" + SECURE_COMMS_PP_INBOUNDS: "KUBERNETES_PHASE:mytunnel:podns:6666" + SECURE_COMMS_KBS_ADDR: "false" + ... +``` + ## Setup for CoCo with Trustee ### Deploy CAA @@ -168,6 +196,4 @@ Alternatively, the client and server can be separately executed in independent t ## Future Plans - Add DeleteResource() support in KBS, KBC, api-server-rest, than cleanup resources added by Secure Comms to KBS whenever a Peer Pod fail to be created or when a Peer Pod is terminated. -- Add support for running the vxlan tunnel traffic via a Secure Comms SSH tunnel -- Add support for non-confidential Peer Pods which do not go via an Attestation Phase. - Add support for KBS identities allowing a Peer Pod to register its own identity in KBS and replace the current Secure Comms mechanism which delivers a private key to the Peer Pod via the KBS diff --git a/src/cloud-api-adaptor/entrypoint.sh b/src/cloud-api-adaptor/entrypoint.sh index 3810e6bb9f..eac62036ee 100755 --- a/src/cloud-api-adaptor/entrypoint.sh +++ b/src/cloud-api-adaptor/entrypoint.sh @@ -24,6 +24,8 @@ optionals+="" [[ "${SECURE_COMMS}" == "true" ]] && optionals+="-secure-comms " [[ "${SECURE_COMMS_INBOUNDS}" ]] && optionals+="-secure-comms-inbounds ${SECURE_COMMS_INBOUNDS} " [[ "${SECURE_COMMS_OUTBOUNDS}" ]] && optionals+="-secure-comms-outbounds ${SECURE_COMMS_OUTBOUNDS} " +[[ "${SECURE_COMMS_PP_INBOUNDS}" ]] && optionals+="-secure-comms-pp-inbounds ${SECURE_COMMS_PP_INBOUNDS} " +[[ "${SECURE_COMMS_PP_OUTBOUNDS}" ]] && optionals+="-secure-comms-pp-outbounds ${SECURE_COMMS_PP_OUTBOUNDS} " [[ "${SECURE_COMMS_KBS_ADDR}" ]] && optionals+="-secure-comms-kbs ${SECURE_COMMS_KBS_ADDR} " [[ "${PEERPODS_LIMIT_PER_NODE}" ]] && optionals+="-peerpods-limit-per-node ${PEERPODS_LIMIT_PER_NODE} " diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go index 2ff72a7db1..9e4412eb64 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go @@ -16,6 +16,7 @@ import ( "path/filepath" "strings" "sync" + "time" "github.com/containerd/containerd/pkg/cri/annotations" pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" @@ -27,6 +28,7 @@ import ( "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/wnssh" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util/tlsutil" provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers" putil "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util/cloudinit" @@ -48,6 +50,24 @@ type InitData struct { Data map[string]string `toml:"data,omitempty"` } +type ServerConfig struct { + TLSConfig *tlsutil.TLSConfig + SocketPath string + PauseImage string + PodsDir string + ForwarderPort string + ProxyTimeout time.Duration + Initdata string + EnableCloudConfigVerify bool + SecureComms bool + SecureCommsInbounds string + SecureCommsOutbounds string + SecureCommsPpInbounds string + SecureCommsPpOutbounds string + SecureCommsKbsAddress string + PeerPodsLimitPerNode int +} + var logger = log.New(log.Writer(), "[adaptor/cloud] ", log.LstdFlags|log.Lmsgprefix) func (s *cloudService) addSandbox(sid sandboxID, sandbox *sandbox) error { @@ -87,15 +107,20 @@ func (s *cloudService) removeSandbox(id sandboxID) error { } func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNode podnetwork.WorkerNode, - secureComms bool, secureCommsInbounds, secureCommsOutbounds, kbsAddress, podsDir, - daemonPort, initdata, sshport string) Service { + serverConfig *ServerConfig, sshport string) Service { var err error var sshClient *wnssh.SshClient - if secureComms { - inbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:0"}, strings.Split(secureCommsInbounds, ",")...) - outbounds := append([]string{"BOTH_PHASES:KBS:" + kbsAddress}, strings.Split(secureCommsOutbounds, ",")...) - sshClient, err = wnssh.InitSshClient(inbounds, outbounds, kbsAddress, sshport) + if serverConfig.SecureComms { + inbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:0"}, strings.Split(serverConfig.SecureCommsInbounds, ",")...) + + var outbounds []string + outbounds = append(outbounds, strings.Split(serverConfig.SecureCommsOutbounds, ",")...) + if len(serverConfig.SecureCommsKbsAddress) > 0 { + outbounds = append(outbounds, "BOTH_PHASES:KBS:"+serverConfig.SecureCommsKbsAddress) + } + + sshClient, err = wnssh.InitSshClient(inbounds, outbounds, serverConfig.SecureCommsKbsAddress, sshport) if err != nil { log.Fatalf("InitSshClient %v", err) } @@ -105,9 +130,7 @@ func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNo provider: provider, proxyFactory: proxyFactory, sandboxes: map[sandboxID]*sandbox{}, - podsDir: podsDir, - daemonPort: daemonPort, - initdata: initdata, + serverConfig: serverConfig, workerNode: workerNode, sshClient: sshClient, } @@ -224,7 +247,7 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r return nil, fmt.Errorf("failed to inspect netns %s: %w", netNSPath, err) } - podDir := filepath.Join(s.podsDir, string(sid)) + podDir := filepath.Join(s.serverConfig.PodsDir, string(sid)) if err := os.MkdirAll(podDir, os.ModePerm); err != nil { return nil, fmt.Errorf("creating a pod directory: %s, %w", podDir, err) } @@ -268,6 +291,22 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r daemonConfig.TLSServerKey = string(keyPEM) } + var ci *wnssh.SshClientInstance + + if s.sshClient != nil { + var ppPrivateKey []byte + ci, ppPrivateKey = s.sshClient.InitPP(context.Background(), string(sid)) + if ci == nil { + return nil, fmt.Errorf("failed sshClient.InitPP") + } + + daemonConfig.WnPublicKey = s.sshClient.GetWnPublicKey() + daemonConfig.PpPrivateKey = ppPrivateKey + daemonConfig.SecureCommsOutbounds = s.serverConfig.SecureCommsPpOutbounds + daemonConfig.SecureCommsInbounds = s.serverConfig.SecureCommsPpInbounds + daemonConfig.SecureComms = true + } + daemonJSON, err := json.MarshalIndent(daemonConfig, "", " ") if err != nil { return nil, fmt.Errorf("generating JSON data: %w", err) @@ -308,19 +347,19 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r logger.Printf("initdata in Pod annotation: %s", initdataStr) if initdataStr == "" { - logger.Printf("initdata in pod annotation is empty, use global initdata: %s", s.initdata) - initdataStr = s.initdata + logger.Printf("initdata in pod annotation is empty, use global initdata: %s", s.serverConfig.Initdata) + initdataStr = s.serverConfig.Initdata } if initdataStr != "" { decodedBytes, err := base64.StdEncoding.DecodeString(initdataStr) if err != nil { - return nil, fmt.Errorf("Error base64 decode initdata: %w", err) + return nil, fmt.Errorf("error base64 decode initdata: %w", err) } initdata := InitData{} err = toml.Unmarshal(decodedBytes, &initdata) if err != nil { - return nil, fmt.Errorf("Error unmarshalling initdata: %w", err) + return nil, fmt.Errorf("error unmarshalling initdata: %w", err) } cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{ @@ -330,14 +369,15 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r } sandbox := &sandbox{ - id: sid, - podName: pod, - podNamespace: namespace, - netNSPath: netNSPath, - agentProxy: agentProxy, - podNetwork: podNetworkConfig, - cloudConfig: cloudConfig, - spec: vmSpec, + id: sid, + podName: pod, + podNamespace: namespace, + netNSPath: netNSPath, + agentProxy: agentProxy, + podNetwork: podNetworkConfig, + cloudConfig: cloudConfig, + spec: vmSpec, + sshClientInst: ci, } if err := s.addSandbox(sid, sandbox); err != nil { @@ -381,24 +421,16 @@ func (s *cloudService) StartVM(ctx context.Context, req *pb.StartVMRequest) (res logger.Printf("created an instance %s for sandbox %s", instance.Name, sid) instanceIP := instance.IPs[0].String() - forwarderPort := s.daemonPort + forwarderPort := s.serverConfig.ForwarderPort if s.sshClient != nil { - ci := s.sshClient.InitPP(context.Background(), string(sid), instance.IPs) - if ci == nil { - return nil, fmt.Errorf("failed sshClient.InitPP") - } - - if err := ci.Start(); err != nil { + if err := sandbox.sshClientInst.Start(instance.IPs); err != nil { return nil, fmt.Errorf("failed SshClientInstance.Start: %w", err) } // Set agentProxy instanceIP = "127.0.0.1" - forwarderPort = ci.GetPort("KATAAGENT") - - // Set ci in sandbox - sandbox.sshClientInst = ci + forwarderPort = sandbox.sshClientInst.GetPort("KATAAGENT") } if err := s.workerNode.Setup(sandbox.netNSPath, instance.IPs, sandbox.podNetwork); err != nil { diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go index a8030ef72c..0b829a4505 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go @@ -117,7 +117,14 @@ func TestCloudService(t *testing.T) { podsDir: dir, } - s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, false, "", "", "", dir, forwarder.DefaultListenPort, "", "") + cfg := &ServerConfig{ + SecureComms: false, + PodsDir: dir, + ForwarderPort: forwarder.DefaultListenPort, + } + + // false, "", "", "", "", "", dir, forwarder.DefaultListenPort, "" + s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, cfg, "") assert.NotNil(t, s) @@ -160,7 +167,12 @@ func TestCloudServiceWithSecureComms(t *testing.T) { // create a podvm gkc := test.NewGetKeyClient("9019") ctx2, cancel := context.WithCancel(context.Background()) - sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:9019"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7111"}, ppssh.GetSecret(gkc.GetKey), sshport) + + ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(gkc.GetKey)) + ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY) + ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY) + + sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:9019"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7111"}, ppSecrets, sshport) _ = sshServer.Start(ctx2) defer func() { cancel() @@ -172,7 +184,14 @@ func TestCloudServiceWithSecureComms(t *testing.T) { podsDir: dir, } - s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, true, "", "", "127.0.0.1:9009", dir, forwarder.DefaultListenPort, "", sshport) + cfg := &ServerConfig{ + SecureComms: true, + PodsDir: dir, + ForwarderPort: forwarder.DefaultListenPort, + SecureCommsKbsAddress: "127.0.0.1:9009", + } + + s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, cfg, sshport) assert.NotNil(t, s) diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go index 6d71b69339..e91fff0774 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go @@ -31,12 +31,10 @@ type cloudService struct { workerNode podnetwork.WorkerNode sandboxes map[sandboxID]*sandbox cond *sync.Cond - podsDir string - daemonPort string mutex sync.Mutex ppService *k8sops.PeerPodService - initdata string sshClient *wnssh.SshClient + serverConfig *ServerConfig } type sandboxID string diff --git a/src/cloud-api-adaptor/pkg/adaptor/server.go b/src/cloud-api-adaptor/pkg/adaptor/server.go index 5539777a5b..0eae22fef9 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/server.go +++ b/src/cloud-api-adaptor/pkg/adaptor/server.go @@ -10,7 +10,6 @@ import ( "os" "path/filepath" "sync" - "time" "github.com/containerd/ttrpc" pbHypervisor "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" @@ -21,7 +20,6 @@ import ( "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/vminfo" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/sshutil" - "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util/tlsutil" pbPodVMInfo "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/proto/podvminfo" provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers" ) @@ -33,22 +31,6 @@ const ( DefaultPodsDir = "/run/peerpod/pods" ) -type ServerConfig struct { - TLSConfig *tlsutil.TLSConfig - SocketPath string - PauseImage string - PodsDir string - ForwarderPort string - ProxyTimeout time.Duration - Initdata string - EnableCloudConfigVerify bool - SecureComms bool - SecureCommsInbounds string - SecureCommsOutbounds string - SecureCommsKbsAddress string - PeerPodsLimitPerNode int -} - type Server interface { Start(ctx context.Context) error Shutdown() error @@ -68,14 +50,12 @@ type server struct { PeerPodsLimitPerNode int } -func NewServer(provider provider.Provider, cfg *ServerConfig, workerNode podnetwork.WorkerNode) Server { +func NewServer(provider provider.Provider, cfg *cloud.ServerConfig, workerNode podnetwork.WorkerNode) Server { logger.Printf("server config: %#v", cfg) agentFactory := proxy.NewFactory(cfg.PauseImage, cfg.TLSConfig, cfg.ProxyTimeout) - cloudService := cloud.NewService(provider, agentFactory, workerNode, - cfg.SecureComms, cfg.SecureCommsInbounds, cfg.SecureCommsOutbounds, - cfg.SecureCommsKbsAddress, cfg.PodsDir, cfg.ForwarderPort, cfg.Initdata, sshutil.SSHPORT) + cloudService := cloud.NewService(provider, agentFactory, workerNode, cfg, sshutil.SSHPORT) vmInfoService := vminfo.NewService(cloudService) return &server{ diff --git a/src/cloud-api-adaptor/pkg/adaptor/server_test.go b/src/cloud-api-adaptor/pkg/adaptor/server_test.go index baa9fc095a..da51ae12b5 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/server_test.go +++ b/src/cloud-api-adaptor/pkg/adaptor/server_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/proxy" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork/tunneler" provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers" @@ -150,7 +151,7 @@ func newServer(t *testing.T, socketPath, podsDir string) Server { port := startAgentServer(t) provider := &mockProvider{} - serverConfig := &ServerConfig{ + serverConfig := &cloud.ServerConfig{ SocketPath: socketPath, PodsDir: podsDir, ForwarderPort: port, diff --git a/src/cloud-api-adaptor/pkg/adaptor/shim_test.go b/src/cloud-api-adaptor/pkg/adaptor/shim_test.go index 688973cc4c..264ee587c7 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/shim_test.go +++ b/src/cloud-api-adaptor/pkg/adaptor/shim_test.go @@ -17,6 +17,7 @@ import ( "testing" "time" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud" daemon "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder/interceptor" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork" @@ -86,7 +87,7 @@ func TestShim(t *testing.T) { workerNode = podnetwork.NewWorkerNode(t, "", 0, 0) } - serverConfig := &ServerConfig{ + serverConfig := &cloud.ServerConfig{ SocketPath: helperSocketPath, PodsDir: podsDir, ForwarderPort: port, diff --git a/src/cloud-api-adaptor/pkg/forwarder/forwarder.go b/src/cloud-api-adaptor/pkg/forwarder/forwarder.go index fd8a694d0b..d175cea859 100644 --- a/src/cloud-api-adaptor/pkg/forwarder/forwarder.go +++ b/src/cloud-api-adaptor/pkg/forwarder/forwarder.go @@ -42,6 +42,12 @@ type Config struct { TLSServerKey string `json:"tls-server-key,omitempty"` TLSServerCert string `json:"tls-server-cert,omitempty"` TLSClientCA string `json:"tls-client-ca,omitempty"` + + PpPrivateKey []byte `json:"sc-pp-prv,omitempty"` + WnPublicKey []byte `json:"sc-wn-pub,omitempty"` + SecureCommsInbounds string `json:"sc-inbounds,omitempty"` + SecureCommsOutbounds string `json:"sc-outbounds,omitempty"` + SecureComms bool `json:"sc,omitempty"` } type Daemon interface { diff --git a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go index 1707f6473e..48bc6221eb 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go +++ b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go @@ -5,7 +5,6 @@ import ( ) type PpSecrets struct { - keys []string secrets map[string][]byte getSecret GetSecret } @@ -14,44 +13,54 @@ type GetSecret func(name string) ([]byte, error) func NewPpSecrets(getSecret GetSecret) *PpSecrets { return &PpSecrets{ - keys: []string{}, secrets: make(map[string][]byte), getSecret: getSecret, } } -func (fs *PpSecrets) AddKey(key string) { - fs.keys = append(fs.keys, key) +func (sec *PpSecrets) AddKey(key string) { + if _, ok := sec.secrets[key]; ok { + return + } + sec.secrets[key] = nil } -func (fs *PpSecrets) GetKey(key string) []byte { - return fs.secrets[key] +func (sec *PpSecrets) GetKey(key string) []byte { + return sec.secrets[key] } -func (fs *PpSecrets) Go() { - sleeptime := time.Duration(1) +func (sec *PpSecrets) SetKey(key string, keydata []byte) { + sec.secrets[key] = keydata +} - for len(fs.keys) > 0 { - key := fs.keys[0] - logger.Printf("PpSecrets obtaining key %s", key) +func (sec *PpSecrets) Go() { + sleeptime := time.Duration(1) - data, err := fs.getSecret(key) - if err == nil && len(data) > 0 { - logger.Printf("PpSecrets %s success", key) - fs.secrets[key] = data - fs.keys = fs.keys[1:] + for key, keydata := range sec.secrets { + if keydata != nil { continue } - if err != nil { - logger.Printf("PpSecrets %s getSecret err: %v", key, err) - } else { - logger.Printf("PpSecrets %s getSecret returned an empty secret", key) - } + logger.Printf("PpSecrets obtaining key %s", key) + + // loop until we get a valid key + for { + keydata, err := sec.getSecret(key) + if err == nil && len(keydata) > 0 { + logger.Printf("PpSecrets %s success", key) + sec.secrets[key] = keydata + break + } + if err != nil { + logger.Printf("PpSecrets %s getSecret err: %v", key, err) + } else { + logger.Printf("PpSecrets %s getSecret returned an empty secret", key) + } - time.Sleep(sleeptime * time.Second) - sleeptime *= 2 - if sleeptime > 30 { - sleeptime = 30 + time.Sleep(sleeptime * time.Second) + sleeptime *= 2 + if sleeptime > 30 { + sleeptime = 30 + } } } } diff --git a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go index d13a243836..08feb291d4 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go +++ b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go @@ -30,7 +30,7 @@ type SshServer struct { outbounds sshproxy.Outbounds wg sync.WaitGroup readyCh chan struct{} - getSecret GetSecret + ppSecrets *PpSecrets sshport string listener net.Listener ctx context.Context @@ -42,9 +42,9 @@ type SshServer struct { // Structure of an inbound tag: "::" // Structure of an outbound tag: ":::" // Phase may be "A" (Attestation), "K" (Kubernetes), or "B" (Both) -func NewSshServer(inbound_strings, outbounds_strings []string, getSecret GetSecret, sshport string) *SshServer { +func NewSshServer(inbound_strings, outbounds_strings []string, ppSecrets *PpSecrets, sshport string) *SshServer { s := &SshServer{ - getSecret: getSecret, + ppSecrets: ppSecrets, sshport: sshport, readyCh: make(chan struct{}), } @@ -119,11 +119,8 @@ func (s *SshServer) attestationPhase() *ssh.ServerConfig { for ctx.Err() == nil { logger.Printf("Attestation phase: getting keys from KBS\n") - ppSecrets := NewPpSecrets(s.getSecret) - ppSecrets.AddKey(WN_PUBLIC_KEY) - ppSecrets.AddKey(PP_PRIVATE_KEY) - ppSecrets.Go() // wait for the keys - config, err := initKubernetesPhaseSshConfig(ppSecrets) + s.ppSecrets.Go() // wait for the keys + config, err := initKubernetesPhaseSshConfig(s.ppSecrets) if err == nil { logger.Printf("Attestation phase: InitKubernetesPhaseSshConfig is ready\n") peer.Upgrade() diff --git a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go index c6746a8e39..bee6f6b6db 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go +++ b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go @@ -183,7 +183,12 @@ func TestPpssh(t *testing.T) { // Forwarder Initialization ctx := context.Background() ctx, cancel := context.WithCancel(ctx) - sshServer := NewSshServer([]string{"BOTH_PHASES:KBS:9002"}, []string{"KUBERNETES_PHASE:ABC:127.0.0.1:7105"}, GetSecret(getKey), sshport) + + ppSecrets := NewPpSecrets(GetSecret(getKey)) + ppSecrets.AddKey(WN_PUBLIC_KEY) + ppSecrets.AddKey(PP_PRIVATE_KEY) + + sshServer := NewSshServer([]string{"BOTH_PHASES:KBS:9002"}, []string{"KUBERNETES_PHASE:ABC:127.0.0.1:7105"}, ppSecrets, sshport) _ = sshServer.Start(ctx) clientSshPeer, conn := getAttestationClient(t, sshport) clientSshPeer.AddTags(inbounds, outbounds) diff --git a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go index d252d609f1..87a9628056 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go +++ b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go @@ -25,6 +25,7 @@ type SshClient struct { inboundStrings []string outboundStrings []string sshport string + wnPublicKey []byte } type SshClientInstance struct { @@ -72,22 +73,25 @@ func InitSshClient(inbound_strings, outbound_strings []string, kbsAddress string return nil, fmt.Errorf("unable to parse private key: %v", err) } - kbscPrivateKey, _, err := kubemgr.KubeMgr.ReadSecret(sshutil.KBS_CLIENT_SECRET) - if err != nil { - return nil, fmt.Errorf("failed to read KBS client secret: %w", err) - } + var kc *KbsClient + if len(kbsAddress) > 0 { + kbscPrivateKey, _, err := kubemgr.KubeMgr.ReadSecret(sshutil.KBS_CLIENT_SECRET) + if err != nil { + return nil, fmt.Errorf("failed to read KBS client secret: %w", err) + } - kc := InitKbsClient(kbsAddress) - err = kc.SetPemSecret(kbscPrivateKey) - if err != nil { - return nil, fmt.Errorf("KbsClient - %v", err) - } + kc = InitKbsClient(kbsAddress) + err = kc.SetPemSecret(kbscPrivateKey) + if err != nil { + return nil, fmt.Errorf("KbsClient - %v", err) + } - wnSecretPath := "default/sshclient/publicKey" - logger.Printf("Updating KBS with secret for: %s", wnSecretPath) - err = kc.PostResource(wnSecretPath, wnPublicKey) - if err != nil { - return nil, fmt.Errorf("failed to PostResource WN Secret: %v", err) + wnSecretPath := "default/sshclient/publicKey" + logger.Printf("Updating KBS with secret for: %s", wnSecretPath) + err = kc.PostResource(wnSecretPath, wnPublicKey) + if err != nil { + return nil, fmt.Errorf("failed to PostResource WN Secret: %v", err) + } } sshClient := &SshClient{ @@ -96,6 +100,7 @@ func InitSshClient(inbound_strings, outbound_strings []string, kbsAddress string inboundStrings: inbound_strings, outboundStrings: outbound_strings, sshport: sshport, + wnPublicKey: wnPublicKey, } return sshClient, nil @@ -123,9 +128,13 @@ func (ci *SshClientInstance) DisconnectPP(sid string) { kubemgr.KubeMgr.DeleteSecret(PpSecretName(sid)) } -func (c *SshClient) InitPP(ctx context.Context, sid string, ipAddr []netip.Addr) *SshClientInstance { +func (c *SshClient) GetWnPublicKey() []byte { + return c.wnPublicKey +} + +func (c *SshClient) InitPP(ctx context.Context, sid string) (ci *SshClientInstance, ppPrivateKey []byte) { // Create peerPod Secret named peerPodId - var ppPublicKey, ppPrivateKey []byte + var ppPublicKey []byte var err error var kubernetesPhase bool @@ -136,43 +145,39 @@ func (c *SshClient) InitPP(ctx context.Context, sid string, ipAddr []netip.Addr) ppPrivateKey, ppPublicKey, err = kubemgr.KubeMgr.CreateSecret(PpSecretName(sid)) if err != nil { logger.Printf("Failed to create PP secret: %v", err) - return nil + return } } else { // we already have a store secret for this PP kubernetesPhase = true } - // >>> Update the KBS about the SID's Secret !!! <<< - sidSecretPath := fmt.Sprintf("default/pp-%s/privateKey", sid) - logger.Printf("Updating KBS with secret for: %s", sidSecretPath) - err = c.kc.PostResource(sidSecretPath, ppPrivateKey) - if err != nil { - logger.Printf("Failed to PostResource PP Secret: %v", err) - return nil - } + if c.kc != nil { + // >>> Update the KBS about the SID's Secret !!! <<< + sidSecretPath := fmt.Sprintf("default/pp-%s/privateKey", sid) + logger.Printf("Updating KBS with secret for: %s", sidSecretPath) + err = c.kc.PostResource(sidSecretPath, ppPrivateKey) + if err != nil { + logger.Printf("Failed to PostResource PP Secret: %v", err) + return + } + } var ppSshPublicKeyBytes []byte if len(ppPublicKey) > 0 { ppSshPublicKey, _, _, _, err := ssh.ParseAuthorizedKey(ppPublicKey) if err != nil { logger.Printf("Unable to ParseAuthorizedKey serverPublicKey: %v", err) - return nil + return } ppSshPublicKeyBytes = ppSshPublicKey.Marshal() } - ppAddr := make([]string, len(ipAddr)) - for i, ip := range ipAddr { - ppAddr[i] = ip.String() + ":" + c.sshport - } - ctx, cancel := context.WithCancel(ctx) - ci := &SshClientInstance{ + ci = &SshClientInstance{ sid: sid, ppPublicKey: ppSshPublicKeyBytes, - ppAddr: ppAddr, sshClient: c, ctx: ctx, cancel: cancel, @@ -188,10 +193,17 @@ func (c *SshClient) InitPP(ctx context.Context, sid string, ipAddr []netip.Addr) logger.Fatalf("Failed to parse outbound tag %v: %v", c.outboundStrings, err) } - return ci + return } -func (ci *SshClientInstance) Start() error { +func (ci *SshClientInstance) Start(ipAddr []netip.Addr) error { + ppAddr := make([]string, len(ipAddr)) + for i, ip := range ipAddr { + ppAddr[i] = ip.String() + ":" + ci.sshClient.sshport + } + + ci.ppAddr = ppAddr + if !ci.kubernetesPhase { // Attestation phase logger.Println("Attestation phase: starting") diff --git a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go index 64b6d3304b..ecfb3f440b 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go +++ b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go @@ -32,7 +32,7 @@ func TestSshProxyReverseKBS(t *testing.T) { ////////// CAA StartVM ipAddr, _ := netip.ParseAddr("127.0.0.1") // ipAddr of the VM ipAddrs := []netip.Addr{ipAddr} - ci := sshClient.InitPP(context.Background(), "sid", ipAddrs) + ci, _ := sshClient.InitPP(context.Background(), "sid") if ci == nil { log.Fatalf("failed InitiatePeerPodTunnel") } @@ -46,12 +46,17 @@ func TestSshProxyReverseKBS(t *testing.T) { // create a podvm gkc := test.NewGetKeyClient("7030") ctx2, cancel2 := context.WithCancel(context.Background()) - sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7030", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7121"}, ppssh.GetSecret(gkc.GetKey), sshport) + + ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(gkc.GetKey)) + ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY) + ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY) + + sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7030", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7121"}, ppSecrets, sshport) _ = sshServer.Start(ctx2) // Forwarder Initialization - if err := ci.Start(); err != nil { + if err := ci.Start(ipAddrs); err != nil { log.Fatalf("failed ci.Start: %s", err) } diff --git a/src/cloud-api-adaptor/test/securecomms/ppssh.go b/src/cloud-api-adaptor/test/securecomms/ppssh.go index b9a15d82d4..337f3f39e0 100644 --- a/src/cloud-api-adaptor/test/securecomms/ppssh.go +++ b/src/cloud-api-adaptor/test/securecomms/ppssh.go @@ -20,7 +20,12 @@ func PP() { // Forwarder Initialization ctx := context.Background() ctx, cancel := context.WithCancel(ctx) - sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7000", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7131"}, ppssh.GetSecret(getKey), sshutil.SSHPORT) + + ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(getKey)) + ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY) + ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY) + + sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7000", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7131"}, ppSecrets, sshutil.SSHPORT) _ = sshServer.Start(ctx) time.Sleep(1 * time.Minute) cancel() diff --git a/src/cloud-api-adaptor/test/securecomms/wnssh.go b/src/cloud-api-adaptor/test/securecomms/wnssh.go index 026125e032..406d78c48b 100644 --- a/src/cloud-api-adaptor/test/securecomms/wnssh.go +++ b/src/cloud-api-adaptor/test/securecomms/wnssh.go @@ -30,7 +30,7 @@ func WN() bool { ipAddr, _ := netip.ParseAddr("127.0.0.1") // ipAddr of the VM ipAddrs := []netip.Addr{ipAddr} ctx := context.Background() - ci := sshClient.InitPP(ctx, "sid", ipAddrs) + ci, _ := sshClient.InitPP(ctx, "sid") if ci == nil { log.Fatalf("failed InitiatePeerPodTunnel") } @@ -41,7 +41,7 @@ func WN() bool { log.Fatalf("failed find port") } - if err := ci.Start(); err != nil { + if err := ci.Start(ipAddrs); err != nil { log.Fatalf("failed ci.Start: %s", err) }