Skip to content

Commit c04060f

Browse files
davidhadasdavidhadas
committed
SecureComms: Add support daemonConfig
Support configuring the APF Secure Comms from the CAA side including: - WN public Key - PP private key - Activating Secure Comms - inbouns and outbounds of th PP This is useful for activating Secure Comms from the CAA and without Trustee. It can be used for Testing without producing dedicated podvms which activate Secure Comms and set Inbounds/Outbounds by default. It can also be used for non-Coco peerpods. Signed-off-by: David Hadas <[email protected]>
1 parent 2255e7e commit c04060f

File tree

19 files changed

+341
-148
lines changed

19 files changed

+341
-148
lines changed

‎src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,9 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
9393
return nil, err
9494
}
9595

96-
if secureComms {
96+
if secureComms || cfg.daemonConfig.SecureComms {
97+
var inbounds, outbounds []string
98+
9799
ppssh.Singleton()
98100
host, port, err := net.SplitHostPort(cfg.listenAddr)
99101
if err != nil {
@@ -103,15 +105,32 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
103105
logger.Printf("Address %s is changed to 127.0.0.1:%s since secure-comms is enabled.", cfg.listenAddr, port)
104106
cfg.listenAddr = "127.0.0.1:" + port
105107
}
106-
inbounds := append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...)
107-
outbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...)
108108

109109
// Create a Client that will approach the api-server-rest service at the podns
110110
// To obtain secrets from KBS, we approach the api-server-rest service which then approaches the CDH asking for a secret resource
111111
// the CDH than contact the KBS (possibly after approaching Attestation Agent for a token) and the KBS serves the requested key
112112
// The communication between the CDH (and Attestation Agent) and the KBS is performed via an SSH tunnel named "KBS"
113113
apic := apic.NewApiClient(API_SERVER_REST_PORT, cfg.kataAgentNamespace)
114-
services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppssh.GetSecret(apic.GetKey), sshutil.SSHPORT))
114+
115+
ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(apic.GetKey))
116+
117+
if secureComms {
118+
// CoCo in production
119+
ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY)
120+
ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY)
121+
inbounds = append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...)
122+
outbounds = append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...)
123+
124+
} else {
125+
// Never here under CoCo in production
126+
// Set secureComms using daemonConfig for testing
127+
ppSecrets.SetKey(ppssh.WN_PUBLIC_KEY, cfg.daemonConfig.WnPublicKey)
128+
ppSecrets.SetKey(ppssh.PP_PRIVATE_KEY, cfg.daemonConfig.PpPrivateKey)
129+
inbounds = append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(cfg.daemonConfig.SecureCommsInbounds, ",")...)
130+
outbounds = append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(cfg.daemonConfig.SecureCommsOutbounds, ",")...)
131+
}
132+
133+
services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppSecrets, sshutil.SSHPORT))
115134
} else {
116135
if !disableTLS {
117136
cfg.tlsConfig = &tlsConfig

‎src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212

1313
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/cmd"
1414
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor"
15+
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud"
1516
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/proxy"
1617
daemon "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder"
1718
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork/tunneler/vxlan"
@@ -28,7 +29,7 @@ const (
2829
)
2930

3031
type daemonConfig struct {
31-
serverConfig adaptor.ServerConfig
32+
serverConfig cloud.ServerConfig
3233
networkConfig
3334
}
3435

@@ -86,12 +87,14 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
8687
}
8788

8889
var (
89-
disableTLS bool
90-
tlsConfig tlsutil.TLSConfig
91-
secureComms bool
92-
secureCommsInbounds string
93-
secureCommsOutbounds string
94-
secureCommsKbsAddr string
90+
disableTLS bool
91+
tlsConfig tlsutil.TLSConfig
92+
secureComms bool
93+
secureCommsInbounds string
94+
secureCommsOutbounds string
95+
secureCommsPpInbounds string
96+
secureCommsPpOutbounds string
97+
secureCommsKbsAddr string
9598
)
9699

97100
cmd.Parse(programName, os.Args[1:], func(flags *flag.FlagSet) {
@@ -112,9 +115,11 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
112115
flags.BoolVar(&tlsConfig.SkipVerify, "tls-skip-verify", false, "Skip TLS certificate verification - use it only for testing")
113116
flags.BoolVar(&disableTLS, "disable-tls", false, "Disable TLS encryption - use it only for testing")
114117
flags.BoolVar(&secureComms, "secure-comms", false, "Use SSH to secure communication between cluster and peer pods")
115-
flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "Inbound tags for secure communication tunnels")
116-
flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "Outbound tags for secure communication tunnels")
117-
flags.StringVar(&secureCommsKbsAddr, "secure-comms-kbs", "kbs-service.kbs-operator-system:8080", "Address of a KBS Service for Secure-Comms")
118+
flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "WN Inbound tags for secure communication tunnels")
119+
flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "WN Outbound tags for secure communication tunnels")
120+
flags.StringVar(&secureCommsPpInbounds, "secure-comms-pp-inbounds", "", "PP Inbound tags for secure communication tunnels")
121+
flags.StringVar(&secureCommsPpOutbounds, "secure-comms-pp-outbounds", "", "PP Outbound tags for secure communication tunnels")
122+
flags.StringVar(&secureCommsKbsAddr, "secure-comms-kbs", "kbs-service.trustee-operator-system:8080", "Address of a Trustee Service for Secure-Comms")
118123
flags.DurationVar(&cfg.serverConfig.ProxyTimeout, "proxy-timeout", proxy.DefaultProxyTimeout, "Maximum timeout in minutes for establishing agent proxy connection")
119124

120125
flags.StringVar(&cfg.networkConfig.TunnelType, "tunnel-type", podnetwork.DefaultTunnelType, "Tunnel provider")
@@ -141,6 +146,8 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
141146
cfg.serverConfig.SecureComms = true
142147
cfg.serverConfig.SecureCommsInbounds = secureCommsInbounds
143148
cfg.serverConfig.SecureCommsOutbounds = secureCommsOutbounds
149+
cfg.serverConfig.SecureCommsPpInbounds = secureCommsPpInbounds
150+
cfg.serverConfig.SecureCommsPpOutbounds = secureCommsPpOutbounds
144151
cfg.serverConfig.SecureCommsKbsAddress = secureCommsKbsAddr
145152
} else {
146153
if !disableTLS {

‎src/cloud-api-adaptor/docs/SecureComms.md

Lines changed: 80 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,34 @@ Once the "Kubernetes Phase" SSH channel is established, Secure Comms connects th
2727

2828
See [Secure Comms Architecture Slides](./SecureComms.pdf) for more details.
2929

30-
## Setup
30+
## Setup for for testing without Trustee (and for non-CoCo peerpods)
31+
32+
### Deploy CAA
33+
Use any of the option for installing CAA depending on the cloud driver used.
34+
35+
36+
### Activate Secure-Comms feature from CAA side
37+
Activate Secure-Comms from CAA side by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`.
38+
39+
```sh
40+
kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f -
41+
```
42+
43+
You may also include additional Inbounds and Outbounds configurations to the Adaptor side using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points.
44+
You may also add Inbounds and Outbounds configurations to the Forwarder side using the `SECURE_COMMS_PP_INBOUNDS` and `SECURE_COMMS_PP_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)
45+
46+
Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example:
47+
```sh
48+
apiVersion: v1
49+
data:
50+
...
51+
SECURE_COMMS: "true"
52+
SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777"
53+
SECURE_COMMS_PP_INBOUNDS: "KUBERNETES_PHASE:mytunnel:podns:6666"
54+
...
55+
```
56+
57+
## Setup for CoCo with Trustee
3158

3259
### Deploy CAA
3360
Use any of the option for installing CAA depending on the cloud driver used.
@@ -37,19 +64,25 @@ Deploy Trustee-Operator by following instructions at [trustee Operator Getting S
3764

3865
Make sure to uncomment the secret generation as recommended for both public and private key (`kbs-auth-public-key` and `kbs-client` secrets).
3966

40-
Copy the kbs-client secret from the `kbs-operator-system` namespace to the `confidential-containers-system` ns. This can be done using:
67+
```sh
68+
kubectl get secrets -n trustee-operator-system
69+
NAME TYPE DATA AGE
70+
kbs-auth-public-key Opaque 1 28h
71+
kbs-client Opaque 1 28h
72+
```
73+
74+
Copy the kbs-client secret from the `trustee-operator-system` namespace to the `confidential-containers-system` ns. This can be done using:
4175

4276
```sh
43-
kubectl get secret kbs-client -n kbs-operator-system -o json|jq --arg ns "confidential-containers-system" 'del(.metadata["creationTimestamp","resourceVersion","selfLink","uid","annotations"]) | .metadata.namespace |= $ns' |kubectl apply -f -
77+
kubectl get secret kbs-client -n trustee-operator-system -o json|jq --arg ns "confidential-containers-system" 'del(.metadata["creationTimestamp","resourceVersion","selfLink","uid","annotations"]) | .metadata.namespace |= $ns' |kubectl apply -f -
4478
```
4579

4680
For a testing environment, you may need to change the policy of the KBS and AS using the KBS Client to allow all or fit your own policy. One way to do that is:
4781

4882
```sh
49-
kubectl -n kbs-operator-system exec deployment/trustee-deployment --container as -it -- /bin/bash
50-
sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego
83+
kubectl -n trustee-operator-system exec deployment/trustee-deployment --container as -it -- sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego
5184

52-
kubectl -n kbs-operator-system get cm resource-policy -o yaml | sed "s/default allow = false/default allow = true/"|kubectl apply -f -
85+
kubectl -n trustee-operator-system get cm resource-policy -o yaml | sed "s/default allow = false/default allow = true/"|kubectl apply -f -
5386
```
5487

5588
### Build a podvm that enforces Secure-Comms
@@ -59,27 +92,60 @@ Change the `src/cloud-api-adaptor/podvm/files/etc/systemd/system/agent-protocol-
5992
ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -secure-comms -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
6093
```
6194

62-
You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. See more details regarding Inbounds and Outbounds below.
95+
You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)
96+
97+
For example:
98+
```sh
99+
ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -secure-comms -secure-comms-inbounds KUBERNETES_PHASE:mytunnel:podns:6666 -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
100+
```
63101

64102
Once you changed `podvm/files/etc/systemd/system/agent-protocol-forwarder.service`, you will need to [rebuild the podvm](./../podvm/README.md).
65103

66104

67105
### Activate CAA Secure-Comms feature
68-
Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to add to the `peer-pods-cm` config map at the `confidential-containers-system` namespace:
106+
Activate Secure-Comms of CAA by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`.
107+
108+
```sh
109+
kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f -
110+
```
111+
112+
Set InitData to point KBC services to IP address 127.0.0.1
113+
```sh
114+
cat <<EOF > /tmp/initdata.txt
115+
algorithm = "sha384"
116+
version = "0.1.0"
117+
118+
[data]
119+
"cdh.toml" = '''
120+
socket = 'unix:///run/confidential-containers/cdh.sock'
121+
credentials = []
122+
123+
[kbc]
124+
name = 'cc_kbc'
125+
url = 'http://127.0.0.1:8080'
126+
'''
127+
EOF
128+
export INITDATA=`base64 /tmp/initdata.txt`
129+
kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed 's/^INITDATA: .*/INITDATA: $INITDATA/'|kubectl apply -f -
130+
131+
```
132+
133+
You may also include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)
134+
135+
Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example:
69136
```sh
70137
apiVersion: v1
71138
data:
72139
...
73140
SECURE_COMMS: "true"
141+
SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777"
74142
...
75143
```
76144

77-
You may also include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. See more details regarding Inbounds and Outbounds below.
78-
79145
You may also set the KBS address using the `SECURE_COMMS_KBS_ADDR` config point.
80146

81147

82-
### Adding named tunnels to the SSH channel
148+
## Adding named tunnels to the SSH channel
83149
Named tunnels can be added to the SSH channel. Adding a named tunnel requires adding an Inbound at one of the SSH channel peers and an Outbound at the other SSH channel peer. The Inbound and Outbound both carry the name of the tunnel being created.
84150

85151
|---------Tunnel----------|
@@ -88,9 +154,11 @@ Client->Inbound----------->Outbound->Server
88154

89155
Inbounds and Outbounds take the form of a comma separated inbound/outbound tags such that Inbounds are formed as "InboundTag1,InboundTag2,InboundTag3,..." and Outbounds are formed as "OutboundTag1,OutboundTag2,outboundTag3,..."
90156

91-
Each Inbound tag is structured as `Phase:Name:Port` where:
157+
158+
Each Inbound tag is structured as `Phase:Name:Namespace:Port` or `Phase:Name:Port` where:
92159
- Phase can be 'KUBERNETES_PHASE' to represent an outbound available during the Kubernetes phase, 'ATTESTATION_PHASE' to represent an outbound available during the Attestation phase, or 'BOTH_PHASES' to represent an outbound available during both phases.
93160
- Name is the name of the tunnel
161+
- Namespace (if available) is a linux network namespace where the local service should be available.
94162
- Port is the local service port being opened to serve as ingress of the tunnel.
95163

96164
Each outbound tag is structured as `Phase:Name:Host:Port` or `Phase:Name:Port` where:
@@ -117,5 +185,4 @@ Alternatively, the client and server can be separately executed in independent t
117185

118186
- Add DeleteResource() support in KBS, KBC, api-server-rest, than cleanup resources added by Secure Comms to KBS whenever a Peer Pod fail to be created or when a Peer Pod is terminated.
119187
- Add support for running the vxlan tunnel traffic via a Secure Comms SSH tunnel
120-
- Add support for non-confidential Peer Pods which do not go via an Attestation Phase.
121188
- Add support for KBS identities allowing a Peer Pod to register its own identity in KBS and replace the current Secure Comms mechanism which delivers a private key to the Peer Pod via the KBS

‎src/cloud-api-adaptor/entrypoint.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ optionals+=""
2424
[[ "${SECURE_COMMS}" == "true" ]] && optionals+="-secure-comms "
2525
[[ "${SECURE_COMMS_INBOUNDS}" ]] && optionals+="-secure-comms-inbounds ${SECURE_COMMS_INBOUNDS} "
2626
[[ "${SECURE_COMMS_OUTBOUNDS}" ]] && optionals+="-secure-comms-outbounds ${SECURE_COMMS_OUTBOUNDS} "
27+
[[ "${SECURE_COMMS_PP_INBOUNDS}" ]] && optionals+="-secure-comms-pp-inbounds ${SECURE_COMMS_PP_INBOUNDS} "
28+
[[ "${SECURE_COMMS_PP_OUTBOUNDS}" ]] && optionals+="-secure-comms-pp-outbounds ${SECURE_COMMS_PP_OUTBOUNDS} "
2729
[[ "${SECURE_COMMS_KBS_ADDR}" ]] && optionals+="-secure-comms-kbs ${SECURE_COMMS_KBS_ADDR} "
2830
[[ "${PEERPODS_LIMIT_PER_NODE}" ]] && optionals+="-peerpods-limit-per-node ${PEERPODS_LIMIT_PER_NODE} "
2931

0 commit comments

Comments
 (0)