Skip to content

Commit 3574106

Browse files
Aryan-sharma11Aryan-sharma11
authored
Merge pull request #1831 from daemon1024/dogfood
Enabling BPFLSM based KSP protection on Kubearmor itself
2 parents 4731b3c + c85b541 commit 3574106

17 files changed

+70
-36
lines changed

Diff for: ‎KubeArmor/BPF/enforcer.bpf.c

+5-3
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,10 @@ int BPF_PROG(enforce_proc, struct linux_binprm *bprm, int ret) {
7575
if (src_offset == NULL)
7676
fromSourceCheck = false;
7777

78-
void *src_ptr = &src_buf->buf[*src_offset];
78+
void *src_ptr;
79+
if (src_buf->buf[*src_offset]) {
80+
src_ptr = &src_buf->buf[*src_offset];
81+
}
7982
if (src_ptr == NULL)
8083
fromSourceCheck = false;
8184

@@ -152,10 +155,9 @@ int BPF_PROG(enforce_proc, struct linux_binprm *bprm, int ret) {
152155
goto decision;
153156
}
154157

155-
156158
// match exec name
157159
struct qstr d_name;
158-
d_name = BPF_CORE_READ(f_path.dentry,d_name);
160+
d_name = BPF_CORE_READ(f_path.dentry, d_name);
159161
bpf_map_update_elem(&bufk, &two, z, BPF_ANY);
160162
bpf_probe_read_str(pk->path, MAX_STRING_SIZE, d_name.name);
161163

Diff for: ‎KubeArmor/BPF/shared.h

+16-15
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,9 @@ static inline void get_outer_key(struct outer_key *pokey,
272272
struct task_struct *t) {
273273
pokey->pid_ns = get_task_pid_ns_id(t);
274274
pokey->mnt_ns = get_task_mnt_ns_id(t);
275+
// TODO: Use cgroup ns as well for host process identification to support enforcement on deployments using hostpidns
276+
// u32 cg_ns = BPF_CORE_READ(t, nsproxy, cgroup_ns, ns).inum;
277+
// if (pokey->pid_ns == PROC_PID_INIT_INO && cg_ns == PROC_CGROUP_INIT_INO) {
275278
if (pokey->pid_ns == PROC_PID_INIT_INO) {
276279
pokey->pid_ns = 0;
277280
pokey->mnt_ns = 0;
@@ -288,20 +291,13 @@ static __always_inline u32 init_context(event *event_data) {
288291
event_data->host_ppid = get_task_ppid(task);
289292
event_data->host_pid = bpf_get_current_pid_tgid() >> 32;
290293

291-
u32 pid = get_task_ns_tgid(task);
292-
if (event_data->host_pid == pid) { // host
293-
event_data->pid_id = 0;
294-
event_data->mnt_id = 0;
295-
296-
event_data->ppid = get_task_ppid(task);
297-
event_data->pid = bpf_get_current_pid_tgid() >> 32;
298-
} else { // container
299-
event_data->pid_id = get_task_pid_ns_id(task);
300-
event_data->mnt_id = get_task_mnt_ns_id(task);
294+
struct outer_key okey;
295+
get_outer_key(&okey, task);
296+
event_data->pid_id = okey.pid_ns;
297+
event_data->mnt_id = okey.mnt_ns;
301298

302-
event_data->ppid = get_task_ns_ppid(task);
303-
event_data->pid = pid;
304-
}
299+
event_data->ppid = get_task_ppid(task);
300+
event_data->pid = get_task_ns_tgid(task);
305301

306302
event_data->uid = bpf_get_current_uid_gid();
307303

@@ -487,10 +483,15 @@ static inline int match_and_enforce_path_hooks(struct path *f_path, u32 id,
487483
if (src_offset == NULL)
488484
fromSourceCheck = false;
489485

490-
void *ptr = &src_buf->buf[*src_offset];
486+
void *src_ptr;
487+
if (src_buf->buf[*src_offset]) {
488+
src_ptr = &src_buf->buf[*src_offset];
489+
}
490+
if (src_ptr == NULL)
491+
fromSourceCheck = false;
491492

492493
if (fromSourceCheck) {
493-
bpf_probe_read_str(store->source, MAX_STRING_SIZE, ptr);
494+
bpf_probe_read_str(store->source, MAX_STRING_SIZE, src_ptr);
494495

495496
val = bpf_map_lookup_elem(inner, store);
496497

Diff for: ‎KubeArmor/config/config.go

+9
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ type KubearmorConfig struct {
6060
MaxAlertPerSec int // Maximum alerts allowed per second
6161
ThrottleSec int // Number of seconds for which subsequent alerts will be dropped
6262
AnnotateResources bool // enable annotations by kubearmor if kubearmor-controller is not present
63+
64+
ProcFsMount string // path where procfs is hosted
6365
}
6466

6567
// GlobalCfg Global configuration for Kubearmor
@@ -105,6 +107,7 @@ const (
105107
ConfigMaxAlertPerSec string = "maxAlertPerSec"
106108
ConfigThrottleSec string = "throttleSec"
107109
ConfigAnnotateResources string = "annotateResources"
110+
ConfigProcFsMount string = "procfsMount"
108111
)
109112

110113
func readCmdLineParams() {
@@ -161,6 +164,8 @@ func readCmdLineParams() {
161164

162165
annotateResources := flag.Bool(ConfigAnnotateResources, false, "for kubearmor deployment without kubearmor-controller")
163166

167+
procFsMount := flag.String(ConfigProcFsMount, "/proc", "Path to the BPF filesystem to use for storing maps")
168+
164169
flags := []string{}
165170
flag.VisitAll(func(f *flag.Flag) {
166171
kv := fmt.Sprintf("%s:%v", f.Name, f.Value)
@@ -222,6 +227,8 @@ func readCmdLineParams() {
222227
viper.SetDefault(ConfigThrottleSec, *throttleSec)
223228

224229
viper.SetDefault(ConfigAnnotateResources, *annotateResources)
230+
231+
viper.SetDefault(ConfigProcFsMount, *procFsMount)
225232
}
226233

227234
// LoadConfig Load configuration
@@ -322,6 +329,8 @@ func LoadConfig() error {
322329
GlobalCfg.ThrottleSec = viper.GetInt(ConfigThrottleSec)
323330
GlobalCfg.AnnotateResources = viper.GetBool(ConfigAnnotateResources)
324331

332+
GlobalCfg.ProcFsMount = viper.GetString(ConfigProcFsMount)
333+
325334
kg.Printf("Final Configuration [%+v]", GlobalCfg)
326335

327336
return nil

Diff for: ‎KubeArmor/core/containerdHandler.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"context"
99
"fmt"
1010
"os"
11+
"path/filepath"
1112
"strconv"
1213
"strings"
1314
"time"
@@ -193,13 +194,13 @@ func (ch *ContainerdHandler) GetContainerInfo(ctx context.Context, containerID s
193194

194195
pid := strconv.Itoa(int(taskRes.Processes[0].Pid))
195196

196-
if data, err := os.Readlink("/proc/" + pid + "/ns/pid"); err == nil {
197+
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/pid")); err == nil {
197198
if _, err := fmt.Sscanf(data, "pid:[%d]\n", &container.PidNS); err != nil {
198199
kg.Warnf("Unable to get PidNS (%s, %s, %s)", containerID, pid, err.Error())
199200
}
200201
}
201202

202-
if data, err := os.Readlink("/proc/" + pid + "/ns/mnt"); err == nil {
203+
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/mnt")); err == nil {
203204
if _, err := fmt.Sscanf(data, "mnt:[%d]\n", &container.MntNS); err != nil {
204205
kg.Warnf("Unable to get MntNS (%s, %s, %s)", containerID, pid, err.Error())
205206
}

Diff for: ‎KubeArmor/core/crioHandler.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"encoding/json"
99
"fmt"
1010
"os"
11+
"path/filepath"
1112
"strconv"
1213
"time"
1314

@@ -130,15 +131,15 @@ func (ch *CrioHandler) GetContainerInfo(ctx context.Context, containerID string,
130131

131132
pid := strconv.Itoa(containerInfo.Pid)
132133

133-
if data, err := os.Readlink("/proc/" + pid + "/ns/pid"); err == nil {
134+
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/pid")); err == nil {
134135
if _, err := fmt.Sscanf(data, "pid:[%d]\n", &container.PidNS); err != nil {
135136
kg.Warnf("Unable to get PidNS (%s, %s, %s)", containerID, pid, err.Error())
136137
}
137138
} else {
138139
return container, err
139140
}
140141

141-
if data, err := os.Readlink("/proc/" + pid + "/ns/mnt"); err == nil {
142+
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/mnt")); err == nil {
142143
if _, err := fmt.Sscanf(data, "mnt:[%d]\n", &container.MntNS); err != nil {
143144
kg.Warnf("Unable to get MntNS (%s, %s, %s)", containerID, pid, err.Error())
144145
}

Diff for: ‎KubeArmor/core/dockerHandler.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"errors"
99
"fmt"
1010
"os"
11+
"path/filepath"
1112
"slices"
1213
"strconv"
1314
"strings"
@@ -144,13 +145,13 @@ func (dh *DockerHandler) GetContainerInfo(containerID string, OwnerInfo map[stri
144145

145146
pid := strconv.Itoa(inspect.State.Pid)
146147

147-
if data, err := os.Readlink("/proc/" + pid + "/ns/pid"); err == nil {
148+
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/pid")); err == nil {
148149
if _, err := fmt.Sscanf(data, "pid:[%d]\n", &container.PidNS); err != nil {
149150
kg.Warnf("Unable to get PidNS (%s, %s, %s)", containerID, pid, err.Error())
150151
}
151152
}
152153

153-
if data, err := os.Readlink("/proc/" + pid + "/ns/mnt"); err == nil {
154+
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/mnt")); err == nil {
154155
if _, err := fmt.Sscanf(data, "mnt:[%d]\n", &container.MntNS); err != nil {
155156
kg.Warnf("Unable to get MntNS (%s, %s, %s)", containerID, pid, err.Error())
156157
}

Diff for: ‎KubeArmor/core/kubeUpdate.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -731,9 +731,9 @@ func (dm *KubeArmorDaemon) WatchK8sPods() {
731731
}
732732

733733
// exception: kubearmor
734-
if _, ok := pod.Labels["kubearmor-app"]; ok {
735-
pod.Annotations["kubearmor-policy"] = "audited"
736-
}
734+
// if _, ok := pod.Labels["kubearmor-app"]; ok {
735+
// pod.Annotations["kubearmor-policy"] = "audited"
736+
// }
737737

738738
// == Visibility == //
739739

Diff for: ‎KubeArmor/enforcer/appArmorEnforcer.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -114,11 +114,11 @@ profile apparmor-default flags=(attach_disconnected,mediate_deleted) {
114114

115115
existingProfiles := []string{}
116116

117-
if pids, err := os.ReadDir(filepath.Clean("/proc")); err == nil {
117+
if pids, err := os.ReadDir(filepath.Clean(cfg.GlobalCfg.ProcFsMount)); err == nil {
118118
for _, f := range pids {
119119
if f.IsDir() {
120120
if _, err := strconv.Atoi(f.Name()); err == nil {
121-
if content, err := os.ReadFile(filepath.Clean("/proc/" + f.Name() + "/attr/current")); err == nil {
121+
if content, err := os.ReadFile(filepath.Clean(cfg.GlobalCfg.ProcFsMount + "/" + f.Name() + "/attr/current")); err == nil {
122122
line := strings.Split(string(content), "\n")[0]
123123
words := strings.Split(line, " ")
124124

Diff for: ‎KubeArmor/enforcer/bpflsm/enforcer.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ import (
2323
tp "github.com/kubearmor/KubeArmor/KubeArmor/types"
2424
)
2525

26-
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang enforcer ../../BPF/enforcer.bpf.c -- -I/usr/include/ -O2 -g
27-
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang enforcer_path ../../BPF/enforcer_path.bpf.c -- -I/usr/include/ -O2 -g
26+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang enforcer ../../BPF/enforcer.bpf.c -- -I/usr/include/ -O2 -g -fno-stack-protector
27+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang enforcer_path ../../BPF/enforcer_path.bpf.c -- -I/usr/include/ -O2 -g -fno-stack-protector
2828

2929
// ===================== //
3030
// == BPFLSM Enforcer == //

Diff for: ‎KubeArmor/enforcer/bpflsm/enforcer_bpfeb.o

-10.9 KB
Binary file not shown.

Diff for: ‎KubeArmor/enforcer/bpflsm/enforcer_bpfel.o

-10.9 KB
Binary file not shown.

Diff for: ‎KubeArmor/enforcer/bpflsm/enforcer_path_bpfeb.o

-2.6 KB
Binary file not shown.

Diff for: ‎KubeArmor/enforcer/bpflsm/enforcer_path_bpfel.o

-2.6 KB
Binary file not shown.

Diff for: ‎KubeArmor/main_test.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313

1414
var clusterPtr, gRPCPtr, logPathPtr *string
1515
var enableKubeArmorPolicyPtr, enableKubeArmorHostPolicyPtr, enableKubeArmorVMPtr, coverageTestPtr, enableK8sEnv, tlsEnabled *bool
16-
var defaultFilePosturePtr, defaultCapabilitiesPosturePtr, defaultNetworkPosturePtr, hostDefaultCapabilitiesPosturePtr, hostDefaultNetworkPosturePtr, hostDefaultFilePosturePtr *string
16+
var defaultFilePosturePtr, defaultCapabilitiesPosturePtr, defaultNetworkPosturePtr, hostDefaultCapabilitiesPosturePtr, hostDefaultNetworkPosturePtr, hostDefaultFilePosturePtr, procFsMountPtr *string
1717

1818
func init() {
1919
// options (string)
@@ -32,6 +32,8 @@ func init() {
3232
hostDefaultNetworkPosturePtr = flag.String("hostDefaultNetworkPosture", "block", "configuring default enforcement action in global network context {allow|audit|block}")
3333
hostDefaultCapabilitiesPosturePtr = flag.String("hostDefaultCapabilitiesPosture", "block", "configuring default enforcement action in global capability context {allow|audit|block}")
3434

35+
procFsMountPtr = flag.String("procfsMount", "/proc", "Path to the BPF filesystem to use for storing maps")
36+
3537
// options (boolean)
3638
enableKubeArmorPolicyPtr = flag.Bool("enableKubeArmorPolicy", true, "enabling KubeArmorPolicy")
3739
enableKubeArmorHostPolicyPtr = flag.Bool("enableKubeArmorHostPolicy", true, "enabling KubeArmorHostPolicy")
@@ -42,6 +44,7 @@ func init() {
4244

4345
// options (boolean)
4446
coverageTestPtr = flag.Bool("coverageTest", false, "enabling CoverageTest")
47+
4548
}
4649

4750
// TestMain - test to drive external testing coverage
@@ -64,6 +67,7 @@ func TestMain(t *testing.T) {
6467
fmt.Sprintf("-enableKubeArmorHostPolicy=%s", strconv.FormatBool(*enableKubeArmorHostPolicyPtr)),
6568
fmt.Sprintf("-coverageTest=%s", strconv.FormatBool(*coverageTestPtr)),
6669
fmt.Sprintf("-tlsEnabled=%s", strconv.FormatBool(*tlsEnabled)),
70+
fmt.Sprintf("-procfsMount=%s", *procFsMountPtr),
6771
}
6872

6973
t.Log("[INFO] Executed KubeArmor")

Diff for: ‎KubeArmor/monitor/processTree.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package monitor
55

66
import (
77
"os"
8+
"path/filepath"
89
"strconv"
910
"strings"
1011
"sync"
@@ -231,7 +232,7 @@ func (mon *SystemMonitor) GetParentExecPath(containerID string, ctx SyscallConte
231232

232233
if readlink {
233234
// just in case that it couldn't still get the full path
234-
if data, err := os.Readlink("/proc/" + strconv.FormatUint(uint64(ctx.HostPPID), 10) + "/exe"); err == nil && data != "" && data != "/" {
235+
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, strconv.FormatUint(uint64(ctx.HostPPID), 10), "/exe")); err == nil && data != "" && data != "/" {
235236
// // Store it in the ActiveHostPidMap so we don't need to read procfs again
236237
// // We don't call BuildPidNode Here cause that will put this into a cyclic function call loop
237238
// if pidMap, ok := ActiveHostPidMap[containerID]; ok {
@@ -276,7 +277,7 @@ func (mon *SystemMonitor) GetExecPath(containerID string, ctx SyscallContext, re
276277

277278
if readlink {
278279
// just in case that it couldn't still get the full path
279-
if data, err := os.Readlink("/proc/" + strconv.FormatUint(uint64(ctx.HostPID), 10) + "/exe"); err == nil && data != "" && data != "/" {
280+
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, strconv.FormatUint(uint64(ctx.HostPID), 10), "/exe")); err == nil && data != "" && data != "/" {
280281
// // Store it in the ActiveHostPidMap so we don't need to read procfs again
281282
// if pidMap, ok := ActiveHostPidMap[containerID]; ok {
282283
// if node, ok := pidMap[ctx.HostPID]; ok {
@@ -318,7 +319,7 @@ func (mon *SystemMonitor) GetCommand(containerID string, ctx SyscallContext, rea
318319

319320
if readlink {
320321
// just in case that it couldn't still get the full path
321-
if data, err := os.Readlink("/proc/" + strconv.FormatUint(uint64(ctx.HostPID), 10) + "/exe"); err == nil && data != "" && data != "/" {
322+
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, strconv.FormatUint(uint64(ctx.HostPID), 10), "/exe")); err == nil && data != "" && data != "/" {
322323
return data
323324
} else if err != nil {
324325
mon.Logger.Debugf("Could not read path from procfs due to %s", err.Error())

Diff for: ‎deployments/get/objects.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,7 @@ func GenerateDaemonSet(env, namespace string) *appsv1.DaemonSet {
264264
var terminationGracePeriodSeconds = int64(60)
265265
var args = []string{
266266
"-gRPC=" + strconv.Itoa(int(port)),
267+
"-procfsMount=/host/procfs",
267268
}
268269

269270
var containerVolumeMounts = []corev1.VolumeMount{
@@ -381,7 +382,6 @@ func GenerateDaemonSet(env, namespace string) *appsv1.DaemonSet {
381382
Operator: "Exists",
382383
},
383384
},
384-
HostPID: true,
385385
HostNetwork: true,
386386
RestartPolicy: "Always",
387387
DNSPolicy: "ClusterFirstWithHostNet",

Diff for: ‎pkg/KubeArmorOperator/common/defaults.go

+14
Original file line numberDiff line numberDiff line change
@@ -237,13 +237,27 @@ var CommonVolumes = []corev1.Volume{
237237
},
238238
},
239239
},
240+
{
241+
Name: "proc-fs-mount",
242+
VolumeSource: corev1.VolumeSource{
243+
HostPath: &corev1.HostPathVolumeSource{
244+
Path: "/proc",
245+
Type: &HostPathDirectory,
246+
},
247+
},
248+
},
240249
}
241250

242251
var CommonVolumesMount = []corev1.VolumeMount{
243252
{
244253
Name: "sys-kernel-debug-path",
245254
MountPath: "/sys/kernel/debug",
246255
},
256+
{
257+
Name: "proc-fs-mount",
258+
MountPath: "/host/procfs",
259+
ReadOnly: true,
260+
},
247261
}
248262

249263
var KubeArmorCaVolume = []corev1.Volume{

0 commit comments

Comments
 (0)