diff --git a/cmd/install.go b/cmd/install.go index 929a6867..c9abc0f9 100644 --- a/cmd/install.go +++ b/cmd/install.go @@ -22,11 +22,11 @@ var installCmd = &cobra.Command{ if err := installOptions.Env.CheckAndSetValidEnvironmentOption(cmd.Flag("env").Value.String()); err != nil { return fmt.Errorf("error in checking environment option: %v", err) } - if err := install.K8sLegacyInstaller(client, installOptions); err != nil { + if err := install.K8sLegacyInstaller(k8sClient, installOptions); err != nil { return fmt.Errorf("error installing kubearmor in legacy mode: %v", err) } } else { - if err := install.K8sInstaller(client, installOptions); err != nil { + if err := install.K8sInstaller(k8sClient, installOptions); err != nil { return fmt.Errorf("error installing kubearmor: %v", err) } } diff --git a/cmd/log.go b/cmd/log.go index c0994c7a..8581ca31 100644 --- a/cmd/log.go +++ b/cmd/log.go @@ -17,7 +17,7 @@ var logCmd = &cobra.Command{ Long: `Observe Logs from KubeArmor`, RunE: func(cmd *cobra.Command, args []string) error { log.StopChan = make(chan struct{}) - return log.StartObserver(client, logOptions) + return log.StartObserver(k8sClient, logOptions) }, } diff --git a/cmd/probe.go b/cmd/probe.go index 47b59738..5b195fee 100644 --- a/cmd/probe.go +++ b/cmd/probe.go @@ -22,9 +22,8 @@ and what KubeArmor features will be supported e.g: observability, enforcement, e If KubeArmor is running, It probes which environment KubeArmor is running on (e.g: systemd mode, kubernetes etc.), the supported KubeArmor features in the environment, the pods being handled by KubeArmor and the policies running on each of these pods`, RunE: func(cmd *cobra.Command, args []string) error { - err := probe.PrintProbeResultCmd(client, probeInstallOptions) + err := probe.PrintProbeResultCmd(k8sClient, probeInstallOptions) return err - }, } diff --git a/cmd/recommend.go b/cmd/recommend.go index d9949c3f..5a74947f 100644 --- a/cmd/recommend.go +++ b/cmd/recommend.go @@ -4,11 +4,13 @@ package cmd import ( + "context" "github.com/kubearmor/kubearmor-client/recommend" "github.com/kubearmor/kubearmor-client/recommend/common" genericpolicies "github.com/kubearmor/kubearmor-client/recommend/engines/generic_policies" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var recommendOptions common.Options @@ -19,8 +21,19 @@ var recommendCmd = &cobra.Command{ Short: "Recommend Policies", Long: `Recommend policies based on container image, k8s manifest or the actual runtime env`, RunE: func(cmd *cobra.Command, args []string) error { - err := recommend.Recommend(client, recommendOptions, genericpolicies.GenericPolicy{}) - return err + if recommendOptions.K8s { + // Check if k8sClient can connect to the server by listing namespaces + _, err := k8sClient.K8sClientset.CoreV1().Namespaces().List(context.Background(), v1.ListOptions{}) + if err != nil { + if len(recommendOptions.Images) == 0 { // only log the client if no images are provided + log.Error("K8s client is not initialized, using docker client instead") + } + return recommend.Recommend(dockerClient, recommendOptions, genericpolicies.GenericPolicy{}) + } + return recommend.Recommend(k8sClient, recommendOptions, genericpolicies.GenericPolicy{}) + } else { + return recommend.Recommend(dockerClient, recommendOptions, genericpolicies.GenericPolicy{}) + } }, } var updateCmd = &cobra.Command{ @@ -28,7 +41,6 @@ var updateCmd = &cobra.Command{ Short: "Updates policy-template cache", Long: "Updates the local cache of policy-templates ($HOME/.cache/karmor)", RunE: func(cmd *cobra.Command, args []string) error { - if _, err := genericpolicies.DownloadAndUnzipRelease(); err != nil { return err } @@ -50,4 +62,5 @@ func init() { recommendCmd.Flags().StringVarP(&recommendOptions.ReportFile, "report", "r", "report.txt", "report file") recommendCmd.Flags().StringSliceVarP(&recommendOptions.Tags, "tag", "t", []string{}, "tags (comma-separated) to apply. Eg. PCI-DSS, MITRE") recommendCmd.Flags().StringVarP(&recommendOptions.Config, "config", "c", common.UserHome()+"/.docker/config.json", "absolute path to image registry configuration file") + recommendCmd.Flags().BoolVarP(&recommendOptions.K8s, "k8s", "k", true, "Use k8s client instead of docker client") } diff --git a/cmd/root.go b/cmd/root.go index a1cfdb97..bdb76ed6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,12 +5,14 @@ package cmd import ( + "github.com/kubearmor/kubearmor-client/docker" "github.com/kubearmor/kubearmor-client/k8s" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) -var client *k8s.Client +var k8sClient *k8s.Client +var dockerClient *docker.Client // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ @@ -18,12 +20,18 @@ var rootCmd = &cobra.Command{ var err error //Initialise k8sClient for all child commands to inherit - client, err = k8s.ConnectK8sClient() - // fmt.Printf("%v", client.K8sClientset) + k8sClient, err = k8s.ConnectK8sClient() if err != nil { log.Error().Msgf("unable to create Kubernetes clients: %s", err.Error()) return err } + + // Initialise dockerClient for all child commands to inherit + dockerClient, err = docker.ConnectDockerClient() + if err != nil { + log.Error().Msgf("unable to create Docker clients: %s", err.Error()) + return err + } return nil }, Use: "karmor", diff --git a/cmd/rotate-tls.go b/cmd/rotate-tls.go index 5d77e6ef..e5048c8d 100644 --- a/cmd/rotate-tls.go +++ b/cmd/rotate-tls.go @@ -11,7 +11,7 @@ var rotateCmd = &cobra.Command{ Short: "Rotate webhook controller tls certificates", Long: `Rotate webhook controller tls certificates`, RunE: func(cmd *cobra.Command, args []string) error { - if err := rotatetls.RotateTLS(client, namespace); err != nil { + if err := rotatetls.RotateTLS(k8sClient, namespace); err != nil { return err } return nil @@ -20,6 +20,5 @@ var rotateCmd = &cobra.Command{ func init() { rootCmd.AddCommand(rotateCmd) - rotateCmd.Flags().StringVarP(&namespace, "namespace", "n", "kubearmor", "Namespace for resources") } diff --git a/cmd/selfupdate.go b/cmd/selfupdate.go index 7deda385..e460275c 100644 --- a/cmd/selfupdate.go +++ b/cmd/selfupdate.go @@ -14,7 +14,7 @@ var selfUpdateCmd = &cobra.Command{ Short: "selfupdate this cli tool", Long: `selfupdate this cli tool for checking the latest release on the github`, RunE: func(cmd *cobra.Command, args []string) error { - if err := selfupdate.SelfUpdate(client); err != nil { + if err := selfupdate.SelfUpdate(k8sClient); err != nil { return err } return nil diff --git a/cmd/sysdump.go b/cmd/sysdump.go index f03652e4..4a130da7 100644 --- a/cmd/sysdump.go +++ b/cmd/sysdump.go @@ -16,7 +16,7 @@ var sysdumpCmd = &cobra.Command{ Short: "Collect system dump information for troubleshooting and error report", Long: `Collect system dump information for troubleshooting and error reports`, RunE: func(cmd *cobra.Command, args []string) error { - if err := sysdump.Collect(client, dumpOptions); err != nil { + if err := sysdump.Collect(k8sClient, dumpOptions); err != nil { return err } return nil diff --git a/cmd/uninstall.go b/cmd/uninstall.go index 68fc961f..be51930b 100644 --- a/cmd/uninstall.go +++ b/cmd/uninstall.go @@ -16,8 +16,8 @@ var uninstallCmd = &cobra.Command{ Short: "Uninstall KubeArmor from a Kubernetes Cluster", Long: `Uninstall KubeArmor from a Kubernetes Clusters`, RunE: func(cmd *cobra.Command, args []string) error { - if err := install.K8sUninstaller(client, uninstallOptions); err != nil { - if err := install.K8sLegacyUninstaller(client, uninstallOptions); err != nil { + if err := install.K8sUninstaller(k8sClient, uninstallOptions); err != nil { + if err := install.K8sLegacyUninstaller(k8sClient, uninstallOptions); err != nil { return err } } diff --git a/cmd/version.go b/cmd/version.go index dfa46a00..aac8c90b 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -14,7 +14,7 @@ var versionCmd = &cobra.Command{ Short: "Display version information", Long: `Display version information`, RunE: func(cmd *cobra.Command, args []string) error { - if err := version.PrintVersion(client); err != nil { + if err := version.PrintVersion(k8sClient); err != nil { return err } return nil diff --git a/docker/client.go b/docker/client.go new file mode 100644 index 00000000..015f986d --- /dev/null +++ b/docker/client.go @@ -0,0 +1,40 @@ +package docker + +import ( + "context" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/client" + "github.com/kubearmor/kubearmor-client/recommend/common" + "strings" +) + +type Client struct { + *client.Client +} + +func ConnectDockerClient() (*Client, error) { + cli, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + return nil, err + } + return &Client{cli}, nil +} + +func (c *Client) ListObjects(o common.Options) ([]common.Object, error) { + var result []common.Object + containers, err := c.Client.ContainerList(context.Background(), container.ListOptions{ + Filters: filters.NewArgs(), + }) + if err != nil { + return nil, err + } + for _, ctr := range containers { + result = append(result, common.Object{ + Name: strings.TrimPrefix(ctr.Names[0], "/"), + Images: []string{ctr.Image}, + Labels: ctr.Labels, + }) + } + return result, nil +} diff --git a/k8s/client.go b/k8s/client.go index 27117c25..be915fa9 100644 --- a/k8s/client.go +++ b/k8s/client.go @@ -6,7 +6,7 @@ package k8s import ( "context" - + "github.com/kubearmor/kubearmor-client/recommend/common" "github.com/rs/zerolog/log" apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -102,3 +102,162 @@ func GetKubeArmorCaSecret(client kubernetes.Interface) (string, string) { } return secret.Items[0].Name, secret.Items[0].Namespace } + +func (k *Client) ListObjects(o common.Options) ([]common.Object, error) { + labelSelector := v1.FormatLabelSelector(&v1.LabelSelector{MatchLabels: common.LabelArrayToLabelMap(o.Labels)}) + if labelSelector == "" { + labelSelector = "" + } + // CronJobs + cronJobs, err := k.K8sClientset.BatchV1().CronJobs(o.Namespace).List(context.Background(), v1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + log.Error().Msgf("error listing cronjobs: %v", err) + return nil, err + } + + // DaemonSets + daemonSets, err := k.K8sClientset.AppsV1().DaemonSets(o.Namespace).List(context.Background(), v1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + log.Error().Msgf("error listing daemonsets: %v", err) + return nil, err + } + + // Deployments + deployments, err := k.K8sClientset.AppsV1().Deployments(o.Namespace).List(context.Background(), v1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + log.Error().Msgf("error listing deployments: %v", err) + return nil, err + } + + // Jobs + jobs, err := k.K8sClientset.BatchV1().Jobs(o.Namespace).List(context.Background(), v1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + log.Error().Msgf("error listing jobs: %v", err) + return nil, err + } + + // ReplicaSets + replicaSets, err := k.K8sClientset.AppsV1().ReplicaSets(o.Namespace).List(context.Background(), v1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + log.Error().Msgf("error listing replicasets: %v", err) + return nil, err + } + + // StatefulSets + statefulSets, err := k.K8sClientset.AppsV1().StatefulSets(o.Namespace).List(context.Background(), v1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + log.Error().Msgf("error listing statefulsets: %v", err) + return nil, err + } + + var result []common.Object + + for _, cj := range cronJobs.Items { + var images []string + for _, container := range cj.Spec.JobTemplate.Spec.Template.Spec.Containers { + images = append(images, container.Image) + } + + result = append(result, common.Object{ + Name: cj.Name, + Namespace: cj.Namespace, + Labels: cj.Spec.JobTemplate.Spec.Template.Labels, + Images: images, + }) + } + + for _, ds := range daemonSets.Items { + var images []string + for _, container := range ds.Spec.Template.Spec.Containers { + images = append(images, container.Image) + } + + result = append(result, common.Object{ + Name: ds.Name, + Namespace: ds.Namespace, + Labels: ds.Spec.Template.Labels, + Images: images, + }) + } + + for _, dp := range deployments.Items { + var images []string + for _, container := range dp.Spec.Template.Spec.Containers { + images = append(images, container.Image) + } + + result = append(result, common.Object{ + Name: dp.Name, + Namespace: dp.Namespace, + Labels: dp.Spec.Template.Labels, + Images: images, + }) + } + + for _, j := range jobs.Items { + var images []string + for _, container := range j.Spec.Template.Spec.Containers { + images = append(images, container.Image) + } + + result = append(result, common.Object{ + Name: j.Name, + Namespace: j.Namespace, + Labels: j.Spec.Template.Labels, + Images: images, + }) + } + + for _, rs := range replicaSets.Items { + isOwned := false + for _, owner := range rs.OwnerReferences { + if owner.Kind == "Deployment" || owner.Kind == "StatefulSet" || owner.Kind == "DaemonSet" || owner.Kind == "ReplicaSet" { + isOwned = true + break + } + } + if isOwned { + continue + } + + var images []string + for _, container := range rs.Spec.Template.Spec.Containers { + images = append(images, container.Image) + } + result = append(result, common.Object{ + Name: rs.Name, + Namespace: rs.Namespace, + Labels: rs.Spec.Template.Labels, + Images: images, + }) + } + + for _, sts := range statefulSets.Items { + var images []string + for _, container := range sts.Spec.Template.Spec.Containers { + images = append(images, container.Image) + } + + result = append(result, common.Object{ + Name: sts.Name, + Namespace: sts.Namespace, + Labels: sts.Spec.Template.Labels, + Images: images, + }) + } + + log.Printf("+%v", result) + return result, nil +} diff --git a/recommend/common/common.go b/recommend/common/common.go index cf4baeae..4c6d6139 100644 --- a/recommend/common/common.go +++ b/recommend/common/common.go @@ -7,6 +7,7 @@ package common import ( "os" "runtime" + "strings" pol "github.com/kubearmor/KubeArmor/pkg/KubeArmorController/api/security.kubearmor.com/v1" ) @@ -14,6 +15,21 @@ import ( // Handler interface var Handler interface{} +// LabelMap is an alias for map[string]string +type LabelMap = map[string]string + +type Client interface { + ListObjects(o Options) ([]Object, error) +} + +// Object contains brief information about a k8s object +type Object struct { + Name string + Namespace string + Labels LabelMap + Images []string +} + // MatchSpec spec to match for defining policy type MatchSpec struct { Name string `json:"name" yaml:"name"` @@ -46,6 +62,7 @@ type Options struct { OutDir string ReportFile string Config string + K8s bool } // UserHome function returns users home directory @@ -59,3 +76,18 @@ func UserHome() string { } return os.Getenv("HOME") } + +func labelSplitter(r rune) bool { + return r == ':' || r == '=' +} +func LabelArrayToLabelMap(labels []string) LabelMap { + labelMap := LabelMap{} + for _, label := range labels { + kvPair := strings.FieldsFunc(label, labelSplitter) + if len(kvPair) != 2 { + continue + } + labelMap[kvPair[0]] = kvPair[1] + } + return labelMap +} diff --git a/recommend/image/image.go b/recommend/image/image.go index 81746f91..0a4ea021 100644 --- a/recommend/image/image.go +++ b/recommend/image/image.go @@ -234,7 +234,11 @@ func (img *Info) GetPolicyDir(outDir string) string { } } else { // policy recommendation based on k8s manifest - policyDir = fmt.Sprintf("%s-%s", img.Namespace, img.Deployment) + if img.Namespace == "" { + policyDir = fmt.Sprintf("%s", img.Deployment) + } else { + policyDir = fmt.Sprintf("%s-%s", img.Namespace, img.Deployment) + } } return filepath.Join(outDir, policyDir) } diff --git a/recommend/recommend.go b/recommend/recommend.go index 9ab1ed66..4cdbde1b 100644 --- a/recommend/recommend.go +++ b/recommend/recommend.go @@ -5,7 +5,6 @@ package recommend import ( - "context" "errors" "fmt" "os" @@ -13,7 +12,6 @@ import ( "strings" "github.com/fatih/color" - "github.com/kubearmor/kubearmor-client/k8s" "github.com/kubearmor/kubearmor-client/recommend/common" "github.com/kubearmor/kubearmor-client/recommend/engines" "github.com/kubearmor/kubearmor-client/recommend/image" @@ -22,49 +20,10 @@ import ( "sigs.k8s.io/yaml" log "github.com/sirupsen/logrus" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var options common.Options -// Deployment contains brief information about a k8s deployment -type Deployment struct { - Name string - Namespace string - Labels LabelMap - Images []string -} - -// LabelMap is an alias for map[string]string -type LabelMap = map[string]string - -func labelSplitter(r rune) bool { - return r == ':' || r == '=' -} - -func labelArrayToLabelMap(labels []string) LabelMap { - labelMap := LabelMap{} - for _, label := range labels { - kvPair := strings.FieldsFunc(label, labelSplitter) - if len(kvPair) != 2 { - continue - } - labelMap[kvPair[0]] = kvPair[1] - } - return labelMap -} - -func matchLabels(filter, selector LabelMap) bool { - match := true - for k, v := range filter { - if selector[k] != v { - match = false - break - } - } - return match -} - func unique(s []string) []string { inResult := make(map[string]bool) var result []string @@ -108,7 +67,7 @@ func finalReport() { log.WithError(err).Fatal("failed to read report file") return } - fmt.Println(string(data)) + fmt.Println(strings.Trim(string(data), "\n")) } func writePolicyFile(policMap map[string][]byte, msMap map[string]interface{}) { @@ -137,44 +96,26 @@ func writePolicyFile(policMap map[string][]byte, msMap map[string]interface{}) { } // Recommend handler for karmor cli tool -func Recommend(c *k8s.Client, o common.Options, policyGenerators ...engines.Engine) error { +func Recommend(client common.Client, o common.Options, policyGenerators ...engines.Engine) error { var policyMap map[string][]byte var msMap map[string]interface{} var err error - deployments := []Deployment{} + var Objects []common.Object - labelMap := labelArrayToLabelMap(o.Labels) + labelMap := common.LabelArrayToLabelMap(o.Labels) if len(o.Images) == 0 { - // recommendation based on k8s manifest - dps, err := c.K8sClientset.AppsV1().Deployments(o.Namespace).List(context.TODO(), v1.ListOptions{}) + Objects, err = client.ListObjects(o) if err != nil { return err } - for _, dp := range dps.Items { - - if !matchLabels(labelMap, dp.Spec.Template.Labels) { - continue - } - images := []string{} - for _, container := range dp.Spec.Template.Spec.Containers { - images = append(images, container.Image) - } - - deployments = append(deployments, Deployment{ - Name: dp.Name, - Namespace: dp.Namespace, - Labels: dp.Spec.Template.Labels, - Images: images, - }) - } - if len(deployments) == 0 { + if len(Objects) == 0 { log.WithFields(log.Fields{ "namespace": o.Namespace, - }).Error("no k8s deployments found, hence nothing to recommend!") + }).Error("no Object found to secure, hence nothing to recommend!") return nil } } else { - deployments = append(deployments, Deployment{ + Objects = append(Objects, common.Object{ Namespace: o.Namespace, Labels: labelMap, Images: o.Images, @@ -196,14 +137,14 @@ func Recommend(c *k8s.Client, o common.Options, policyGenerators ...engines.Engi if err := gen.Init(); err != nil { log.WithError(err).Error("policy generator init failed") } - for _, deployment := range deployments { - for _, i := range deployment.Images { + for _, obj := range Objects { + for _, v := range obj.Images { img := image.Info{ - Name: i, - Namespace: deployment.Namespace, - Labels: deployment.Labels, - Image: i, - Deployment: deployment.Name, + Name: v, + Namespace: obj.Namespace, + Labels: obj.Labels, + Image: v, + Deployment: obj.Name, } reg.Analyze(&img) if policyMap, msMap, err = gen.Scan(&img, o); err != nil {