diff --git a/Makefile b/Makefile index 0619301e39..bfbb57abe1 100644 --- a/Makefile +++ b/Makefile @@ -283,7 +283,7 @@ bin/static/%: cmd/% FORCE | go-check @echo Building bin/static/k8s-sat… $(E)$(go_build_static) bin/static/k8s-sat ./spire-k8s-sat-plugin/cmd/... @echo Building bin/static/keymanager-k8s… - $(E)$(go_build) bin/static/keymanager-k8s ./spire-k8s-secret-plugin/ + $(E)$(go_build_static) bin/static/keymanager-k8s ./spire-k8s-secret-plugin/ bin/static/%: spire-k8s-sat-plugin/% FORCE @echo Building $@… diff --git a/go.work.sum b/go.work.sum index 0ac3f5f983..b3b57ae545 100644 --- a/go.work.sum +++ b/go.work.sum @@ -25,6 +25,7 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/hcl v1.0.1-0.20190430135223-99e2f22d1c94 h1:LaH4JWe6Q7ICdxL5raxQjSRw7Pj8uTtAENrjejIYZIg= github.com/hashicorp/hcl v1.0.1-0.20190430135223-99e2f22d1c94/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= diff --git a/pkg/agent/manager/manager.go b/pkg/agent/manager/manager.go index 493cce46b3..543cc62211 100644 --- a/pkg/agent/manager/manager.go +++ b/pkg/agent/manager/manager.go @@ -358,18 +358,24 @@ func (m *manager) runBundleObserver(ctx context.Context) error { } func (m *manager) storeSVID(svidChain []*x509.Certificate, reattestable bool) { + m.c.Log.Debugf("Updating expiring SVIDs in K8S Secrets") if err := m.storage.StoreSVID(svidChain, reattestable); err != nil { m.c.Log.WithError(err).Warn("Could not store SVID") + } else { + m.c.Log.Debugf("SVID updated in K8S Secrets") } } func (m *manager) storeBundle(bundle *bundleutil.Bundle) { + m.c.Log.Debugf("Updating expiring Bundle in K8S Secrets") var rootCAs []*x509.Certificate if bundle != nil { rootCAs = bundle.RootCAs() } if err := m.storage.StoreBundle(rootCAs); err != nil { m.c.Log.WithError(err).Error("Could not store bundle") + } else { + m.c.Log.Debugf("Bundle updated in K8S Secrets") } } diff --git a/pkg/agent/storage/legacy.go b/pkg/agent/storage/legacy.go index 1370f75df2..db0e9e6fd8 100644 --- a/pkg/agent/storage/legacy.go +++ b/pkg/agent/storage/legacy.go @@ -3,7 +3,6 @@ package storage import ( "bytes" "crypto/x509" - "encoding/pem" "errors" "fmt" "io/fs" @@ -14,6 +13,7 @@ import ( "github.com/accuknox/spire/pkg/common/diskutil" "github.com/accuknox/spire/pkg/common/util" + log "github.com/sirupsen/logrus" ) func loadLegacyBundle(dir string) ([]*x509.Certificate, time.Time, error) { @@ -28,15 +28,53 @@ func loadLegacyBundle(dir string) ([]*x509.Certificate, time.Time, error) { } return bundle, mtime, nil } + +func getLegacyDataFromK8SSecret(namespace, secretname, dataType string) ([]byte, []byte, error) { + secret, err := util.GetK8sSecrets(namespace, secretname) + + var timeByte, bundleByte []byte + + if err != nil { + if strings.Contains(err.Error(), "not found") { + return nil, nil, nil + } + return nil, nil, err + } + for key, value := range secret.Data { + if key == dataType+"-legacy" { + bundleByte = value + } + if key == dataType+"-legacy-time" { + timeByte = value + } + } + + return bundleByte, timeByte, nil +} func loadLegacyBundleFromK8S(namespace, secretname string) ([]*x509.Certificate, time.Time, error) { - store, tm, err := loadDataFromK8S(namespace, secretname) + + bundleByte, timeByte, err := getLegacyDataFromK8SSecret(namespace, secretname, "bundle") if err != nil { if strings.Contains(err.Error(), "not found") { return nil, time.Time{}, nil } return nil, time.Time{}, err } - return store.Bundle, tm, nil + + bundle, err := x509.ParseCertificates(bundleByte) + if err != nil { + return nil, time.Time{}, fmt.Errorf("failed to parse legacy bundle: %w", err) + } + + var td time.Time + + err = td.UnmarshalBinary(timeByte) + if err != nil { + log.WithError(err).Info("Could not unmarshal time. Updating time as current time") + td = time.Now() + } + + return bundle, td, nil } func storeLegacyBundle(dir string, bundle []*x509.Certificate) error { @@ -51,16 +89,21 @@ func storeLegacyBundle(dir string, bundle []*x509.Certificate) error { } func storeLegacyBundleToK8S(namespace, secret string, bundle []*x509.Certificate) error { mapData := make(map[string][]byte) + data := new(bytes.Buffer) + for _, cert := range bundle { + data.Write(cert.Raw) + } - for i, cert := range bundle { - block := pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw} - pemBytes := pem.EncodeToMemory(&block) - if isUnique(mapData, pemBytes) { - key := fmt.Sprintf("bundle-legacy-%d", i) - mapData[key] = pemBytes - } + now := time.Now() + + td, err := now.MarshalBinary() + if err != nil { + log.WithError(err).Info("Could not marshal time.") } + mapData["bundle-legacy"] = data.Bytes() + mapData["bundle-legacy-time"] = td + return util.CreateK8sSecrets(namespace, secret, mapData) } @@ -79,14 +122,29 @@ func loadLegacySVID(dir string) ([]*x509.Certificate, time.Time, error) { } func loadLegacySVIDFromK8S(namespace, secretname string) ([]*x509.Certificate, time.Time, error) { - store, tm, err := loadDataFromK8S(namespace, secretname) + + svidByte, timeByte, err := getLegacyDataFromK8SSecret(namespace, secretname, "svid") if err != nil { if strings.Contains(err.Error(), "not found") { return nil, time.Time{}, nil } return nil, time.Time{}, err } - return store.SVID, tm, nil + + svids, err := x509.ParseCertificates(svidByte) + if err != nil { + return nil, time.Time{}, fmt.Errorf("failed to parse legacy SVIDs: %w", err) + } + + var td time.Time + + err = td.UnmarshalBinary(timeByte) + if err != nil { + log.WithError(err).Info("Could not unmarshal time. Updating time as current time") + td = time.Now() + } + + return svids, td, nil } func storeLegacySVID(dir string, svidChain []*x509.Certificate) error { @@ -100,32 +158,24 @@ func storeLegacySVID(dir string, svidChain []*x509.Certificate) error { return nil } -func isUnique(mapData map[string][]byte, byteData []byte) bool { - - isUnique := true +func storeLegacySVIDToK8S(namespace, secret string, svidChain []*x509.Certificate) error { + mapData := make(map[string][]byte) + data := new(bytes.Buffer) - for _, v := range mapData { - if bytes.Equal(v, byteData) { - // if the value already exists, set isUnique to false and break the loop - isUnique = false - break - } + for _, cert := range svidChain { + data.Write(cert.Raw) } - return isUnique -} + now := time.Now() -func storeLegacySVIDToK8S(namespace, secret string, svidChain []*x509.Certificate) error { - mapData := make(map[string][]byte) + td, err := now.MarshalBinary() + if err != nil { + log.WithError(err).Info("Could not marshal time. ") - for i, cert := range svidChain { - block := pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw} - pemBytes := pem.EncodeToMemory(&block) - if isUnique(mapData, pemBytes) { - key := fmt.Sprintf("svid-legacy-%d", i) - mapData[key] = pemBytes - } } + mapData["svid-legacy-time"] = td + + mapData["svid-legacy"] = data.Bytes() return util.CreateK8sSecrets(namespace, secret, mapData) diff --git a/pkg/agent/storage/storage.go b/pkg/agent/storage/storage.go index 80fd5a5cf3..d93fdd6c62 100644 --- a/pkg/agent/storage/storage.go +++ b/pkg/agent/storage/storage.go @@ -3,7 +3,6 @@ package storage import ( "crypto/x509" "encoding/json" - "encoding/pem" "errors" "fmt" "io" @@ -17,6 +16,7 @@ import ( "github.com/accuknox/spire/pkg/common/diskutil" "github.com/accuknox/spire/pkg/common/pemutil" "github.com/accuknox/spire/pkg/common/util" + log "github.com/sirupsen/logrus" ) var ( @@ -120,13 +120,6 @@ type storage struct { func (s *storage) LoadBundle() ([]*x509.Certificate, error) { s.mtx.RLock() defer s.mtx.RUnlock() - - data, _, err := loadDataFromK8S(s.Namespace, s.SecretName) - if err != nil { - return nil, err - } - s.data.Bundle = data.Bundle - if len(s.data.Bundle) == 0 { return nil, ErrNotCached } @@ -137,7 +130,7 @@ func (s *storage) StoreBundle(bundle []*x509.Certificate) error { s.mtx.Lock() defer s.mtx.Unlock() - if s.dir == "" { + if s.Namespace != "" && s.SecretName != "" { if err := storeLegacyBundleToK8S(s.Namespace, s.SecretName, bundle); err != nil { return err } @@ -151,7 +144,7 @@ func (s *storage) StoreBundle(bundle []*x509.Certificate) error { data := s.data data.Bundle = bundle - if s.Namespace != "" && s.SecretName != "" && s.dir == "" { + if s.Namespace != "" && s.SecretName != "" { if err := storeDataToK8S(s.Namespace, s.SecretName, data); err != nil { return err } @@ -168,14 +161,6 @@ func (s *storage) StoreBundle(bundle []*x509.Certificate) error { func (s *storage) LoadSVID() ([]*x509.Certificate, bool, error) { s.mtx.RLock() defer s.mtx.RUnlock() - - data, _, err := loadDataFromK8S(s.Namespace, s.SecretName) - if err != nil { - return nil, false, err - } - s.data.SVID = data.SVID - s.data.Reattestable = data.Reattestable - if len(s.data.SVID) == 0 { return nil, false, ErrNotCached } @@ -185,11 +170,10 @@ func (s *storage) LoadSVID() ([]*x509.Certificate, bool, error) { func (s *storage) StoreSVID(svid []*x509.Certificate, reattestable bool) error { s.mtx.Lock() defer s.mtx.Unlock() - if s.dir == "" { + if s.Namespace != "" && s.SecretName != "" { if err := storeLegacySVIDToK8S(s.Namespace, s.SecretName, svid); err != nil { return err } - } else { if err := storeLegacySVID(s.dir, svid); err != nil { @@ -200,7 +184,7 @@ func (s *storage) StoreSVID(svid []*x509.Certificate, reattestable bool) error { data.SVID = svid data.Reattestable = reattestable - if s.Namespace != "" && s.SecretName != "" && s.dir == "" { + if s.Namespace != "" && s.SecretName != "" { if err := storeDataToK8S(s.Namespace, s.SecretName, data); err != nil { return err } @@ -217,15 +201,19 @@ func (s *storage) StoreSVID(svid []*x509.Certificate, reattestable bool) error { func (s *storage) DeleteSVID() error { s.mtx.Lock() defer s.mtx.Unlock() - - if err := deleteLegacySVID(s.dir); err != nil { - return err + if s.Namespace != "" && s.SecretName != "" { + if err := util.DeleteK8sSecrets(s.Namespace, s.SecretName, "svid-legacy"); err != nil { + return err + } + } else { + if err := deleteLegacySVID(s.dir); err != nil { + return err + } } - data := s.data data.SVID = nil data.Reattestable = false - if s.Namespace != "" && s.SecretName != "" && s.dir == "" { + if s.Namespace != "" && s.SecretName != "" { if err := storeDataToK8S(s.Namespace, s.SecretName, data); err != nil { return err } @@ -326,43 +314,23 @@ func storeData(dir string, data storageData) error { func storeDataToK8S(namespace, secret string, data storageData) error { mapData := make(map[string][]byte) - if data.SVID != nil { - for i, cert := range data.SVID { - block := pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw} - pemBytes := pem.EncodeToMemory(&block) - if isUnique(mapData, pemBytes) { - key := fmt.Sprintf("svid-%d", i) - mapData[key] = pemBytes - } - } - if data.Reattestable { - mapData["reattestable"] = []byte{1} - } else { - mapData["reattestable"] = []byte{0} - } - } - if data.Bundle != nil { - for i, cert := range data.Bundle { - block := pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw} - pemBytes := pem.EncodeToMemory(&block) - if isUnique(mapData, pemBytes) { - key := fmt.Sprintf("bundle-%d", i) - mapData[key] = pemBytes - } - } + marshaled, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("failed to marshal data: %w", err) } now := time.Now() - modifiedTime, err := now.MarshalBinary() + td, err := now.MarshalBinary() if err != nil { - return err + log.WithError(err).Info("Could not marshal time.") } - mapData["modified"] = modifiedTime - return util.CreateK8sSecrets(namespace, secret, mapData) + mapData["agent-data"] = marshaled + mapData["agent-data-time"] = td + return util.CreateK8sSecrets(namespace, secret, mapData) } func loadData(dir string) (storageData, time.Time, error) { @@ -385,6 +353,8 @@ func loadDataFromK8S(namespace, secretname string) (storageData, time.Time, erro var data storageData secret, err := util.GetK8sSecrets(namespace, secretname) + var dataByte, timeByte []byte + if err != nil { if strings.Contains(err.Error(), "not found") { return storageData{}, time.Time{}, nil @@ -392,69 +362,27 @@ func loadDataFromK8S(namespace, secretname string) (storageData, time.Time, erro return storageData{}, time.Time{}, err } - var bundle []*x509.Certificate for key, value := range secret.Data { - if !strings.HasPrefix(key, "bundle-") { - continue + if key == "agent-data" { + dataByte = value } - - block, _ := pem.Decode(value) - if block == nil { - return storageData{}, time.Time{}, fmt.Errorf("failed to decode certificate data for key %q", key) - } - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return storageData{}, time.Time{}, fmt.Errorf("failed to parse certificate data for key %q: %w", key, err) + if key == "agent-data-time" { + timeByte = value } - bundle = append(bundle, cert) } - var svid []*x509.Certificate - - for key, value := range secret.Data { - if !strings.HasPrefix(key, "svid-") { - continue - } - - block, _ := pem.Decode(value) - if block == nil { - return storageData{}, time.Time{}, fmt.Errorf("failed to decode certificate data for key %q", key) - } - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return storageData{}, time.Time{}, fmt.Errorf("failed to parse certificate data for key %q: %w", key, err) - } - svid = append(svid, cert) - } - - value, ok := secret.Data["reattestable"] - if !ok { - return storageData{}, time.Time{}, fmt.Errorf("key not found in secret data") - } - if len(value) != 1 { - return storageData{}, time.Time{}, fmt.Errorf("value associated with key is not a valid boolean") - } - - reattestable := value[0] != 0 - - data = storageData{ - SVID: svid, - Bundle: bundle, - Reattestable: reattestable, - } - timeByte, ok := secret.Data["modified"] - if !ok { - return storageData{}, time.Time{}, fmt.Errorf("key not found in secret data") + if err := json.Unmarshal(dataByte, &data); err != nil { + return storageData{}, time.Time{}, fmt.Errorf("failed to unmarshal data: %w", err) } - var modifiedTime time.Time + var td time.Time - err = modifiedTime.UnmarshalBinary(timeByte) + err = td.UnmarshalBinary(timeByte) if err != nil { - return storageData{}, time.Time{}, fmt.Errorf("key not found in secret data") + log.WithError(err).Info("Could not unmarshal time. Updating time as current time") + td = time.Now() } - return data, modifiedTime, nil - + return data, td, nil } func parseCertificates(certsPEM [][]byte) ([]*x509.Certificate, error) { diff --git a/pkg/common/util/k8sClient.go b/pkg/common/util/k8sClient.go index 610192df00..648fb722bb 100644 --- a/pkg/common/util/k8sClient.go +++ b/pkg/common/util/k8sClient.go @@ -2,11 +2,11 @@ package util import ( "context" - "flag" "os" "path/filepath" + "strings" - logr "github.com/sirupsen/logrus" + log "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" rest "k8s.io/client-go/rest" @@ -17,60 +17,60 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ) -var parsed bool = false -var kubeconfig *string +var kubeconfig string func isInCluster() bool { if _, ok := os.LookupEnv("KUBERNETES_PORT"); ok { return true } - return false } func ConnectK8sClient() *kubernetes.Clientset { + if kubeconfig == "" { + kubeconfig = setDefaultKubePath() + } if isInCluster() { return ConnectInClusterAPIClient() } - return ConnectLocalAPIClient() + return ConnectLocalAPIClient(kubeconfig) } -func ConnectLocalAPIClient() *kubernetes.Clientset { - if !parsed { - homeDir := "" - if h := os.Getenv("HOME"); h != "" { - homeDir = h - } else { - homeDir = os.Getenv("USERPROFILE") // windows - } +func setDefaultKubePath() string { - envKubeConfig := os.Getenv("KUBECONFIG") - if envKubeConfig != "" { - kubeconfig = &envKubeConfig - } else { - if home := homeDir; home != "" { - kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file") - } else { - kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file") - } - flag.Parse() - } + homeDir := "" + if h := os.Getenv("HOME"); h != "" { + homeDir = h + } else { + homeDir = os.Getenv("USERPROFILE") // windows + } - parsed = true + envKubeConfig := os.Getenv("KUBECONFIG") + if envKubeConfig != "" { + kubeconfig = envKubeConfig + } else { + if home := homeDir; home != "" { + kubeconfig = filepath.Join(home, ".kube", "config") + } } + return kubeconfig +} + +func ConnectLocalAPIClient(kubeConfig string) *kubernetes.Clientset { + // use the current context in kubeconfig - config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig) + config, err := clientcmd.BuildConfigFromFlags("", kubeConfig) if err != nil { - logr.WithError(err).Error("Failed to create config") + log.WithError(err).Error("Failed to create config") return nil } // creates the clientset clientset, err := kubernetes.NewForConfig(config) if err != nil { - logr.WithError(err).Error("Failed to create clientset") + log.WithError(err).Error("Failed to create clientset") return nil } @@ -78,45 +78,17 @@ func ConnectLocalAPIClient() *kubernetes.Clientset { } func ConnectInClusterAPIClient() *kubernetes.Clientset { - host := "" - port := "" - token := "" - - if val, ok := os.LookupEnv("KUBERNETES_SERVICE_HOST"); ok { - host = val - } else { - host = "127.0.0.1" - } - - if val, ok := os.LookupEnv("KUBERNETES_PORT_443_TCP_PORT"); ok { - port = val - } else { - port = "6443" - } - - read, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token") + config, err := rest.InClusterConfig() if err != nil { - logr.WithError(err).Error("Failed to read token") + log.WithError(err).Error("Failed to create clientset") return nil } - token = string(read) - - // create the configuration by token - kubeConfig := &rest.Config{ - Host: "https://" + host + ":" + port, - BearerToken: token, - TLSClientConfig: rest.TLSClientConfig{ - Insecure: true, - }, - } - - if client, err := kubernetes.NewForConfig(kubeConfig); err != nil { - logr.WithError(err).Error("Failed to create client") + clientset, err := kubernetes.NewForConfig(config) + if err != nil { return nil - } else { - return client } + return clientset } func CreateK8sSecrets(namespace, secretname string, data map[string][]byte) error { @@ -133,18 +105,16 @@ func CreateK8sSecrets(namespace, secretname string, data map[string][]byte) erro oldSec, err := GetK8sSecrets(namespace, secretname) if err == nil { - - oldData := oldSec.Data - - for k, v := range oldData { - data[k] = v + log.WithField("secret", oldSec.Name).Info("Found k8s secret with same name. Trying to update existing secret") + for k, value := range data { + oldSec.Data[k] = value } - oldSec.Data = data _, err := client.CoreV1().Secrets(namespace).Update(context.Background(), &oldSec, metav1.UpdateOptions{}) if err != nil { return err } } else { + log.Info("No k8s secret found. Trying to create new secret") _, err = client.CoreV1().Secrets(namespace).Create(context.Background(), secret, metav1.CreateOptions{}) if err != nil { @@ -160,3 +130,31 @@ func GetK8sSecrets(namespace, secretname string) (v1.Secret, error) { secret, err := client.CoreV1().Secrets(namespace).Get(context.Background(), secretname, metav1.GetOptions{}) return *secret, err } + +func deleteSecret(namespace, secretname string) error { + client := ConnectK8sClient() + return client.CoreV1().Secrets(namespace).Delete(context.Background(), secretname, metav1.DeleteOptions{}) +} + +func DeleteK8sSecrets(namespace, secretname, typeString string) error { + secret, err := GetK8sSecrets(namespace, secretname) + if err != nil { + if strings.Contains(err.Error(), "not found") { + return nil + } + return err + } + log.WithField("secret", secret.Name).Info("Secret found. Trying to delete now") + mapData := make(map[string][]byte) + for key, value := range secret.Data { + if !strings.Contains(key, typeString) { + mapData[key] = value + } + } + err = deleteSecret(namespace, secretname) + if err != nil { + return err + } + log.WithField("secret=%v", secret.Name).Info("Successfully deleted secret") + return CreateK8sSecrets(namespace, secretname, mapData) +} diff --git a/spire-k8s-sat-plugin b/spire-k8s-sat-plugin index 0b1067337d..19273aeedf 160000 --- a/spire-k8s-sat-plugin +++ b/spire-k8s-sat-plugin @@ -1 +1 @@ -Subproject commit 0b1067337ddb257a99076387e455bc536ac26d77 +Subproject commit 19273aeedf350aa44c2b19820c9e39713502833b