diff --git a/pkg/apis/ctrlmesh/types.go b/pkg/apis/ctrlmesh/types.go index f9375d0..3979819 100644 --- a/pkg/apis/ctrlmesh/types.go +++ b/pkg/apis/ctrlmesh/types.go @@ -31,11 +31,13 @@ const ( // Labels const ( - CtrlmeshControlPrefix = "ctrlmesh.kusionstack.io/" - CtrlmeshIgnoreWebhookLabel = "ctrlmesh.kusionstack.io/ignore-webhook" - CtrlmeshIgnoreValidateLabel = "ctrlmesh.kusionstack.io/ignore-validate" - CtrlmeshDefaultReplicasLabel = "ctrlmesh.kusionstack.io/default-replicas" - CtrlmeshEnableProxyLabel = "ctrlmesh.kusionstack.io/enable-proxy" + CtrlmeshControlPrefix = "ctrlmesh.kusionstack.io/" + CtrlmeshIgnoreWebhookLabel = "ctrlmesh.kusionstack.io/ignore-webhook" + CtrlmeshIgnoreValidateLabel = "ctrlmesh.kusionstack.io/ignore-validate" + CtrlmeshDefaultReplicasLabel = "ctrlmesh.kusionstack.io/default-replicas" + CtrlmeshEnableProxyLabel = "ctrlmesh.kusionstack.io/enable-proxy" + CtrlmeshEnableIptableMode = "ctrlmesh.kusionstack.io/enable-iptables" + CtrlmeshAutoShardingRootLabel = "ctrlmesh.kusionstack.io/auto-sharding-root" CtrlmeshInRollingLabel = "ctrlmesh.kusionstack.io/rolling" CtrlmeshDisableFakeKubeconfigArgLabel = "ctrlmesh.kusionstack.io/disable-fake-kubeconfig-arg" diff --git a/pkg/cmd/proxy/main.go b/pkg/cmd/proxy/main.go index f91ee91..f30d2d2 100644 --- a/pkg/cmd/proxy/main.go +++ b/pkg/cmd/proxy/main.go @@ -20,6 +20,7 @@ import ( "context" "flag" "fmt" + "io/ioutil" "net" "net/http" "os" @@ -35,7 +36,6 @@ import ( "github.com/KusionStack/controller-mesh/pkg/apis/ctrlmesh/constants" "github.com/KusionStack/controller-mesh/pkg/client" - proxyapiserver "github.com/KusionStack/controller-mesh/pkg/proxy/apiserver" proxycache "github.com/KusionStack/controller-mesh/pkg/proxy/cache" "github.com/KusionStack/controller-mesh/pkg/proxy/circuitbreaker" @@ -56,6 +56,8 @@ var ( webhookCertDir = flag.String(constants.ProxyWebhookCertDirFlag, "", "The directory where the webhook certs generated or mounted.") proxyIptablePort = flag.Int(constants.ProxyIptablesFlag, constants.ProxyIptablesPort, "port that http-tproxy listens on") + + enableIpTable = os.Getenv(constants.EnvIPTable) == "true" ) func main() { @@ -66,8 +68,18 @@ func main() { klog.Fatalf("Environment %s=%s %s=%s not exist.", constants.EnvPodNamespace, os.Getenv(constants.EnvPodNamespace), constants.EnvPodName, os.Getenv(constants.EnvPodName)) } - cfg := ctrl.GetConfigOrDie() - cfg.UserAgent = "ctrlmesh" + var cfg *rest.Config + + if enableIpTable { + var err error + cfg, err = getRestConfig() + if err != nil { + klog.Fatalf("Failed to get rest config: %v", err) + } + } else { + cfg = ctrl.GetConfigOrDie() + } + //cfg.UserAgent = "ctrlmesh" if err := client.NewRegistry(cfg); err != nil { klog.Fatalf("Failed to new client registry: %v", err) } @@ -165,3 +177,41 @@ func serveHTTP(ctx context.Context, readyHandler *healthz.Handler) { klog.Fatalf("Serve HTTP shutting down on :%d: %v", *metricsHealthPort, err) } } + +func getRestConfig() (*rest.Config, error) { + const ( + tokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" + //rootCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/..data/ca.crt" + ) + host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT") + if len(host) == 0 || len(port) == 0 { + return nil, rest.ErrNotInCluster + } + + token, err := ioutil.ReadFile(tokenFile) + if err != nil { + return nil, err + } + + tlsClientConfig := rest.TLSClientConfig{Insecure: true} + + //if _, err := certutil.NewPool(rootCAFile); err != nil { + // klog.Errorf("Expected to load root CA config from %s, but got err: %v", rootCAFile, err) + //} else { + // tlsClientConfig.CAFile = rootCAFile + //} + + cfg := &rest.Config{ + // TODO: switch to using cluster DNS. + Host: "https://" + net.JoinHostPort(host, port), + TLSClientConfig: tlsClientConfig, + BearerToken: string(token), + BearerTokenFile: tokenFile, + + Burst: 3000, + QPS: 2000.0, + } + klog.V(3).Infof("Starting with rest config: %v", utils.DumpJSON(cfg)) + + return cfg, nil +} diff --git a/pkg/webhook/pod/injector.go b/pkg/webhook/pod/injector.go index db5c7bb..50de8f7 100644 --- a/pkg/webhook/pod/injector.go +++ b/pkg/webhook/pod/injector.go @@ -131,7 +131,7 @@ func (h *MutatingHandler) injectByShardingConfig(ctx context.Context, pod *v1.Po if *proxyImage == "" { return fmt.Errorf("the images for ControllerMesh init or proxy container have not set in args") } - + enableIpTable := pod.Labels[ctrlmesh.CtrlmeshEnableIptableMode] == "true" imagePullPolicy := v1.PullAlways if *proxyImagePullPolicy == string(v1.PullIfNotPresent) { imagePullPolicy = v1.PullIfNotPresent @@ -177,6 +177,13 @@ func (h *MutatingHandler) injectByShardingConfig(ctx context.Context, pod *v1.Po }, } + if enableIpTable { + proxyContainer.Env = append(proxyContainer.Env, v1.EnvVar{ + Name: constants.EnvIPTable, + Value: "true", + }) + } + if val, ok := pod.Annotations[ctrlmesh.CtrlmeshProxyContainerResourceAnno]; ok { req := &v1.ResourceRequirements{} if err := json.Unmarshal([]byte(val), req); err != nil { @@ -213,15 +220,6 @@ func (h *MutatingHandler) injectByShardingConfig(ctx context.Context, pod *v1.Po proxyContainer.Env = append(proxyContainer.Env, apiserverHostPortEnvs...) } - ipTableEnvs := getEnv(pod, constants.EnvIPTable) - enableIpTable := false - if len(ipTableEnvs) > 0 { - initContainer.Env = append(initContainer.Env, ipTableEnvs...) - //proxyContainer.Env = append(proxyContainer.Env, ipTableEnvs...) - if ipTableEnvs[0].Value == "true" { - enableIpTable = true - } - } if !enableIpTable { if err := h.applyFakeConfigMap(pod); err != nil { return err @@ -271,7 +269,7 @@ func (h *MutatingHandler) injectByShardingConfig(ctx context.Context, pod *v1.Po proxyContainer.VolumeMounts = append(proxyContainer.VolumeMounts, certVolumeMounts[0]) } } - if *initImage != "" { + if enableIpTable && *initImage != "" { pod.Spec.InitContainers = append([]v1.Container{*initContainer}, pod.Spec.InitContainers...) } if pod.Labels == nil {