diff --git a/Makefile b/Makefile index 0428708fe5e..13252fb69d5 100644 --- a/Makefile +++ b/Makefile @@ -520,6 +520,7 @@ kind-install-chart: kind-load-image kind-untaint-control-plane --set global.images.kubeovn.tag=$(VERSION) \ --set networking.NET_STACK=$(shell echo $${NET_STACK:-ipv4} | sed 's/^dual$$/dual_stack/') \ --set networking.ENABLE_SSL=$(shell echo $${ENABLE_SSL:-false}) \ + --set func.SECURE_SERVING=$(shell echo $${SECURE_SERVING:-false}) \ --set func.ENABLE_BIND_LOCAL_IP=$(shell echo $${ENABLE_BIND_LOCAL_IP:-true}) \ --set func.ENABLE_OVN_IPSEC=$(shell echo $${ENABLE_OVN_IPSEC:-false}) \ --set func.ENABLE_ANP=$(shell echo $${ENABLE_ANP:-false}) \ @@ -535,6 +536,7 @@ kind-upgrade-chart: kind-load-image --set global.images.kubeovn.tag=$(VERSION) \ --set networking.NET_STACK=$(shell echo $${NET_STACK:-ipv4} | sed 's/^dual$$/dual_stack/') \ --set networking.ENABLE_SSL=$(shell echo $${ENABLE_SSL:-false}) \ + --set func.SECURE_SERVING=$(shell echo $${SECURE_SERVING:-false}) \ --set func.ENABLE_BIND_LOCAL_IP=$(shell echo $${ENABLE_BIND_LOCAL_IP:-true}) \ --set func.ENABLE_OVN_IPSEC=$(shell echo $${ENABLE_OVN_IPSEC:-false}) \ --set func.ENABLE_ANP=$(shell echo $${ENABLE_ANP:-false}) \ diff --git a/charts/kube-ovn/templates/controller-deploy.yaml b/charts/kube-ovn/templates/controller-deploy.yaml index 598a22451cf..095f67eeb8d 100644 --- a/charts/kube-ovn/templates/controller-deploy.yaml +++ b/charts/kube-ovn/templates/controller-deploy.yaml @@ -191,21 +191,17 @@ spec: - mountPath: /var/run/tls name: kube-ovn-tls readinessProbe: - exec: - command: - - /kube-ovn/kube-ovn-healthcheck - - --port=10660 - - --tls={{- .Values.func.SECURE_SERVING }} - - --enable-metrics={{- .Values.networking.ENABLE_METRICS }} + httpGet: + port: 10660 + path: /readyz + scheme: '{{ ternary "HTTPS" "HTTP" .Values.func.SECURE_SERVING }}' periodSeconds: 3 timeoutSeconds: 5 livenessProbe: - exec: - command: - - /kube-ovn/kube-ovn-healthcheck - - --port=10660 - - --tls={{- .Values.func.SECURE_SERVING }} - - --enable-metrics={{- .Values.networking.ENABLE_METRICS }} + httpGet: + port: 10660 + path: /livez + scheme: '{{ ternary "HTTPS" "HTTP" .Values.func.SECURE_SERVING }}' initialDelaySeconds: 300 periodSeconds: 7 failureThreshold: 5 diff --git a/charts/kube-ovn/templates/monitor-deploy.yaml b/charts/kube-ovn/templates/monitor-deploy.yaml index b452759b655..e4c3322c332 100644 --- a/charts/kube-ovn/templates/monitor-deploy.yaml +++ b/charts/kube-ovn/templates/monitor-deploy.yaml @@ -126,24 +126,20 @@ spec: initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - exec: - command: - - /kube-ovn/kube-ovn-healthcheck - - --port=10661 - - --tls={{- .Values.func.SECURE_SERVING }} - - --enable-metrics={{- .Values.networking.ENABLE_METRICS }} + httpGet: + port: 10661 + path: /livez + scheme: '{{ ternary "HTTPS" "HTTP" .Values.func.SECURE_SERVING }}' timeoutSeconds: 5 readinessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - exec: - command: - - /kube-ovn/kube-ovn-healthcheck - - --port=10661 - - --tls={{- .Values.func.SECURE_SERVING }} - - --enable-metrics={{- .Values.networking.ENABLE_METRICS }} + httpGet: + port: 10661 + path: /readyz + scheme: '{{ ternary "HTTPS" "HTTP" .Values.func.SECURE_SERVING }}' timeoutSeconds: 5 nodeSelector: kubernetes.io/os: "linux" diff --git a/charts/kube-ovn/templates/ovncni-ds.yaml b/charts/kube-ovn/templates/ovncni-ds.yaml index f08e4f73cc6..9457f0d0684 100644 --- a/charts/kube-ovn/templates/ovncni-ds.yaml +++ b/charts/kube-ovn/templates/ovncni-ds.yaml @@ -194,24 +194,20 @@ spec: failureThreshold: 3 periodSeconds: 7 successThreshold: 1 - exec: - command: - - /kube-ovn/kube-ovn-healthcheck - - --port=10665 - - --tls={{- .Values.func.SECURE_SERVING }} - - --enable-metrics={{- .Values.networking.ENABLE_METRICS }} + httpGet: + port: 10665 + path: /readyz + scheme: '{{ ternary "HTTPS" "HTTP" .Values.func.SECURE_SERVING }}' timeoutSeconds: 5 livenessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - exec: - command: - - /kube-ovn/kube-ovn-healthcheck - - --port=10665 - - --tls={{- .Values.func.SECURE_SERVING }} - - --enable-metrics={{- .Values.networking.ENABLE_METRICS }} + httpGet: + port: 10665 + path: /livez + scheme: '{{ ternary "HTTPS" "HTTP" .Values.func.SECURE_SERVING }}' timeoutSeconds: 5 resources: requests: diff --git a/cmd/cmdmain.go b/cmd/cmdmain.go index 064e3057ab2..b26978411ab 100644 --- a/cmd/cmdmain.go +++ b/cmd/cmdmain.go @@ -11,7 +11,6 @@ import ( "k8s.io/klog/v2" - "github.com/kubeovn/kube-ovn/cmd/health_check" "github.com/kubeovn/kube-ovn/cmd/ovn_ic_controller" "github.com/kubeovn/kube-ovn/cmd/ovn_leader_checker" "github.com/kubeovn/kube-ovn/cmd/ovn_monitor" @@ -24,7 +23,6 @@ const ( CmdMonitor = "kube-ovn-monitor" CmdSpeaker = "kube-ovn-speaker" CmdWebhook = "kube-ovn-webhook" - CmdHealthCheck = "kube-ovn-healthcheck" CmdOvnLeaderChecker = "kube-ovn-leader-checker" CmdOvnICController = "kube-ovn-ic-controller" ) @@ -97,8 +95,6 @@ func main() { speaker.CmdMain() case CmdWebhook: webhook.CmdMain() - case CmdHealthCheck: - health_check.CmdMain() case CmdOvnLeaderChecker: ovn_leader_checker.CmdMain() case CmdOvnICController: diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go index 0be8165f26f..6ef758a2759 100644 --- a/cmd/controller/controller.go +++ b/cmd/controller/controller.go @@ -97,10 +97,14 @@ func CmdMain() { if err != nil { util.LogFatalAndExit(err, "failed to listen on %s", util.JoinHostPort(metricsAddr, config.PprofPort)) } + mux := http.NewServeMux() + mux.HandleFunc("/healthz", util.DefaultHealthCheckHandler) + mux.HandleFunc("/livez", util.DefaultHealthCheckHandler) + mux.HandleFunc("/readyz", util.DefaultHealthCheckHandler) svr := manager.Server{ Name: "health-check", Server: &http.Server{ - Handler: http.NewServeMux(), + Handler: mux, MaxHeaderBytes: 1 << 20, IdleTimeout: 90 * time.Second, ReadHeaderTimeout: 32 * time.Second, diff --git a/cmd/daemon/cniserver.go b/cmd/daemon/cniserver.go index 9bad4c0b5e4..84778af4a09 100644 --- a/cmd/daemon/cniserver.go +++ b/cmd/daemon/cniserver.go @@ -159,10 +159,14 @@ func main() { if err != nil { util.LogFatalAndExit(err, "failed to listen on %s", util.JoinHostPort(addr, config.PprofPort)) } + mux := http.NewServeMux() + mux.HandleFunc("/healthz", util.DefaultHealthCheckHandler) + mux.HandleFunc("/livez", util.DefaultHealthCheckHandler) + mux.HandleFunc("/readyz", util.DefaultHealthCheckHandler) svr := manager.Server{ Name: "health-check", Server: &http.Server{ - Handler: http.NewServeMux(), + Handler: mux, MaxHeaderBytes: 1 << 20, IdleTimeout: 90 * time.Second, ReadHeaderTimeout: 32 * time.Second, diff --git a/cmd/health_check/health_check.go b/cmd/health_check/health_check.go deleted file mode 100644 index ec51bb4da81..00000000000 --- a/cmd/health_check/health_check.go +++ /dev/null @@ -1,58 +0,0 @@ -package health_check - -import ( - "flag" - "net" - "os" - "time" - - "github.com/spf13/pflag" - "k8s.io/klog/v2" - - "github.com/kubeovn/kube-ovn/pkg/util" -) - -func CmdMain() { - port := pflag.Int32("port", 0, "Target port") - tls := pflag.Bool("tls", false, "Dial the server with TLS") - enableMetrics := pflag.Bool("enable-metrics", true, "Whether to support metrics query") - - klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) - klog.InitFlags(klogFlags) - - // sync the glog and klog flags. - pflag.CommandLine.VisitAll(func(f1 *pflag.Flag) { - f2 := klogFlags.Lookup(f1.Name) - if f2 != nil { - value := f1.Value.String() - if err := f2.Value.Set(value); err != nil { - util.LogFatalAndExit(err, "failed to set pflag") - } - } - }) - - pflag.CommandLine.AddGoFlagSet(klogFlags) - pflag.CommandLine.AddGoFlagSet(flag.CommandLine) - pflag.Parse() - - if *port <= 0 { - klog.Errorf("invalid port: %d", port) - os.Exit(1) - } - - ip := os.Getenv("POD_IP") - if net.ParseIP(ip) == nil { - klog.Errorf("invalid ip: %q", ip) - os.Exit(1) - } - - addr := util.JoinHostPort(ip, *port) - if *enableMetrics && *tls { - addr = "tls://" + addr - } else { - addr = "tcp://" + addr - } - if err := util.DialTCP(addr, time.Second, false); err != nil { - util.LogFatalAndExit(err, "failed to probe the socket") - } -} diff --git a/cmd/ovn_monitor/ovn_monitor.go b/cmd/ovn_monitor/ovn_monitor.go index 06da89d0fbd..46c6931d570 100644 --- a/cmd/ovn_monitor/ovn_monitor.go +++ b/cmd/ovn_monitor/ovn_monitor.go @@ -51,10 +51,14 @@ func CmdMain() { if err != nil { util.LogFatalAndExit(err, "failed to listen on %s", util.JoinHostPort(metricsAddr, config.MetricsPort)) } + mux := http.NewServeMux() + mux.HandleFunc("/healthz", util.DefaultHealthCheckHandler) + mux.HandleFunc("/livez", util.DefaultHealthCheckHandler) + mux.HandleFunc("/readyz", util.DefaultHealthCheckHandler) svr := manager.Server{ Name: "health-check", Server: &http.Server{ - Handler: http.NewServeMux(), + Handler: mux, MaxHeaderBytes: 1 << 20, IdleTimeout: 90 * time.Second, ReadHeaderTimeout: 32 * time.Second, diff --git a/dist/images/Dockerfile b/dist/images/Dockerfile index 30a89c6b8a4..11ebb8f4732 100644 --- a/dist/images/Dockerfile +++ b/dist/images/Dockerfile @@ -14,7 +14,6 @@ COPY kube-ovn-controller /kube-ovn/kube-ovn-controller RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-monitor && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-webhook && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-healthcheck && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller && \ ln -s /kube-ovn/kube-ovn-controller /kube-ovn/kube-ovn-pinger && \ diff --git a/dist/images/install.sh b/dist/images/install.sh index 6c1695d49dc..2db55c7810b 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -47,6 +47,11 @@ OVSDB_CON_TIMEOUT=${OVSDB_CON_TIMEOUT:-3} OVSDB_INACTIVITY_TIMEOUT=${OVSDB_INACTIVITY_TIMEOUT:-10} ENABLE_LIVE_MIGRATION_OPTIMIZE=${ENABLE_LIVE_MIGRATION_OPTIMIZE:-true} +PROBE_HTTP_SCHEME="HTTP" +if [ "$SECURE_SERVING" = "true" ]; then + PROBE_HTTP_SCHEME="HTTPS" +fi + # debug DEBUG_WRAPPER=${DEBUG_WRAPPER:-} RUN_AS_USER=65534 # run as nobody @@ -4780,21 +4785,17 @@ spec: - mountPath: /var/run/tls name: kube-ovn-tls readinessProbe: - exec: - command: - - /kube-ovn/kube-ovn-healthcheck - - --port=10660 - - --tls=${SECURE_SERVING} - - --enable-metrics=$ENABLE_METRICS + httpGet: + port: 10660 + path: /readyz + scheme: ${PROBE_HTTP_SCHEME} periodSeconds: 3 timeoutSeconds: 5 livenessProbe: - exec: - command: - - /kube-ovn/kube-ovn-healthcheck - - --port=10660 - - --tls=${SECURE_SERVING} - - --enable-metrics=$ENABLE_METRICS + httpGet: + port: 10660 + path: /livez + scheme: ${PROBE_HTTP_SCHEME} initialDelaySeconds: 300 periodSeconds: 7 failureThreshold: 5 @@ -5002,23 +5003,19 @@ spec: initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - exec: - command: - - /kube-ovn/kube-ovn-healthcheck - - --port=10665 - - --tls=${SECURE_SERVING} - - --enable-metrics=$ENABLE_METRICS + httpGet: + port: 10665 + path: /livez + scheme: ${PROBE_HTTP_SCHEME} timeoutSeconds: 5 readinessProbe: failureThreshold: 3 periodSeconds: 7 successThreshold: 1 - exec: - command: - - /kube-ovn/kube-ovn-healthcheck - - --port=10665 - - --tls=${SECURE_SERVING} - - --enable-metrics=$ENABLE_METRICS + httpGet: + port: 10665 + path: /readyz + scheme: ${PROBE_HTTP_SCHEME} timeoutSeconds: 5 resources: requests: @@ -5347,24 +5344,20 @@ spec: initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - exec: - command: - - /kube-ovn/kube-ovn-healthcheck - - --port=10661 - - --tls=${SECURE_SERVING} - - --enable-metrics=$ENABLE_METRICS + httpGet: + port: 10661 + path: /livez + scheme: ${PROBE_HTTP_SCHEME} timeoutSeconds: 5 readinessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - exec: - command: - - /kube-ovn/kube-ovn-healthcheck - - --port=10661 - - --tls=${SECURE_SERVING} - - --enable-metrics=$ENABLE_METRICS + httpGet: + port: 10661 + path: /readyz + scheme: ${PROBE_HTTP_SCHEME} timeoutSeconds: 5 nodeSelector: kubernetes.io/os: "linux" diff --git a/go.mod b/go.mod index 958d3783b12..fcd6070d8a5 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/docker/docker v27.5.1+incompatible github.com/emicklei/go-restful/v3 v3.12.1 github.com/evanphx/json-patch/v5 v5.9.11 + github.com/go-logr/logr v1.4.2 github.com/go-logr/stdr v1.2.2 github.com/google/uuid v1.6.0 github.com/httprunner/httprunner/v4 v4.3.7-0.20240124083022-402b74876a59 @@ -118,7 +119,6 @@ require ( github.com/go-kit/kit v0.13.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect diff --git a/pkg/metrics/server.go b/pkg/metrics/server.go index ef22a0bcee0..53830f06308 100644 --- a/pkg/metrics/server.go +++ b/pkg/metrics/server.go @@ -6,13 +6,37 @@ import ( "net/http" "net/http/pprof" + "github.com/go-logr/logr" "k8s.io/client-go/rest" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + "github.com/kubeovn/kube-ovn/pkg/util" ) +func filterProvider(c *rest.Config, httpClient *http.Client) (server.Filter, error) { + return func(log logr.Logger, handler http.Handler) (http.Handler, error) { + filter, err := filters.WithAuthenticationAndAuthorization(c, httpClient) + if err != nil { + return nil, fmt.Errorf("failed to create filter: %w", err) + } + h, err := filter(log, handler) + if err != nil { + return nil, fmt.Errorf("failed to create handler: %w", err) + } + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/healthz", "/livez", "/readyz": + handler.ServeHTTP(w, req) + default: + h.ServeHTTP(w, req) + } + }), nil + }, nil +} + func Run(ctx context.Context, config *rest.Config, addr string, secureServing, withPprof bool) error { if config == nil { config = ctrl.GetConfigOrDie() @@ -28,16 +52,19 @@ func Run(ctx context.Context, config *rest.Config, addr string, secureServing, w BindAddress: addr, } if secureServing { - options.FilterProvider = filters.WithAuthenticationAndAuthorization + options.FilterProvider = filterProvider + } + options.ExtraHandlers = map[string]http.Handler{ + "/healthz": http.HandlerFunc(util.DefaultHealthCheckHandler), + "/livez": http.HandlerFunc(util.DefaultHealthCheckHandler), + "/readyz": http.HandlerFunc(util.DefaultHealthCheckHandler), } if withPprof { - options.ExtraHandlers = map[string]http.Handler{ - "/debug/pprof/": http.HandlerFunc(pprof.Index), - "/debug/pprof/cmdline": http.HandlerFunc(pprof.Cmdline), - "/debug/pprof/profile": http.HandlerFunc(pprof.Profile), - "/debug/pprof/symbol": http.HandlerFunc(pprof.Symbol), - "/debug/pprof/trace": http.HandlerFunc(pprof.Trace), - } + options.ExtraHandlers["/debug/pprof/"] = http.HandlerFunc(pprof.Index) + options.ExtraHandlers["/debug/pprof/cmdline"] = http.HandlerFunc(pprof.Cmdline) + options.ExtraHandlers["/debug/pprof/profile"] = http.HandlerFunc(pprof.Profile) + options.ExtraHandlers["/debug/pprof/symbol"] = http.HandlerFunc(pprof.Symbol) + options.ExtraHandlers["/debug/pprof/trace"] = http.HandlerFunc(pprof.Trace) } svr, err := server.NewServer(options, config, client) if err != nil { diff --git a/pkg/util/health_check.go b/pkg/util/health_check.go new file mode 100644 index 00000000000..f71989361cf --- /dev/null +++ b/pkg/util/health_check.go @@ -0,0 +1,17 @@ +package util + +import ( + "net/http" + + "k8s.io/klog/v2" +) + +func DefaultHealthCheckHandler(w http.ResponseWriter, _ *http.Request) { + if _, err := w.Write([]byte("ok")); err != nil { + klog.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) +}