diff --git a/.github/ISSUE_TEMPLATE/BUG-ISSUE.yml b/.github/ISSUE_TEMPLATE/BUG-ISSUE.yml index ff7c3caaa90..afe9da8946b 100644 --- a/.github/ISSUE_TEMPLATE/BUG-ISSUE.yml +++ b/.github/ISSUE_TEMPLATE/BUG-ISSUE.yml @@ -88,3 +88,6 @@ body: line 13 ... ``` + + The `glooctl debug` command can be used to collect state information about your system. + Consider uploading logs, YAML manfests, Envoy listeners, etc. that may be relevant. diff --git a/changelog/v1.19.0-beta2/CLI-debug-additions.yaml b/changelog/v1.19.0-beta2/CLI-debug-additions.yaml new file mode 100644 index 00000000000..2bae51f7d4a --- /dev/null +++ b/changelog/v1.19.0-beta2/CLI-debug-additions.yaml @@ -0,0 +1,5 @@ +changelog: +- type: NEW_FEATURE + description: Adds a new top level `glooctl debug` CLI command as well as subcommands which collect kubernetes, gloo controller, and envoy state information to aid in debugging failures. + issueLink: https://github.com/solo-io/solo-projects/issues/7298 + resolvesIssue: false diff --git a/devel/debugging/glooctl-debug.md b/devel/debugging/glooctl-debug.md new file mode 100644 index 00000000000..d385610177a --- /dev/null +++ b/devel/debugging/glooctl-debug.md @@ -0,0 +1,5 @@ +# Glooctl Debug Commands + +When debugging any sort of error or unexpected behavior, it may be useful to collect information about the state of your system. +The `glooctl debug` command and its subcommands can help with this. +Run `glooctl debug --help` from the command line for details. diff --git a/docs/content/reference/cli/glooctl.md b/docs/content/reference/cli/glooctl.md index 883ea8dda32..bf62cb79e4f 100644 --- a/docs/content/reference/cli/glooctl.md +++ b/docs/content/reference/cli/glooctl.md @@ -38,7 +38,7 @@ glooctl is the unified CLI for Gloo. * [glooctl completion](../glooctl_completion) - generate auto completion for your shell * [glooctl create](../glooctl_create) - Create a Gloo resource * [glooctl dashboard](../glooctl_dashboard) - Open Gloo dashboard -* [glooctl debug](../glooctl_debug) - Debug a Gloo resource (requires Gloo running on Kubernetes) +* [glooctl debug](../glooctl_debug) - Debug Gloo Gateway (requires Gloo running on Kubernetes) * [glooctl delete](../glooctl_delete) - Delete a Gloo resource * [glooctl demo](../glooctl_demo) - Demos (requires 4 tools to be installed and accessible via the PATH: glooctl, kubectl, docker, and kind.) * [glooctl edit](../glooctl_edit) - Edit a Gloo resource diff --git a/docs/content/reference/cli/glooctl_debug.md b/docs/content/reference/cli/glooctl_debug.md index 1ef7ba72acf..c15ef87064d 100644 --- a/docs/content/reference/cli/glooctl_debug.md +++ b/docs/content/reference/cli/glooctl_debug.md @@ -5,7 +5,16 @@ weight: 5 --- ## glooctl debug -Debug a Gloo resource (requires Gloo running on Kubernetes) +Debug Gloo Gateway (requires Gloo running on Kubernetes) + +### Synopsis + +Dumps Kubernetes, Gloo Gateway controller, and Envoy state information to a local directory. This is useful for debugging failures. The dump includes: +- the Kubernetes cluster state +- logs from all pods in the given namespaces +- YAML manifests of all solo.io CRs in the given namespaces +- the gloo controller logs, metrics, xds snapshot, and krt snapshot +- the envoy config dump, stats, clusters, and listeners ``` glooctl debug [flags] @@ -14,7 +23,9 @@ glooctl debug [flags] ### Options ``` - -h, --help help for debug + -d, --directory string directory to write debug info to (default "debug") + -h, --help help for debug + -N, --namespaces stringArray namespaces from which to dump logs and resources (use flag multiple times to specify multiple namespaces, e.g. '-N gloo-system -N default') (default [gloo-system]) ``` ### Options inherited from parent commands @@ -36,6 +47,5 @@ glooctl debug [flags] ### SEE ALSO * [glooctl](../glooctl) - CLI for Gloo -* [glooctl debug logs](../glooctl_debug_logs) - Debug Gloo logs (requires Gloo running on Kubernetes) -* [glooctl debug yaml](../glooctl_debug_yaml) - Dump YAML representing the current Gloo state (requires Gloo running on Kubernetes) +* [glooctl debug yaml](../glooctl_debug_yaml) - Print YAML representing the current Gloo state of a Kubernetes cluster (top level "debug" command is preferred) diff --git a/docs/content/reference/cli/glooctl_debug_logs.md b/docs/content/reference/cli/glooctl_debug_logs.md deleted file mode 100644 index a73adcb748a..00000000000 --- a/docs/content/reference/cli/glooctl_debug_logs.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -title: "glooctl debug logs" -description: "Reference for the 'glooctl debug logs' command." -weight: 5 ---- -## glooctl debug logs - -Debug Gloo logs (requires Gloo running on Kubernetes) - -``` -glooctl debug logs [flags] -``` - -### Options - -``` - --errors-only filter for error logs only - -f, --file string file to be read or written to - -h, --help help for logs - -n, --namespace string namespace for reading or writing resources (default "gloo-system") - --zip save logs to a tar file (specify location with -f) -``` - -### Options inherited from parent commands - -``` - -c, --config string set the path to the glooctl config file (default "/.gloo/glooctl-config.yaml") - --consul-address string address of the Consul server. Use with --use-consul (default "127.0.0.1:8500") - --consul-allow-stale-reads Allows reading using Consul's stale consistency mode. - --consul-datacenter string Datacenter to use. If not provided, the default agent datacenter is used. Use with --use-consul - --consul-root-key string key prefix for the Consul key-value storage. (default "gloo") - --consul-scheme string URI scheme for the Consul server. Use with --use-consul (default "http") - --consul-token string Token is used to provide a per-request ACL token which overrides the agent's default token. Use with --use-consul - -i, --interactive use interactive mode - --kube-context string kube context to use when interacting with kubernetes - --kubeconfig string kubeconfig to use, if not standard one - --use-consul use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies) -``` - -### SEE ALSO - -* [glooctl debug](../glooctl_debug) - Debug a Gloo resource (requires Gloo running on Kubernetes) - diff --git a/docs/content/reference/cli/glooctl_debug_yaml.md b/docs/content/reference/cli/glooctl_debug_yaml.md index 5795f17aa83..e05de422191 100644 --- a/docs/content/reference/cli/glooctl_debug_yaml.md +++ b/docs/content/reference/cli/glooctl_debug_yaml.md @@ -5,7 +5,7 @@ weight: 5 --- ## glooctl debug yaml -Dump YAML representing the current Gloo state (requires Gloo running on Kubernetes) +Print YAML representing the current Gloo state of a Kubernetes cluster (top level "debug" command is preferred) ``` glooctl debug yaml [flags] @@ -29,13 +29,15 @@ glooctl debug yaml [flags] --consul-root-key string key prefix for the Consul key-value storage. (default "gloo") --consul-scheme string URI scheme for the Consul server. Use with --use-consul (default "http") --consul-token string Token is used to provide a per-request ACL token which overrides the agent's default token. Use with --use-consul + -d, --directory string directory to write debug info to (default "debug") -i, --interactive use interactive mode --kube-context string kube context to use when interacting with kubernetes --kubeconfig string kubeconfig to use, if not standard one + -N, --namespaces stringArray namespaces from which to dump logs and resources (use flag multiple times to specify multiple namespaces, e.g. '-N gloo-system -N default') (default [gloo-system]) --use-consul use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies) ``` ### SEE ALSO -* [glooctl debug](../glooctl_debug) - Debug a Gloo resource (requires Gloo running on Kubernetes) +* [glooctl debug](../glooctl_debug) - Debug Gloo Gateway (requires Gloo running on Kubernetes) diff --git a/docs/content/reference/cli/glooctl_proxy.md b/docs/content/reference/cli/glooctl_proxy.md index a5118a2ddca..ad818fe3d5a 100644 --- a/docs/content/reference/cli/glooctl_proxy.md +++ b/docs/content/reference/cli/glooctl_proxy.md @@ -15,7 +15,7 @@ these commands can be used to interact directly with the Proxies Gloo is managin ``` -h, --help help for proxy - --name string the name of the proxy service/deployment to use (default "gateway-proxy") + --name string the name of the proxy pod/deployment to use -n, --namespace string namespace for reading or writing resources (default "gloo-system") --port string the name of the service port to connect to (default "http") ``` diff --git a/docs/content/reference/cli/glooctl_proxy_address.md b/docs/content/reference/cli/glooctl_proxy_address.md index b0a3b4bd653..258f1b2f5fc 100644 --- a/docs/content/reference/cli/glooctl_proxy_address.md +++ b/docs/content/reference/cli/glooctl_proxy_address.md @@ -36,7 +36,7 @@ glooctl proxy address [flags] -i, --interactive use interactive mode --kube-context string kube context to use when interacting with kubernetes --kubeconfig string kubeconfig to use, if not standard one - --name string the name of the proxy service/deployment to use (default "gateway-proxy") + --name string the name of the proxy pod/deployment to use -n, --namespace string namespace for reading or writing resources (default "gloo-system") --port string the name of the service port to connect to (default "http") --use-consul use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies) diff --git a/docs/content/reference/cli/glooctl_proxy_dump.md b/docs/content/reference/cli/glooctl_proxy_dump.md index afaea957914..fb82f7aa74f 100644 --- a/docs/content/reference/cli/glooctl_proxy_dump.md +++ b/docs/content/reference/cli/glooctl_proxy_dump.md @@ -30,7 +30,7 @@ glooctl proxy dump [flags] -i, --interactive use interactive mode --kube-context string kube context to use when interacting with kubernetes --kubeconfig string kubeconfig to use, if not standard one - --name string the name of the proxy service/deployment to use (default "gateway-proxy") + --name string the name of the proxy pod/deployment to use -n, --namespace string namespace for reading or writing resources (default "gloo-system") --port string the name of the service port to connect to (default "http") --use-consul use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies) diff --git a/docs/content/reference/cli/glooctl_proxy_logs.md b/docs/content/reference/cli/glooctl_proxy_logs.md index a8ba2a2b4dc..c73080f78fc 100644 --- a/docs/content/reference/cli/glooctl_proxy_logs.md +++ b/docs/content/reference/cli/glooctl_proxy_logs.md @@ -32,7 +32,7 @@ glooctl proxy logs [flags] -i, --interactive use interactive mode --kube-context string kube context to use when interacting with kubernetes --kubeconfig string kubeconfig to use, if not standard one - --name string the name of the proxy service/deployment to use (default "gateway-proxy") + --name string the name of the proxy pod/deployment to use -n, --namespace string namespace for reading or writing resources (default "gloo-system") --port string the name of the service port to connect to (default "http") --use-consul use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies) diff --git a/docs/content/reference/cli/glooctl_proxy_served-config.md b/docs/content/reference/cli/glooctl_proxy_served-config.md index 4348a025872..1e6779d7ca2 100644 --- a/docs/content/reference/cli/glooctl_proxy_served-config.md +++ b/docs/content/reference/cli/glooctl_proxy_served-config.md @@ -30,7 +30,7 @@ glooctl proxy served-config [flags] -i, --interactive use interactive mode --kube-context string kube context to use when interacting with kubernetes --kubeconfig string kubeconfig to use, if not standard one - --name string the name of the proxy service/deployment to use (default "gateway-proxy") + --name string the name of the proxy pod/deployment to use -n, --namespace string namespace for reading or writing resources (default "gloo-system") --port string the name of the service port to connect to (default "http") --use-consul use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies) diff --git a/docs/content/reference/cli/glooctl_proxy_snapshot.md b/docs/content/reference/cli/glooctl_proxy_snapshot.md index 6f20e07b078..f95a26e0a95 100644 --- a/docs/content/reference/cli/glooctl_proxy_snapshot.md +++ b/docs/content/reference/cli/glooctl_proxy_snapshot.md @@ -31,7 +31,7 @@ glooctl proxy snapshot [flags] -i, --interactive use interactive mode --kube-context string kube context to use when interacting with kubernetes --kubeconfig string kubeconfig to use, if not standard one - --name string the name of the proxy service/deployment to use (default "gateway-proxy") + --name string the name of the proxy pod/deployment to use -n, --namespace string namespace for reading or writing resources (default "gloo-system") --port string the name of the service port to connect to (default "http") --use-consul use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies) diff --git a/docs/content/reference/cli/glooctl_proxy_stats.md b/docs/content/reference/cli/glooctl_proxy_stats.md index bb9de8ee609..996a92a6b94 100644 --- a/docs/content/reference/cli/glooctl_proxy_stats.md +++ b/docs/content/reference/cli/glooctl_proxy_stats.md @@ -30,7 +30,7 @@ glooctl proxy stats [flags] -i, --interactive use interactive mode --kube-context string kube context to use when interacting with kubernetes --kubeconfig string kubeconfig to use, if not standard one - --name string the name of the proxy service/deployment to use (default "gateway-proxy") + --name string the name of the proxy pod/deployment to use -n, --namespace string namespace for reading or writing resources (default "gloo-system") --port string the name of the service port to connect to (default "http") --use-consul use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies) diff --git a/docs/content/reference/cli/glooctl_proxy_url.md b/docs/content/reference/cli/glooctl_proxy_url.md index b32022e5d5f..4ce4e8611ab 100644 --- a/docs/content/reference/cli/glooctl_proxy_url.md +++ b/docs/content/reference/cli/glooctl_proxy_url.md @@ -36,7 +36,7 @@ glooctl proxy url [flags] -i, --interactive use interactive mode --kube-context string kube context to use when interacting with kubernetes --kubeconfig string kubeconfig to use, if not standard one - --name string the name of the proxy service/deployment to use (default "gateway-proxy") + --name string the name of the proxy pod/deployment to use -n, --namespace string namespace for reading or writing resources (default "gloo-system") --port string the name of the service port to connect to (default "http") --use-consul use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies) diff --git a/pkg/cliutil/input_test.go b/pkg/cliutil/input_test.go index f6ab2f742cb..d75169365d0 100644 --- a/pkg/cliutil/input_test.go +++ b/pkg/cliutil/input_test.go @@ -18,6 +18,6 @@ var _ = Describe("GetBoolInput", func() { err := GetBoolInput("test msg", &val) Expect(err).NotTo(HaveOccurred()) Expect(val).To(BeTrue()) - }) + }, nil) }) }) diff --git a/pkg/cliutil/testutil/testutil.go b/pkg/cliutil/testutil/testutil.go index 80aae70f29c..6939e454f1d 100644 --- a/pkg/cliutil/testutil/testutil.go +++ b/pkg/cliutil/testutil/testutil.go @@ -17,7 +17,8 @@ func Stdio(c *expect.Console) terminal.Stdio { return terminal.Stdio{c.Tty(), c.Tty(), c.Tty()} } -func ExpectInteractive(userInput func(*Console), testCli func()) { +// timeout is the max execution time for testCli() +func ExpectInteractive(userInput func(*Console), testCli func(), timeout *time.Duration) { c, state, err := vt10x.NewVT10XConsole() Expect(err).NotTo(HaveOccurred()) defer c.Close() @@ -44,8 +45,12 @@ func ExpectInteractive(userInput func(*Console), testCli func()) { <-doneC }() + after := 10 * time.Second + if timeout != nil { + after = *timeout + } select { - case <-time.After(10 * time.Second): + case <-time.After(after): c.Tty().Close() Fail("test timed out") case <-doneC: diff --git a/pkg/utils/envoyutils/admincli/client.go b/pkg/utils/envoyutils/admincli/client.go index cd37affcc44..3f65f30577e 100644 --- a/pkg/utils/envoyutils/admincli/client.go +++ b/pkg/utils/envoyutils/admincli/client.go @@ -69,11 +69,11 @@ func NewClient() *Client { // // Designed to be used by tests and CLI from outside of a cluster where `kubectl` is present. // In all other cases, `NewClient` is preferred -func NewPortForwardedClient(ctx context.Context, proxySelector, namespace string) (*Client, func(), error) { +func NewPortForwardedClient(ctx context.Context, kubectlCli *kubectl.Cli, proxySelector, namespace string) (*Client, func(), error) { selector := portforward.WithResourceSelector(proxySelector, namespace) // 1. Open a port-forward to the Kubernetes Deployment, so that we can query the Envoy Admin API directly - portForwarder, err := kubectl.NewCli().StartPortForward(ctx, + portForwarder, err := kubectlCli.StartPortForward(ctx, selector, portforward.WithRemotePort(int(defaults.EnvoyAdminPort))) if err != nil { diff --git a/pkg/utils/kubeutils/kubectl/cli.go b/pkg/utils/kubeutils/kubectl/cli.go index 3332675f080..65b2ef39dd1 100644 --- a/pkg/utils/kubeutils/kubectl/cli.go +++ b/pkg/utils/kubeutils/kubectl/cli.go @@ -285,6 +285,12 @@ func (c *Cli) ExecuteOn(ctx context.Context, kubeContext string, args ...string) return c.Execute(ctx, args...) } +// Get executes a `kubectl get` command and returns the contents of stdout, stderr, and any error that occurred while running the command +func (c *Cli) Get(ctx context.Context, args ...string) (string, string, error) { + args = append([]string{"get"}, args...) + return c.Execute(ctx, args...) +} + func (c *Cli) Execute(ctx context.Context, args ...string) (string, string, error) { if c.kubeContext != "" { if !slices.Contains(args, "--context") { diff --git a/pkg/utils/kubeutils/portforward/cli_portforwarder.go b/pkg/utils/kubeutils/portforward/cli_portforwarder.go index f4f35470f4f..90553b8b7ec 100644 --- a/pkg/utils/kubeutils/portforward/cli_portforwarder.go +++ b/pkg/utils/kubeutils/portforward/cli_portforwarder.go @@ -61,6 +61,8 @@ func (c *cliPortForwarder) startOnce(ctx context.Context) error { cmdCtx, "kubectl", "port-forward", + "--context", + c.properties.kubeContext, "-n", c.properties.resourceNamespace, fmt.Sprintf("%s/%s", c.properties.resourceType, c.properties.resourceName), diff --git a/test/helpers/kube_dump.go b/pkg/utils/statedumputils/state_dump.go similarity index 82% rename from test/helpers/kube_dump.go rename to pkg/utils/statedumputils/state_dump.go index d68d04963e7..19654c1b240 100644 --- a/test/helpers/kube_dump.go +++ b/pkg/utils/statedumputils/state_dump.go @@ -1,4 +1,4 @@ -package helpers +package state_dump_utils import ( "bytes" @@ -12,7 +12,6 @@ import ( "time" "github.com/hashicorp/go-multierror" - "github.com/onsi/ginkgo/v2" "github.com/solo-io/gloo/pkg/cliutil/install" "github.com/solo-io/gloo/pkg/utils/envoyutils/admincli" @@ -24,10 +23,10 @@ import ( "github.com/solo-io/go-utils/threadsafe" ) -// StandardGlooDumpOnFail creates adump of the kubernetes state and certain envoy data from -// the admin interface when a test fails. -// Look at `KubeDumpOnFail` && `EnvoyDumpOnFail` for more details -func StandardGlooDumpOnFail(outLog io.Writer, outDir string, namespaces []string) func() { +// StandardCIDumpOnFail creates a dump of the CI system state, kubernetes state, gloo controller state, +// and certain envoy data from the admin interface when a test fails. +// Look at `CISystemDumpOnFail`, `KubeDumpOnFail`, `ControllerDumpOnFail`, && `EnvoyDumpOnFail` for more details +func StandardCIDumpOnFail(outLog io.Writer, outDir string, namespaces []string) func() { return func() { fmt.Printf("Test failed. Dumping state from %s...\n", strings.Join(namespaces, ", ")) @@ -39,6 +38,7 @@ func StandardGlooDumpOnFail(outLog io.Writer, outDir string, namespaces []string // only wipe at the start of the dump wipeOutDir(outDir) + CISystemDumpOnFail(ctx, kubectlCli, outLog, outDir, namespaces)() KubeDumpOnFail(ctx, kubectlCli, outLog, outDir, namespaces)() ControllerDumpOnFail(ctx, kubectlCli, outLog, outDir, namespaces)() EnvoyDumpOnFail(ctx, kubectlCli, outLog, outDir, namespaces)() @@ -47,26 +47,39 @@ func StandardGlooDumpOnFail(outLog io.Writer, outDir string, namespaces []string } } -// KubeDumpOnFail creates a small dump of the kubernetes state when a test fails. -// This is useful for debugging test failures. +// CISystemDumpOnFail creates a small dump of the local docker and process state. +// This is useful for debugging test failures in CI. // The dump includes: // - docker state // - process state -// - kubernetes state +func CISystemDumpOnFail(_ context.Context, _ *kubectl.Cli, _ io.Writer, outDir string, + _ []string) func() { + return func() { + setupOutDir(outDir) + + recordDockerState(fileAtPath(filepath.Join(outDir, "docker-state.log"))) + recordProcessState(fileAtPath(filepath.Join(outDir, "process-state.log"))) + + fmt.Printf("Finished writing Docker and process state information to the \"%s\" directory.\n", outDir) + } +} + +// KubeDumpOnFail creates a small dump of the kubernetes state. +// This is useful for debugging failures. +// The dump includes: +// - kubernetes cluster state // - logs from all pods in the given namespaces // - yaml representations of all solo.io CRs in the given namespaces -func KubeDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, outLog io.Writer, outDir string, +func KubeDumpOnFail(ctx context.Context, _ *kubectl.Cli, _ io.Writer, outDir string, namespaces []string) func() { return func() { setupOutDir(outDir) - recordDockerState(fileAtPath(filepath.Join(outDir, "docker-state.log"))) - recordProcessState(fileAtPath(filepath.Join(outDir, "process-state.log"))) recordKubeState(fileAtPath(filepath.Join(outDir, "kube-state.log"))) - recordKubeDump(outDir, namespaces...) + recordKubeDump(ctx, outDir, namespaces...) - fmt.Printf("Finished dumping kubernetes state\n") + fmt.Printf("Finished writing Kubernetes state information to the \"%s\" directory.\n", outDir) } } @@ -82,6 +95,7 @@ func recordDockerState(f *os.File) { err := dockerCmd.Run() if err != nil { f.WriteString("*** Unable to get docker state ***. Reason: " + err.Error() + " \n") + f.WriteString(dockerState.String() + "\n") return } f.WriteString("*** Docker state ***\n") @@ -101,6 +115,7 @@ func recordProcessState(f *os.File) { err := psCmd.Run() if err != nil { f.WriteString("unable to get process state. Reason: " + err.Error() + " \n") + f.WriteString(psState.String() + "\n") return } f.WriteString("*** Process state ***\n") @@ -112,11 +127,13 @@ func recordKubeState(f *os.File) { defer f.Close() kubeCli := &install.CmdKubectl{} + f.WriteString("*** Kube state ***\n") + kubeState, err := kubeCli.KubectlOut(nil, "get", "all", "-A", "-o", "wide") if err != nil { f.WriteString("*** Unable to get kube state ***\n") - return } + f.WriteString(string(kubeState) + "\n") resourcesToGet := []string{ // Kubernetes resources @@ -152,8 +169,8 @@ func recordKubeState(f *os.File) { kubeResources, err := kubeCli.KubectlOut(nil, "get", strings.Join(resourcesToGet, ","), "-A", "-owide") if err != nil { f.WriteString("*** Unable to get kube resources ***. Reason: " + err.Error() + " \n") - return } + f.WriteString(string(kubeResources) + "\n") // Describe everything to identify the reason for issues such as Pods, LoadBalancers stuck in pending state // (insufficient resources, unable to acquire an IP), etc. @@ -161,34 +178,28 @@ func recordKubeState(f *os.File) { kubeDescribe, err := kubeCli.KubectlOut(nil, "describe", "all", "-A") if err != nil { f.WriteString("*** Unable to get kube describe ***. Reason: " + err.Error() + " \n") - return } + f.WriteString(string(kubeDescribe) + "\n") kubeEndpointsState, err := kubeCli.KubectlOut(nil, "get", "endpoints", "-A") if err != nil { f.WriteString("*** Unable to get endpoint state ***. Reason: " + err.Error() + " \n") - return } - - f.WriteString("*** Kube state ***\n") - f.WriteString(string(kubeState) + "\n") - f.WriteString(string(kubeResources) + "\n") - f.WriteString(string(kubeDescribe) + "\n") f.WriteString(string(kubeEndpointsState) + "\n") f.WriteString("*** End Kube state ***\n") } -func recordKubeDump(outDir string, namespaces ...string) { +func recordKubeDump(ctx context.Context, outDir string, namespaces ...string) { // for each namespace, create a namespace directory that contains... for _, ns := range namespaces { - // ...a pod logs subdirectoy + // ...a pod logs subdirectory if err := recordPods(filepath.Join(outDir, ns, "_pods"), ns); err != nil { fmt.Printf("error recording pod logs: %f, \n", err) } // ...and a subdirectory for each solo.io CRD with non-zero resources - if err := recordCRs(filepath.Join(outDir, ns), ns); err != nil { + if err := recordCRs(ctx, filepath.Join(outDir, ns), ns); err != nil { fmt.Printf("error recording pod logs: %f, \n", err) } } @@ -234,7 +245,7 @@ func recordPods(podDir, namespace string) error { } // recordCRs records all unique CRs floating about to _output/kube2e-artifacts/$namespace/$crd/$cr.yaml -func recordCRs(namespaceDir string, namespace string) error { +func recordCRs(ctx context.Context, namespaceDir string, namespace string) error { crds, _, err := kubeList(namespace, "crd") if err != nil { return err @@ -262,21 +273,23 @@ func recordCRs(namespaceDir string, namespace string) error { // we record each one in its own .yaml representation for _, cr := range crs { - f := fileAtPath(filepath.Join(crdDir, cr+".yaml")) - errF := fileAtPath(filepath.Join(crdDir, cr+"-error.log")) - - crDetails, errOutput, err := kubeGet(namespace, crd, cr) - - if crDetails != "" { - f.WriteString(crDetails) + args := []string{"-n", namespace, crd, cr, "-oyaml"} + stdout, stderr, err := kubectl.NewCli().Get(ctx, args...) + if stdout != "" { + f := fileAtPath(filepath.Join(crdDir, cr+".yaml")) + f.WriteString(stdout) f.Close() } - if errOutput != "" { - errF.WriteString(errOutput) + if stderr != "" { + errF := fileAtPath(filepath.Join(crdDir, cr+"-error.log")) + errF.WriteString(stderr) errF.Close() } - - return err + // We don't expect an error to occur when executing this request. + // If it does, we intentionally print stdout and stderr to the respective files so that we can more easily debug the error. + if err != nil { + return err + } } } @@ -289,17 +302,9 @@ func kubeLogs(namespace string, pod string) (string, string, error) { return kubeExecute(args) } -// kubeGet runs $(kubectl -n $namespace get $kubeType $name -oyaml) and returns the string result -func kubeGet(namespace string, kubeType string, name string) (string, string, error) { - args := []string{"-n", namespace, "get", kubeType, name, "-oyaml"} - return kubeExecute(args) -} - func kubeExecute(args []string) (string, string, error) { - cli := kubectl.NewCli().WithReceiver(ginkgo.GinkgoWriter) - var outLocation threadsafe.Buffer - runError := cli.Command(context.Background(), args...).WithStdout(&outLocation).Run() + runError := kubectl.NewCli().Command(context.Background(), args...).WithStdout(&outLocation).Run() if runError != nil { return outLocation.String(), runError.OutputString(), runError.Cause() } @@ -327,14 +332,14 @@ func kubeList(namespace string, target string) ([]string, string, error) { return toReturn, "", nil } -// ControllerDumpOnFail creates a small dump of the controller state when a test fails. -// This is useful for debugging test failures. +// ControllerDumpOnFail creates a small dump of the gloo controller state. +// This is useful for debugging failures. // The dump includes: // - controller logs // - controller metrics // - controller xds snapshot // - controller krt snapshot -func ControllerDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, outLog io.Writer, +func ControllerDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, _ io.Writer, outDir string, namespaces []string) func() { return func() { for _, ns := range namespaces { @@ -361,10 +366,10 @@ func ControllerDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, outLog i // Open a port-forward to the controller pod's admin port portForwarder, err := kubectlCli.StartPortForward(ctx, portforward.WithPod(podName, ns), - portforward.WithPorts(int(admin.AdminPort), int(admin.AdminPort)), + portforward.WithPorts(admin.AdminPort, admin.AdminPort), ) if err != nil { - fmt.Printf("error starting port forward to controller admin port: %f\n", err) + fmt.Printf("error starting port forward to controller admin port: %s\n", err.Error()) } defer func() { @@ -376,29 +381,33 @@ func ControllerDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, outLog i WithReceiver(io.Discard). WithCurlOptions( curl.WithRetries(3, 0, 10), - curl.WithPort(int(admin.AdminPort)), + curl.WithPort(admin.AdminPort), ) + stderr := &bytes.Buffer{} krtSnapshotFile := fileAtPath(filepath.Join(namespaceOutDir, fmt.Sprintf("%s.krt_snapshot.log", podName))) - err = adminClient.KrtSnapshotCmd(ctx).WithStdout(krtSnapshotFile).Run().Cause() + err = adminClient.KrtSnapshotCmd(ctx).WithStdout(krtSnapshotFile).WithStderr(stderr).Run().Cause() if err != nil { - fmt.Printf("error running krt snapshot command: %f\n", err) + fmt.Printf("error running krt snapshot command: %s\n", err.Error()) + fmt.Println(stderr) } + stderr = &bytes.Buffer{} xdsSnapshotFile := fileAtPath(filepath.Join(namespaceOutDir, fmt.Sprintf("%s.xds_snapshot.log", podName))) - err = adminClient.XdsSnapshotCmd(ctx).WithStdout(xdsSnapshotFile).Run().Cause() + err = adminClient.XdsSnapshotCmd(ctx).WithStdout(xdsSnapshotFile).WithStderr(stderr).Run().Cause() if err != nil { - fmt.Printf("error running xds snapshot command: %f\n", err) + fmt.Printf("error running xds snapshot command: %s\n", err.Error()) + fmt.Println(stderr) } - fmt.Printf("finished dumping controller state\n") + fmt.Printf("Finished writing Gloo Gateway controller state information to the \"%s\" directory.\n", outDir) } } } } -// EnvoyDumpOnFail creates a small dump of the envoy admin interface when a test fails. -// This is useful for debugging test failures. +// EnvoyDumpOnFail creates a small dump of the envoy admin interface. +// This is useful for debugging failures. // The dump includes: // - config dump // - stats @@ -434,7 +443,7 @@ func EnvoyDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, _ io.Writer, setupOutDir(envoyOutDir) for _, proxy := range proxies { - adminCli, shutdown, err := admincli.NewPortForwardedClient(ctx, + adminCli, shutdown, err := admincli.NewPortForwardedClient(ctx, kubectlCli, fmt.Sprintf("pod/%s", proxy), ns) if err != nil { fmt.Printf("error creating admin cli: %f\n", err) @@ -467,7 +476,7 @@ func EnvoyDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, _ io.Writer, fmt.Printf("error running listeners command: %f\n", err) } - fmt.Printf("finished dumping envoy state\n") + fmt.Printf("Finished writing Envoy state information to the \"%s\" directory.\n", outDir) } } } diff --git a/projects/gloo/cli/pkg/cmd/add/route_interactive_test.go b/projects/gloo/cli/pkg/cmd/add/route_interactive_test.go index f44e3267ee9..deb2a8eff84 100644 --- a/projects/gloo/cli/pkg/cmd/add/route_interactive_test.go +++ b/projects/gloo/cli/pkg/cmd/add/route_interactive_test.go @@ -76,6 +76,6 @@ var _ = Describe("Routes interactive", func() { ug := vs.VirtualHost.Routes[0].GetRouteAction().GetUpstreamGroup() Expect(ug.GetName()).To(Equal("default")) Expect(ug.GetNamespace()).To(Equal("default")) - }) + }, nil) }) }) diff --git a/projects/gloo/cli/pkg/cmd/create/authconfig/authconfig_test.go b/projects/gloo/cli/pkg/cmd/create/authconfig/authconfig_test.go index 3269d702ccd..d6066f78612 100644 --- a/projects/gloo/cli/pkg/cmd/create/authconfig/authconfig_test.go +++ b/projects/gloo/cli/pkg/cmd/create/authconfig/authconfig_test.go @@ -270,7 +270,7 @@ var _ = Describe("AuthConfig", func() { Expect(err).NotTo(HaveOccurred()) _, err = helpers.MustAuthConfigClient(ctx).Read("gloo-system", "default", clients.ReadOpts{}) Expect(err).NotTo(HaveOccurred()) - }) + }, nil) }) It("should create ac with oidc auth", func() { @@ -329,7 +329,7 @@ var _ = Describe("AuthConfig", func() { } Expect(*oidc).To(Equal(expected)) - }) + }, nil) }) It("should create ac with apikey auth", func() { @@ -384,7 +384,7 @@ var _ = Describe("AuthConfig", func() { } Expect(*apiKey).To(Equal(expected)) - }) + }, nil) }) It("should create ac with OPA auth", func() { @@ -426,7 +426,7 @@ var _ = Describe("AuthConfig", func() { } Expect(*opa).To(Equal(expected)) - }) + }, nil) }) }) diff --git a/projects/gloo/cli/pkg/cmd/create/secret/secret_interactive_test.go b/projects/gloo/cli/pkg/cmd/create/secret/secret_interactive_test.go index 35e7c5fa2ba..2cf810cab0c 100644 --- a/projects/gloo/cli/pkg/cmd/create/secret/secret_interactive_test.go +++ b/projects/gloo/cli/pkg/cmd/create/secret/secret_interactive_test.go @@ -66,7 +66,7 @@ var _ = Describe("Secret Interactive Mode", func() { expectMeta(opts.Metadata) Expect(opts.Create.InputSecret.AwsSecret.AccessKey).To(Equal(accessKey)) Expect(opts.Create.InputSecret.AwsSecret.SecretKey).To(Equal(secretKey)) - }) + }, nil) }) }) @@ -111,7 +111,7 @@ var _ = Describe("Secret Interactive Mode", func() { // terminals used in testing it would fail. // In a real terminal, the "doesNotComeThrough": "needsInvestigation" key-value pair would be included. Expect(opts.Create.InputSecret.AzureSecret.ApiKeys.MustMap()).To(BeEquivalentTo(map[string]string{key1: val1, key2: val2})) - }) + }, nil) }) }) @@ -154,7 +154,7 @@ var _ = Describe("Secret Interactive Mode", func() { Expect(opts.Create.InputSecret.TlsSecret.PrivateKeyFilename).To(Equal(privateKey)) Expect(opts.Create.InputSecret.TlsSecret.CertChainFilename).To(Equal(certChainFilename)) Expect(opts.Create.InputSecret.TlsSecret.OCSPStapleFilename).To(Equal(ocspStapleFilename)) - }) + }, nil) }) }) }) diff --git a/projects/gloo/cli/pkg/cmd/create/upstream_interactive_test.go b/projects/gloo/cli/pkg/cmd/create/upstream_interactive_test.go index 881c882bbd2..8cc0edd49a1 100644 --- a/projects/gloo/cli/pkg/cmd/create/upstream_interactive_test.go +++ b/projects/gloo/cli/pkg/cmd/create/upstream_interactive_test.go @@ -47,7 +47,7 @@ var _ = Describe("Upstream Interactive Mode", func() { err := AddUpstreamFlagsInteractive(ctx, &upstream) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("interactive mode not currently available for type kube")) - }) + }, nil) }) It("should not be allowed for Consul", func() { @@ -60,7 +60,7 @@ var _ = Describe("Upstream Interactive Mode", func() { err := AddUpstreamFlagsInteractive(ctx, &upstream) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("interactive mode not currently available for type consul")) - }) + }, nil) }) It("should error out for AWS when there's no secret", func() { @@ -76,7 +76,7 @@ var _ = Describe("Upstream Interactive Mode", func() { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("no AWS secrets found. create an AWS credentials secret using " + "glooctl create secret aws --help")) - }) + }, nil) }) Context("AWS with secret", func() { @@ -129,7 +129,7 @@ var _ = Describe("Upstream Interactive Mode", func() { Expect(err).NotTo(HaveOccurred()) Expect(&upstream.Aws.Secret).To(matchers.MatchProto(localSecretRef)) Expect(upstream.Aws.Region).To(Equal(defaultAwsRegion)) - }) + }, nil) }) It("should work with custom region", func() { @@ -148,7 +148,7 @@ var _ = Describe("Upstream Interactive Mode", func() { Expect(err).NotTo(HaveOccurred()) Expect(&upstream.Aws.Secret).To(matchers.MatchProto(localSecretRef)) Expect(upstream.Aws.Region).To(Equal("custom-region")) - }) + }, nil) }) }) @@ -164,7 +164,7 @@ var _ = Describe("Upstream Interactive Mode", func() { err := AddUpstreamFlagsInteractive(ctx, &upstream) Expect(err).NotTo(HaveOccurred()) Expect(upstream.Static.Hosts).To(BeNil()) - }) + }, nil) }) It("should work for static with hosts", func() { @@ -183,7 +183,7 @@ var _ = Describe("Upstream Interactive Mode", func() { err := AddUpstreamFlagsInteractive(ctx, &upstream) Expect(err).NotTo(HaveOccurred()) Expect(upstream.Static.Hosts).To(BeEquivalentTo([]string{"foo", "bar"})) - }) + }, nil) }) It("should error out for Azure when there's no secret", func() { @@ -199,7 +199,7 @@ var _ = Describe("Upstream Interactive Mode", func() { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("no Azure secrets found. create an Azure credentials secret using " + "glooctl create secret azure --help")) - }) + }, nil) }) Context("Azure with secret", func() { @@ -253,7 +253,7 @@ var _ = Describe("Upstream Interactive Mode", func() { Expect(err).NotTo(HaveOccurred()) Expect(upstream.Azure.Secret).To(Equal(localSecretRef)) Expect(upstream.Azure.FunctionAppName).To(Equal("")) - }) + }, nil) }) It("should work with custom function app name", func() { @@ -273,7 +273,7 @@ var _ = Describe("Upstream Interactive Mode", func() { Expect(err).NotTo(HaveOccurred()) Expect(upstream.Azure.Secret).To(Equal(localSecretRef)) Expect(upstream.Azure.FunctionAppName).To(Equal("custom")) - }) + }, nil) }) }) diff --git a/projects/gloo/cli/pkg/cmd/create/virtualservice_glooe_test.go b/projects/gloo/cli/pkg/cmd/create/virtualservice_glooe_test.go index 450af3fe06f..1cc57adf22a 100644 --- a/projects/gloo/cli/pkg/cmd/create/virtualservice_glooe_test.go +++ b/projects/gloo/cli/pkg/cmd/create/virtualservice_glooe_test.go @@ -49,7 +49,7 @@ var _ = Describe("VirtualService", func() { Expect(err).NotTo(HaveOccurred()) _, err = helpers.MustVirtualServiceClient(ctx).Read("gloo-system", "default", clients.ReadOpts{}) Expect(err).NotTo(HaveOccurred()) - }) + }, nil) }) It("should create vs with auth config", func() { @@ -79,7 +79,7 @@ var _ = Describe("VirtualService", func() { acRef := vs.VirtualHost.Options.Extauth.Spec.(*v1.ExtAuthExtension_ConfigRef).ConfigRef Expect(acRef.Name).To(Equal("ac1")) Expect(acRef.Namespace).To(Equal("ns1")) - }) + }, nil) }) }) diff --git a/projects/gloo/cli/pkg/cmd/debug/root.go b/projects/gloo/cli/pkg/cmd/debug/root.go index 10c453c2c14..a9bee085bd5 100644 --- a/projects/gloo/cli/pkg/cmd/debug/root.go +++ b/projects/gloo/cli/pkg/cmd/debug/root.go @@ -1,12 +1,20 @@ package debug import ( + "context" + "fmt" "os" + "time" + "github.com/solo-io/gloo/pkg/cliutil" + "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" + state_dump_utils "github.com/solo-io/gloo/pkg/utils/statedumputils" "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options" "github.com/solo-io/gloo/projects/gloo/cli/pkg/constants" "github.com/solo-io/gloo/projects/gloo/cli/pkg/flagutils" "github.com/solo-io/go-utils/cliutils" + + "github.com/rotisserie/eris" "github.com/spf13/cobra" ) @@ -14,11 +22,43 @@ func RootCmd(opts *options.Options, optionsFunc ...cliutils.OptionsFunc) *cobra. cmd := &cobra.Command{ Use: constants.DEBUG_COMMAND.Use, Short: constants.DEBUG_COMMAND.Short, + Long: constants.DEBUG_COMMAND.Long, + PreRunE: func(cmd *cobra.Command, args []string) error { + var consent bool + if err := cliutil.GetBoolInput( + fmt.Sprintf("This command will overwrite the \"%s\" directory, if present. Are you sure you want to proceed?", opts.Debug.Directory), + &consent, + ); err != nil { + return err + } + + if !consent { + return eris.New(fmt.Sprintf("Aborting: cannot proceed without overwriting \"%s\" directory.\n"+ + "You may use \"--directory\" to specify a different directory to overwrite.", opts.Debug.Directory)) + } + + if err := os.RemoveAll(opts.Debug.Directory); err != nil { + return eris.Wrap(err, "error wiping out directory") + } + return nil + }, RunE: func(cmd *cobra.Command, args []string) error { - return constants.SubcommandError + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + kubectlCli := kubectl.NewCli().WithKubeContext(opts.Top.KubeContext) + + state_dump_utils.KubeDumpOnFail(ctx, kubectlCli, os.Stdout, opts.Debug.Directory, opts.Debug.Namespaces)() + state_dump_utils.ControllerDumpOnFail(ctx, kubectlCli, os.Stdout, opts.Debug.Directory, opts.Debug.Namespaces)() + state_dump_utils.EnvoyDumpOnFail(ctx, kubectlCli, os.Stdout, opts.Debug.Directory, opts.Debug.Namespaces)() + return nil }, } + pflags := cmd.PersistentFlags() + flagutils.AddNamespacesFlag(pflags, &opts.Debug.Namespaces) + flagutils.AddDirectoryFlag(pflags, &opts.Debug.Directory) + cmd.AddCommand(DebugLogCmd(opts)) cmd.AddCommand(DebugYamlCmd(opts)) cliutils.ApplyOptions(cmd, optionsFunc) @@ -27,9 +67,10 @@ func RootCmd(opts *options.Options, optionsFunc ...cliutils.OptionsFunc) *cobra. func DebugLogCmd(opts *options.Options, optionsFunc ...cliutils.OptionsFunc) *cobra.Command { cmd := &cobra.Command{ - Use: constants.DEBUG_LOG_COMMAND.Use, - Aliases: constants.DEBUG_LOG_COMMAND.Aliases, - Short: constants.DEBUG_LOG_COMMAND.Short, + Use: constants.DEBUG_LOG_COMMAND.Use, + Aliases: constants.DEBUG_LOG_COMMAND.Aliases, + Short: constants.DEBUG_LOG_COMMAND.Short, + Deprecated: constants.DEBUG_LOG_COMMAND.Deprecated, RunE: func(cmd *cobra.Command, args []string) error { return DebugLogs(opts, os.Stdout) }, @@ -47,6 +88,9 @@ func DebugYamlCmd(opts *options.Options, optionsFunc ...cliutils.OptionsFunc) *c cmd := &cobra.Command{ Use: constants.DEBUG_YAML_COMMAND.Use, Short: constants.DEBUG_YAML_COMMAND.Short, + PreRun: func(cmd *cobra.Command, args []string) { + fmt.Println("Top level \"debug\" command is preferred over \"debug yaml\".") + }, RunE: func(cmd *cobra.Command, args []string) error { return DebugYaml(opts, os.Stdout) }, diff --git a/projects/gloo/cli/pkg/cmd/debug/root_test.go b/projects/gloo/cli/pkg/cmd/debug/root_test.go index ff2087f002b..4f2eda562e6 100644 --- a/projects/gloo/cli/pkg/cmd/debug/root_test.go +++ b/projects/gloo/cli/pkg/cmd/debug/root_test.go @@ -3,21 +3,94 @@ package debug_test import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/solo-io/gloo/projects/gloo/cli/pkg/constants" + + "os" + "time" + + "github.com/solo-io/gloo/pkg/cliutil/testutil" "github.com/solo-io/gloo/projects/gloo/cli/pkg/helpers" "github.com/solo-io/gloo/projects/gloo/cli/pkg/testutils" ) +const defaultOutDir = "debug" + +var timeout = 20 * time.Second + +var kubeStateFile = func(outDir string) string { + if outDir == "" { + outDir = defaultOutDir + } + return outDir + "/kube-state.log" +} + var _ = Describe("Debug", func() { BeforeEach(func() { helpers.UseMemoryClients() }) - It("should expect a subcommand after debug", func() { - err := testutils.Glooctl("debug") - Expect(err).To(HaveOccurred()) - Expect(err).To(Equal(constants.SubcommandError)) + AfterEach(func() { + Expect(os.RemoveAll(defaultOutDir)).NotTo(HaveOccurred()) + }) + + It("should support the top level debug command and should populate the kube-state.log file", func() { + testutil.ExpectInteractive(func(c *testutil.Console) { + c.ExpectString("This command will overwrite the \"" + defaultOutDir + "\" directory, if present. Are you sure you want to proceed? [y/N]: ") + c.SendLine("y") + c.ExpectEOF() + }, func() { + err := testutils.Glooctl("debug") + Expect(err).NotTo(HaveOccurred()) + + kubeStateBytes, err := os.ReadFile(kubeStateFile("")) + Expect(err).NotTo(HaveOccurred(), kubeStateFile("")+" file should be present") + Expect(kubeStateBytes).NotTo(BeEmpty()) + }, &timeout) + }) + + When("a directory is specified", func() { + + const customDir = "custom-dir" + + AfterEach(func() { + Expect(os.RemoveAll(customDir)).NotTo(HaveOccurred()) + }) + + It("should populate specified directory instead", func() { + testutil.ExpectInteractive(func(c *testutil.Console) { + c.ExpectString("This command will overwrite the \"" + customDir + "\" directory, if present. Are you sure you want to proceed? [y/N]: ") + c.SendLine("y") + c.ExpectEOF() + }, func() { + err := testutils.Glooctl("debug --directory " + customDir) + Expect(err).NotTo(HaveOccurred()) + + kubeStateBytes, err := os.ReadFile(kubeStateFile(customDir)) + Expect(err).NotTo(HaveOccurred(), kubeStateFile(customDir)+" file should be present") + Expect(kubeStateBytes).NotTo(BeEmpty()) + + // default dir should not exist + _, err = os.ReadDir(defaultOutDir) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(os.ErrNotExist)) + }, &timeout) + }) + }) + + It("should error and abort if the user does not consent", func() { + testutil.ExpectInteractive(func(c *testutil.Console) { + c.ExpectString("This command will overwrite the \"" + defaultOutDir + "\" directory, if present. Are you sure you want to proceed? [y/N]: ") + c.SendLine("N") + c.ExpectEOF() + }, func() { + err := testutils.Glooctl("debug") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Aborting: cannot proceed without overwriting \"" + defaultOutDir + "\" directory")) + + _, err = os.ReadDir(defaultOutDir) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(os.ErrNotExist)) + }, nil) }) }) diff --git a/projects/gloo/cli/pkg/cmd/edit/route/extauth_test.go b/projects/gloo/cli/pkg/cmd/edit/route/extauth_test.go index e274d3d6c0a..2a97276a5a5 100644 --- a/projects/gloo/cli/pkg/cmd/edit/route/extauth_test.go +++ b/projects/gloo/cli/pkg/cmd/edit/route/extauth_test.go @@ -104,7 +104,7 @@ var _ = Describe("Extauth", func() { Expect(extension).To(test_matchers.MatchProto(&extauthpb.ExtAuthExtension{ Spec: &extauthpb.ExtAuthExtension_Disable{Disable: true}, })) - }) + }, nil) }) }) diff --git a/projects/gloo/cli/pkg/cmd/edit/settings/extauth_test.go b/projects/gloo/cli/pkg/cmd/edit/settings/extauth_test.go index 596d85c2297..92251d11e16 100644 --- a/projects/gloo/cli/pkg/cmd/edit/settings/extauth_test.go +++ b/projects/gloo/cli/pkg/cmd/edit/settings/extauth_test.go @@ -123,7 +123,7 @@ var _ = Describe("Extauth", func() { Namespace: "gloo-system", }, })) - }) + }, nil) }) }) diff --git a/projects/gloo/cli/pkg/cmd/edit/settings/ratelimit/ratelimit_test.go b/projects/gloo/cli/pkg/cmd/edit/settings/ratelimit/ratelimit_test.go index 3c5b8f9ec5c..f1aa72cdec2 100644 --- a/projects/gloo/cli/pkg/cmd/edit/settings/ratelimit/ratelimit_test.go +++ b/projects/gloo/cli/pkg/cmd/edit/settings/ratelimit/ratelimit_test.go @@ -156,7 +156,7 @@ var _ = Describe("RateLimit", func() { Expect(rlSettings.DenyOnFail).To(Equal(expectedSettings.DenyOnFail)) Expect(rlSettings.EnableXRatelimitHeaders).To(Equal(expectedSettings.EnableXRatelimitHeaders)) Expect(rlSettings.RateLimitBeforeAuth).To(Equal(expectedSettings.RateLimitBeforeAuth)) - }) + }, nil) }) }) diff --git a/projects/gloo/cli/pkg/cmd/edit/upstream/root_test.go b/projects/gloo/cli/pkg/cmd/edit/upstream/root_test.go index d18411b2427..bc9b90ed998 100644 --- a/projects/gloo/cli/pkg/cmd/edit/upstream/root_test.go +++ b/projects/gloo/cli/pkg/cmd/edit/upstream/root_test.go @@ -165,7 +165,7 @@ var _ = Describe("Root", func() { Expect(ref.Name).To(Equal("sslname")) Expect(ref.Namespace).To(Equal("sslnamespace")) Expect(sslconfig.Sni).To(Equal("somesni")) - }) + }, nil) }) }) }) diff --git a/projects/gloo/cli/pkg/cmd/edit/virtualservice/root_test.go b/projects/gloo/cli/pkg/cmd/edit/virtualservice/root_test.go index 6248e6585e6..57dd5c53b63 100644 --- a/projects/gloo/cli/pkg/cmd/edit/virtualservice/root_test.go +++ b/projects/gloo/cli/pkg/cmd/edit/virtualservice/root_test.go @@ -155,7 +155,7 @@ var _ = Describe("Root", func() { Expect(ref.Name).To(Equal("sslname")) Expect(ref.Namespace).To(Equal("sslnamespace")) Expect(sslconfig.SniDomains).To(Equal([]string{"somesni"})) - }) + }, nil) }) }) }) diff --git a/projects/gloo/cli/pkg/cmd/gateway/dump.go b/projects/gloo/cli/pkg/cmd/gateway/dump.go index 747eae7131c..85837932083 100644 --- a/projects/gloo/cli/pkg/cmd/gateway/dump.go +++ b/projects/gloo/cli/pkg/cmd/gateway/dump.go @@ -4,16 +4,14 @@ import ( "archive/zip" "fmt" "os" - "time" - "strings" - - "github.com/solo-io/go-utils/cliutils" + "time" "github.com/solo-io/gloo/pkg/utils/envoyutils/admincli" - + "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options" - "github.com/solo-io/gloo/projects/gloo/pkg/defaults" + + "github.com/solo-io/go-utils/cliutils" "github.com/spf13/cobra" ) @@ -62,7 +60,7 @@ func writeSnapshotCmd(opts *options.Options, optionsFunc ...cliutils.OptionsFunc } func getEnvoyCfgDump(opts *options.Options) error { - adminCli, shutdownFunc, err := admincli.NewPortForwardedClient(opts.Top.Ctx, opts.Proxy.Name, opts.Metadata.GetNamespace()) + adminCli, shutdownFunc, err := admincli.NewPortForwardedClient(opts.Top.Ctx, kubectl.NewCli().WithKubeContext(opts.Top.KubeContext), opts.Proxy.Name, opts.Metadata.GetNamespace()) if err != nil { return err } @@ -73,7 +71,7 @@ func getEnvoyCfgDump(opts *options.Options) error { } func getEnvoyStatsDump(opts *options.Options) error { - adminCli, shutdownFunc, err := admincli.NewPortForwardedClient(opts.Top.Ctx, opts.Proxy.Name, opts.Metadata.GetNamespace()) + adminCli, shutdownFunc, err := admincli.NewPortForwardedClient(opts.Top.Ctx, kubectl.NewCli().WithKubeContext(opts.Top.KubeContext), opts.Proxy.Name, opts.Metadata.GetNamespace()) if err != nil { return err } @@ -92,12 +90,7 @@ func getEnvoyFullDumpToDisk(opts *options.Options) (string, error) { defer proxyOutArchiveFile.Close() defer proxyOutArchive.Close() - proxyNamespace := opts.Metadata.GetNamespace() - if proxyNamespace == "" { - proxyNamespace = defaults.GlooSystem - } - - adminCli, shutdownFunc, err := admincli.NewPortForwardedClient(opts.Top.Ctx, opts.Proxy.Name, opts.Metadata.GetNamespace()) + adminCli, shutdownFunc, err := admincli.NewPortForwardedClient(opts.Top.Ctx, kubectl.NewCli().WithKubeContext(opts.Top.KubeContext), opts.Proxy.Name, opts.Metadata.GetNamespace()) if err != nil { return proxyOutArchiveFile.Name(), err } diff --git a/projects/gloo/cli/pkg/cmd/gateway/root.go b/projects/gloo/cli/pkg/cmd/gateway/root.go index ef3b80bca31..70781469d86 100644 --- a/projects/gloo/cli/pkg/cmd/gateway/root.go +++ b/projects/gloo/cli/pkg/cmd/gateway/root.go @@ -1,7 +1,6 @@ package gateway import ( - "github.com/solo-io/gloo/projects/gateway/pkg/defaults" "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options" "github.com/solo-io/gloo/projects/gloo/cli/pkg/constants" "github.com/solo-io/gloo/projects/gloo/cli/pkg/flagutils" @@ -16,7 +15,7 @@ func RootCmd(opts *options.Options, optionsFunc ...cliutils.OptionsFunc) *cobra. Short: "interact with proxy instances managed by Gloo", Long: "these commands can be used to interact directly with the Proxies Gloo is managing. They are useful for interacting with and debugging the proxies (Envoy instances) directly.", } - cmd.PersistentFlags().StringVar(&opts.Proxy.Name, "name", defaults.GatewayProxyName, "the name of the proxy service/deployment to use") + cmd.PersistentFlags().StringVar(&opts.Proxy.Name, "name", "", "the name of the proxy pod/deployment to use") cmd.MarkPersistentFlagRequired("name") cmd.PersistentFlags().StringVar(&opts.Proxy.Port, "port", "http", "the name of the service port to connect to") diff --git a/projects/gloo/cli/pkg/cmd/options/options.go b/projects/gloo/cli/pkg/cmd/options/options.go index dc2d12101ea..0718aac6a7c 100644 --- a/projects/gloo/cli/pkg/cmd/options/options.go +++ b/projects/gloo/cli/pkg/cmd/options/options.go @@ -36,6 +36,7 @@ type Options struct { Check Check CheckCRD CheckCRD ValidateLicense ValidateLicense + Debug Debug } type Top struct { contextoptions.ContextAccessible @@ -497,3 +498,9 @@ type CheckCRD struct { type ValidateLicense struct { LicenseKey string } + +// options for the "glooctl debug" command, see projects/gloo/cli/pkg/cmd/debug/root.go +type Debug struct { + Directory string + Namespaces []string +} diff --git a/projects/gloo/cli/pkg/constants/commands.go b/projects/gloo/cli/pkg/constants/commands.go index 88fc612201f..de8388e444f 100644 --- a/projects/gloo/cli/pkg/constants/commands.go +++ b/projects/gloo/cli/pkg/constants/commands.go @@ -80,18 +80,27 @@ var ( DEBUG_COMMAND = cobra.Command{ Use: "debug", - Short: "Debug a Gloo resource (requires Gloo running on Kubernetes)", + Short: "Debug Gloo Gateway (requires Gloo running on Kubernetes)", + Long: "Dumps Kubernetes, Gloo Gateway controller, and Envoy state information to a local directory. " + + "This is useful for debugging failures. " + + "The dump includes:\n" + + "- the Kubernetes cluster state\n" + + "- logs from all pods in the given namespaces\n" + + "- YAML manifests of all solo.io CRs in the given namespaces\n" + + "- the gloo controller logs, metrics, xds snapshot, and krt snapshot\n" + + "- the envoy config dump, stats, clusters, and listeners", } DEBUG_LOG_COMMAND = cobra.Command{ - Use: "logs", - Aliases: []string{"log"}, - Short: "Debug Gloo logs (requires Gloo running on Kubernetes)", + Use: "logs", + Aliases: []string{"log"}, + Short: "Print Gloo logs from a Kubernetes cluster", + Deprecated: "will be removed in a future release. Use top level \"debug\" command instead.", } DEBUG_YAML_COMMAND = cobra.Command{ Use: "yaml", - Short: "Dump YAML representing the current Gloo state (requires Gloo running on Kubernetes)", + Short: "Print YAML representing the current Gloo state of a Kubernetes cluster (top level \"debug\" command is preferred)", } DELETE_COMMAND = cobra.Command{ diff --git a/projects/gloo/cli/pkg/flagutils/debug.go b/projects/gloo/cli/pkg/flagutils/debug.go index e4632d76058..cd7a3a8f7e8 100644 --- a/projects/gloo/cli/pkg/flagutils/debug.go +++ b/projects/gloo/cli/pkg/flagutils/debug.go @@ -9,3 +9,11 @@ func AddDebugFlags(set *pflag.FlagSet, top *options.Top) { set.BoolVar(&top.Zip, "zip", false, "save logs to a tar file (specify location with -f)") set.BoolVar(&top.ErrorsOnly, "errors-only", false, "filter for error logs only") } + +func AddDirectoryFlag(set *pflag.FlagSet, strptr *string) { + set.StringVarP(strptr, "directory", "d", "debug", "directory to write debug info to") +} + +func AddNamespacesFlag(set *pflag.FlagSet, strptr *[]string) { + set.StringArrayVarP(strptr, "namespaces", "N", []string{DefaultNamespace}, "namespaces from which to dump logs and resources (use flag multiple times to specify multiple namespaces, e.g. '-N gloo-system -N default')") +} diff --git a/projects/gloo/cli/pkg/surveyutils/route_test.go b/projects/gloo/cli/pkg/surveyutils/route_test.go index b6febc2f421..e631c4530fd 100644 --- a/projects/gloo/cli/pkg/surveyutils/route_test.go +++ b/projects/gloo/cli/pkg/surveyutils/route_test.go @@ -107,7 +107,7 @@ var _ = Describe("Route", func() { _, idx, err := SelectRouteInteractive(&opts, "vsvc prompt:", "route prompt:") Expect(err).NotTo(HaveOccurred()) Expect(idx).To(Equal(1)) - }) + }, nil) }) It("should populate the correct flags", func() { @@ -140,7 +140,7 @@ var _ = Describe("Route", func() { Expect(opts.Add.Route.Destination.Upstream.Namespace).To(Equal("gloo-system")) Expect(opts.Add.Route.Destination.Upstream.Name).To(Equal("gloo-system.some-ns-test-svc-1234")) - }) + }, nil) }) It("should allow you to choose a function", func() { @@ -176,7 +176,7 @@ var _ = Describe("Route", func() { Expect(opts.Add.Route.Destination.Upstream.Name).To(Equal("gloo-system.some-ns-test-svc-5678")) Expect(opts.Add.Route.Destination.DestinationSpec.Aws.LogicalName).To(Equal("function1")) - }) + }, nil) }) It("should allow you to skip choosing a function", func() { @@ -215,6 +215,6 @@ var _ = Describe("Route", func() { Expect(opts.Add.Route.Destination.DestinationSpec.Aws.LogicalName).To(Equal(NoneOfTheAbove)) Expect(opts.Add.Route.Plugins.PrefixRewrite.Value).To(HaveValue(Equal("/api/pets"))) - }) + }, nil) }) }) diff --git a/projects/gloo/pkg/servers/admin/server.go b/projects/gloo/pkg/servers/admin/server.go index d298d9ebd0d..dde1ce3154c 100644 --- a/projects/gloo/pkg/servers/admin/server.go +++ b/projects/gloo/pkg/servers/admin/server.go @@ -55,6 +55,7 @@ func ServerHandlers(ctx context.Context, history iosnapshot.History, dbg *krt.De }) profiles["/snapshots/xds"] = "XDS Snapshot" + // if kubeGateway.enabled is false, krt snapshot will be null m.HandleFunc("/snapshots/krt", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, dbg, r) }) diff --git a/test/kube2e/gateway/gateway_suite_test.go b/test/kube2e/gateway/gateway_suite_test.go index 6290830237f..e881cc666d3 100644 --- a/test/kube2e/gateway/gateway_suite_test.go +++ b/test/kube2e/gateway/gateway_suite_test.go @@ -7,29 +7,27 @@ import ( "testing" "time" - "github.com/solo-io/gloo/test/kubernetes/testutils/cluster" - "github.com/solo-io/skv2/codegen/util" - - kubetestclients "github.com/solo-io/gloo/test/kubernetes/testutils/clients" - - "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" - - kubeutils2 "github.com/solo-io/gloo/test/testutils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" - gatewaydefaults "github.com/solo-io/gloo/projects/gateway/pkg/defaults" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - gloodefaults "github.com/solo-io/gloo/projects/gloo/pkg/defaults" + "github.com/solo-io/skv2/codegen/util" + skhelpers "github.com/solo-io/solo-kit/test/helpers" "github.com/solo-io/gloo/test/helpers" "github.com/solo-io/gloo/test/kube2e" "github.com/solo-io/gloo/test/kube2e/helper" + kubetestclients "github.com/solo-io/gloo/test/kubernetes/testutils/clients" + "github.com/solo-io/gloo/test/kubernetes/testutils/cluster" testruntime "github.com/solo-io/gloo/test/kubernetes/testutils/runtime" - skhelpers "github.com/solo-io/solo-kit/test/helpers" + kubeutils2 "github.com/solo-io/gloo/test/testutils" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" + state_dump_utils "github.com/solo-io/gloo/pkg/utils/statedumputils" + gatewaydefaults "github.com/solo-io/gloo/projects/gateway/pkg/defaults" + gloodefaults "github.com/solo-io/gloo/projects/gloo/pkg/defaults" ) func TestGateway(t *testing.T) { @@ -68,7 +66,7 @@ func StartTestHelper() { outDir := filepath.Join(util.GetModuleRoot(), "_output", "kube2e-artifacts") namespaces := []string{testHelper.InstallNamespace} - skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, outDir, namespaces)) + skhelpers.RegisterPreFailHandler(state_dump_utils.StandardCIDumpOnFail(GinkgoWriter, outDir, namespaces)) kubeCli = kubectl.NewCli().WithReceiver(GinkgoWriter) diff --git a/test/kube2e/gloo/gloo_suite_test.go b/test/kube2e/gloo/gloo_suite_test.go index 91941a5ea6a..063a24290ec 100644 --- a/test/kube2e/gloo/gloo_suite_test.go +++ b/test/kube2e/gloo/gloo_suite_test.go @@ -11,6 +11,7 @@ import ( "github.com/solo-io/skv2/codegen/util" "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" + state_dump_utils "github.com/solo-io/gloo/pkg/utils/statedumputils" "github.com/avast/retry-go" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -69,7 +70,7 @@ var _ = BeforeSuite(func() { outDir := filepath.Join(util.GetModuleRoot(), "_output", "kube2e-artifacts") namespaces := []string{testHelper.InstallNamespace} - skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, outDir, namespaces)) + skhelpers.RegisterPreFailHandler(state_dump_utils.StandardCIDumpOnFail(GinkgoWriter, outDir, namespaces)) // Allow skipping of install step for running multiple times if !glootestutils.ShouldSkipInstall() { diff --git a/test/kube2e/upgrade/upgrade_suite_test.go b/test/kube2e/upgrade/upgrade_suite_test.go index 5ea1cd2e84f..df9acd0a91f 100644 --- a/test/kube2e/upgrade/upgrade_suite_test.go +++ b/test/kube2e/upgrade/upgrade_suite_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/solo-io/gloo/pkg/utils/helmutils" + state_dump_utils "github.com/solo-io/gloo/pkg/utils/statedumputils" . "github.com/onsi/gomega" "github.com/solo-io/gloo/test/kube2e" @@ -53,7 +54,7 @@ var _ = BeforeSuite(func() { outDir := filepath.Join(util.GetModuleRoot(), "_output", "kube2e-artifacts") namespaces := []string{"upgrade", testHelper.InstallNamespace, "other-ns"} - skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, outDir, namespaces)) + skhelpers.RegisterPreFailHandler(state_dump_utils.StandardCIDumpOnFail(GinkgoWriter, outDir, namespaces)) crdDir = filepath.Join(util.GetModuleRoot(), "install", "helm", "gloo", "crds") targetReleasedVersion = kube2e.GetTestReleasedVersion(suiteCtx, "gloo") diff --git a/test/kubernetes/e2e/features/helm/suite.go b/test/kubernetes/e2e/features/helm/suite.go index 192c1b43a1e..4eabe7c0b5c 100644 --- a/test/kubernetes/e2e/features/helm/suite.go +++ b/test/kubernetes/e2e/features/helm/suite.go @@ -15,9 +15,11 @@ import ( v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "github.com/solo-io/gloo/pkg/utils/envoyutils/admincli" + "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" "github.com/solo-io/gloo/test/kubernetes/e2e" "github.com/solo-io/gloo/test/kubernetes/e2e/tests/base" "github.com/solo-io/gloo/test/kubernetes/testutils/helper" + "github.com/solo-io/skv2/codegen/util" "github.com/solo-io/solo-kit/pkg/code-generator/schemagen" ) @@ -44,7 +46,7 @@ func (s *testingSuite) TestProductionRecommendations() { func (s *testingSuite) TestChangedConfigMapTriggersRollout() { expectConfigDumpToContain := func(str string) { - adminCli, shutdown, err := admincli.NewPortForwardedClient(s.Ctx, "deployment/gateway-proxy", s.TestHelper.InstallNamespace) + adminCli, shutdown, err := admincli.NewPortForwardedClient(s.Ctx, kubectl.NewCli(), "deployment/gateway-proxy", s.TestHelper.InstallNamespace) s.NoError(err) defer shutdown() diff --git a/test/kubernetes/e2e/test.go b/test/kubernetes/e2e/test.go index efb71e60652..083a3839068 100644 --- a/test/kubernetes/e2e/test.go +++ b/test/kubernetes/e2e/test.go @@ -11,7 +11,6 @@ import ( "testing" "time" - "github.com/solo-io/gloo/test/helpers" "github.com/solo-io/gloo/test/kubernetes/testutils/actions" "github.com/solo-io/gloo/test/kubernetes/testutils/assertions" "github.com/solo-io/gloo/test/kubernetes/testutils/cluster" @@ -19,6 +18,8 @@ import ( "github.com/solo-io/gloo/test/kubernetes/testutils/helper" testruntime "github.com/solo-io/gloo/test/kubernetes/testutils/runtime" "github.com/solo-io/gloo/test/testutils" + + state_dump_utils "github.com/solo-io/gloo/pkg/utils/statedumputils" ) // MustTestHelper returns the SoloTestHelper used for e2e tests @@ -244,7 +245,7 @@ func (i *TestInstallation) PreFailHandler(ctx context.Context) { i.Assertions.Require.NoError(err) // Dump the logs and state of the cluster - helpers.StandardGlooDumpOnFail(os.Stdout, failureDir, namespaces)() + state_dump_utils.StandardCIDumpOnFail(os.Stdout, failureDir, namespaces)() } // GeneratedFiles is a collection of files that are generated during the execution of a set of tests