Additional CLI debug commands (#10450)

Co-authored-by: changelog-bot <changelog-bot>

.github/ISSUE_TEMPLATE/BUG-ISSUE.yml

@@ -88,3 +88,6 @@ body:
         line 13
         ...
         ```
+
+        The `glooctl debug` command can be used to collect state information about your system.
+        Consider uploading logs, YAML manfests, Envoy listeners, etc. that may be relevant.

changelog/v1.19.0-beta2/CLI-debug-additions.yaml

@@ -0,0 +1,5 @@
+changelog:
+- type: NEW_FEATURE
+  description: Adds a new top level `glooctl debug` CLI command as well as subcommands which collect kubernetes, gloo controller, and envoy state information to aid in debugging failures.
+  issueLink: https://github.com/solo-io/solo-projects/issues/7298
+  resolvesIssue: false

devel/debugging/glooctl-debug.md

@@ -0,0 +1,5 @@
+# Glooctl Debug Commands
+
+When debugging any sort of error or unexpected behavior, it may be useful to collect information about the state of your system.
+The `glooctl debug` command and its subcommands can help with this.
+Run `glooctl debug --help` from the command line for details.

docs/content/reference/cli/glooctl.md

@@ -38,7 +38,7 @@ glooctl is the unified CLI for Gloo.
 * [glooctl completion](../glooctl_completion)	 - generate auto completion for your shell
 * [glooctl create](../glooctl_create)	 - Create a Gloo resource
 * [glooctl dashboard](../glooctl_dashboard)	 - Open Gloo dashboard
-* [glooctl debug](../glooctl_debug)	 - Debug a Gloo resource (requires Gloo running on Kubernetes)
+* [glooctl debug](../glooctl_debug)	 - Debug Gloo Gateway (requires Gloo running on Kubernetes)
 * [glooctl delete](../glooctl_delete)	 - Delete a Gloo resource
 * [glooctl demo](../glooctl_demo)	 - Demos (requires 4 tools to be installed and accessible via the PATH: glooctl, kubectl, docker, and kind.)
 * [glooctl edit](../glooctl_edit)	 - Edit a Gloo resource

docs/content/reference/cli/glooctl_debug.md

@@ -5,7 +5,16 @@ weight: 5
 ---
 ## glooctl debug
 
-Debug a Gloo resource (requires Gloo running on Kubernetes)
+Debug Gloo Gateway (requires Gloo running on Kubernetes)
+
+### Synopsis
+
+Dumps Kubernetes, Gloo Gateway controller, and Envoy state information to a local directory. This is useful for debugging failures. The dump includes:
+- the Kubernetes cluster state
+- logs from all pods in the given namespaces
+- YAML manifests of all solo.io CRs in the given namespaces
+- the gloo controller logs, metrics, xds snapshot, and krt snapshot
+- the envoy config dump, stats, clusters, and listeners
 
 ```
 glooctl debug [flags]
@@ -14,7 +23,9 @@ glooctl debug [flags]
 ### Options
 
 ```
-  -h, --help   help for debug
+  -d, --directory string         directory to write debug info to (default "debug")
+  -h, --help                     help for debug
+  -N, --namespaces stringArray   namespaces from which to dump logs and resources (use flag multiple times to specify multiple namespaces, e.g. '-N gloo-system -N default') (default [gloo-system])
 ```
 
 ### Options inherited from parent commands
@@ -36,6 +47,5 @@ glooctl debug [flags]
 ### SEE ALSO
 
 * [glooctl](../glooctl)	 - CLI for Gloo
-* [glooctl debug logs](../glooctl_debug_logs)	 - Debug Gloo logs (requires Gloo running on Kubernetes)
-* [glooctl debug yaml](../glooctl_debug_yaml)	 - Dump YAML representing the current Gloo state (requires Gloo running on Kubernetes)
+* [glooctl debug yaml](../glooctl_debug_yaml)	 - Print YAML representing the current Gloo state of a Kubernetes cluster (top level "debug" command is preferred)
 

docs/content/reference/cli/glooctl_debug_yaml.md

@@ -5,7 +5,7 @@ weight: 5
 ---
 ## glooctl debug yaml
 
-Dump YAML representing the current Gloo state (requires Gloo running on Kubernetes)
+Print YAML representing the current Gloo state of a Kubernetes cluster (top level "debug" command is preferred)
 
 ```
 glooctl debug yaml [flags]
@@ -29,13 +29,15 @@ glooctl debug yaml [flags]
       --consul-root-key string     key prefix for the Consul key-value storage. (default "gloo")
       --consul-scheme string       URI scheme for the Consul server. Use with --use-consul (default "http")
       --consul-token string        Token is used to provide a per-request ACL token which overrides the agent's default token. Use with --use-consul
+  -d, --directory string           directory to write debug info to (default "debug")
   -i, --interactive                use interactive mode
       --kube-context string        kube context to use when interacting with kubernetes
       --kubeconfig string          kubeconfig to use, if not standard one
+  -N, --namespaces stringArray     namespaces from which to dump logs and resources (use flag multiple times to specify multiple namespaces, e.g. '-N gloo-system -N default') (default [gloo-system])
       --use-consul                 use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies)
 ```
 
 ### SEE ALSO
 
-* [glooctl debug](../glooctl_debug)	 - Debug a Gloo resource (requires Gloo running on Kubernetes)
+* [glooctl debug](../glooctl_debug)	 - Debug Gloo Gateway (requires Gloo running on Kubernetes)
 

docs/content/reference/cli/glooctl_proxy.md

@@ -15,7 +15,7 @@ these commands can be used to interact directly with the Proxies Gloo is managin
 
 ```
   -h, --help               help for proxy
-      --name string        the name of the proxy service/deployment to use (default "gateway-proxy")
+      --name string        the name of the proxy pod/deployment to use
   -n, --namespace string   namespace for reading or writing resources (default "gloo-system")
       --port string        the name of the service port to connect to (default "http")
 ```

docs/content/reference/cli/glooctl_proxy_address.md

@@ -36,7 +36,7 @@ glooctl proxy address [flags]
   -i, --interactive                use interactive mode
       --kube-context string        kube context to use when interacting with kubernetes
       --kubeconfig string          kubeconfig to use, if not standard one
-      --name string                the name of the proxy service/deployment to use (default "gateway-proxy")
+      --name string                the name of the proxy pod/deployment to use
   -n, --namespace string           namespace for reading or writing resources (default "gloo-system")
       --port string                the name of the service port to connect to (default "http")
       --use-consul                 use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies)

docs/content/reference/cli/glooctl_proxy_dump.md

@@ -30,7 +30,7 @@ glooctl proxy dump [flags]
   -i, --interactive                use interactive mode
       --kube-context string        kube context to use when interacting with kubernetes
       --kubeconfig string          kubeconfig to use, if not standard one
-      --name string                the name of the proxy service/deployment to use (default "gateway-proxy")
+      --name string                the name of the proxy pod/deployment to use
   -n, --namespace string           namespace for reading or writing resources (default "gloo-system")
       --port string                the name of the service port to connect to (default "http")
       --use-consul                 use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies)

docs/content/reference/cli/glooctl_proxy_logs.md

@@ -32,7 +32,7 @@ glooctl proxy logs [flags]
   -i, --interactive                use interactive mode
       --kube-context string        kube context to use when interacting with kubernetes
       --kubeconfig string          kubeconfig to use, if not standard one
-      --name string                the name of the proxy service/deployment to use (default "gateway-proxy")
+      --name string                the name of the proxy pod/deployment to use
   -n, --namespace string           namespace for reading or writing resources (default "gloo-system")
       --port string                the name of the service port to connect to (default "http")
       --use-consul                 use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies)

docs/content/reference/cli/glooctl_proxy_served-config.md

@@ -30,7 +30,7 @@ glooctl proxy served-config [flags]
   -i, --interactive                use interactive mode
       --kube-context string        kube context to use when interacting with kubernetes
       --kubeconfig string          kubeconfig to use, if not standard one
-      --name string                the name of the proxy service/deployment to use (default "gateway-proxy")
+      --name string                the name of the proxy pod/deployment to use
   -n, --namespace string           namespace for reading or writing resources (default "gloo-system")
       --port string                the name of the service port to connect to (default "http")
       --use-consul                 use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies)

docs/content/reference/cli/glooctl_proxy_snapshot.md

@@ -31,7 +31,7 @@ glooctl proxy snapshot [flags]
   -i, --interactive                use interactive mode
       --kube-context string        kube context to use when interacting with kubernetes
       --kubeconfig string          kubeconfig to use, if not standard one
-      --name string                the name of the proxy service/deployment to use (default "gateway-proxy")
+      --name string                the name of the proxy pod/deployment to use
   -n, --namespace string           namespace for reading or writing resources (default "gloo-system")
       --port string                the name of the service port to connect to (default "http")
       --use-consul                 use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies)

docs/content/reference/cli/glooctl_proxy_stats.md

@@ -30,7 +30,7 @@ glooctl proxy stats [flags]
   -i, --interactive                use interactive mode
       --kube-context string        kube context to use when interacting with kubernetes
       --kubeconfig string          kubeconfig to use, if not standard one
-      --name string                the name of the proxy service/deployment to use (default "gateway-proxy")
+      --name string                the name of the proxy pod/deployment to use
   -n, --namespace string           namespace for reading or writing resources (default "gloo-system")
       --port string                the name of the service port to connect to (default "http")
       --use-consul                 use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies)

docs/content/reference/cli/glooctl_proxy_url.md

@@ -36,7 +36,7 @@ glooctl proxy url [flags]
   -i, --interactive                use interactive mode
       --kube-context string        kube context to use when interacting with kubernetes
       --kubeconfig string          kubeconfig to use, if not standard one
-      --name string                the name of the proxy service/deployment to use (default "gateway-proxy")
+      --name string                the name of the proxy pod/deployment to use
   -n, --namespace string           namespace for reading or writing resources (default "gloo-system")
       --port string                the name of the service port to connect to (default "http")
       --use-consul                 use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies)

pkg/cliutil/input_test.go

@@ -18,6 +18,6 @@ var _ = Describe("GetBoolInput", func() {
 			err := GetBoolInput("test msg", &val)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(val).To(BeTrue())
-		})
+		}, nil)
 	})
 })

pkg/cliutil/testutil/testutil.go

@@ -17,7 +17,8 @@ func Stdio(c *expect.Console) terminal.Stdio {
 	return terminal.Stdio{c.Tty(), c.Tty(), c.Tty()}
 }
 
-func ExpectInteractive(userInput func(*Console), testCli func()) {
+// timeout is the max execution time for testCli()
+func ExpectInteractive(userInput func(*Console), testCli func(), timeout *time.Duration) {
 	c, state, err := vt10x.NewVT10XConsole()
 	Expect(err).NotTo(HaveOccurred())
 	defer c.Close()
@@ -44,8 +45,12 @@ func ExpectInteractive(userInput func(*Console), testCli func()) {
 		<-doneC
 	}()
 
+	after := 10 * time.Second
+	if timeout != nil {
+		after = *timeout
+	}
 	select {
-	case <-time.After(10 * time.Second):
+	case <-time.After(after):
 		c.Tty().Close()
 		Fail("test timed out")
 	case <-doneC:

pkg/utils/envoyutils/admincli/client.go

@@ -69,11 +69,11 @@ func NewClient() *Client {
 //
 // Designed to be used by tests and CLI from outside of a cluster where `kubectl` is present.
 // In all other cases, `NewClient` is preferred
-func NewPortForwardedClient(ctx context.Context, proxySelector, namespace string) (*Client, func(), error) {
+func NewPortForwardedClient(ctx context.Context, kubectlCli *kubectl.Cli, proxySelector, namespace string) (*Client, func(), error) {
 	selector := portforward.WithResourceSelector(proxySelector, namespace)
 
 	// 1. Open a port-forward to the Kubernetes Deployment, so that we can query the Envoy Admin API directly
-	portForwarder, err := kubectl.NewCli().StartPortForward(ctx,
+	portForwarder, err := kubectlCli.StartPortForward(ctx,
 		selector,
 		portforward.WithRemotePort(int(defaults.EnvoyAdminPort)))
 	if err != nil {

pkg/utils/kubeutils/kubectl/cli.go

@@ -285,6 +285,12 @@ func (c *Cli) ExecuteOn(ctx context.Context, kubeContext string, args ...string)
 	return c.Execute(ctx, args...)
 }
 
+// Get executes a `kubectl get` command and returns the contents of stdout, stderr, and any error that occurred while running the command
+func (c *Cli) Get(ctx context.Context, args ...string) (string, string, error) {
+	args = append([]string{"get"}, args...)
+	return c.Execute(ctx, args...)
+}
+
 func (c *Cli) Execute(ctx context.Context, args ...string) (string, string, error) {
 	if c.kubeContext != "" {
 		if !slices.Contains(args, "--context") {

pkg/utils/kubeutils/portforward/cli_portforwarder.go

@@ -61,6 +61,8 @@ func (c *cliPortForwarder) startOnce(ctx context.Context) error {
 		cmdCtx,
 		"kubectl",
 		"port-forward",
+		"--context",
+		c.properties.kubeContext,
 		"-n",
 		c.properties.resourceNamespace,
 		fmt.Sprintf("%s/%s", c.properties.resourceType, c.properties.resourceName),