Maint: handle degraded api when creating k8sclient (#426)

* Maint: handle degraded api when creating k8sclient

* Review fixes

* Use backplane general error string

* Fix linter errors

Author: typeid

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/prometheus/client_golang v1.22.0
 	github.com/prometheus/common v0.63.0
 	github.com/spf13/cobra v1.8.1
+	github.com/stretchr/testify v1.10.0
 	go.uber.org/mock v0.4.0
 	go.uber.org/zap v1.27.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -122,6 +123,7 @@ require (
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/sagikazarmark/locafero v0.7.0 // indirect

pkg/investigations/clustermonitoringerrorbudgetburn/clustermonitoringerrorbudgetburn.go

@@ -36,6 +36,10 @@ func (c *Investigation) Run(r *investigation.Resources) (result investigation.In
 	// patching the existing RBAC etc...
 	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, r.Name)
 	if err != nil {
+		if errors.Is(err, k8sclient.ErrAPIServerUnavailable) {
+			return result, r.PdClient.EscalateIncidentWithNote("CAD was unable to access cluster's kube-api. Please investigate manually.")
+		}
+
 		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
 	}
 	defer func() {

pkg/investigations/insightsoperatordown/insightsoperatordown.go

@@ -38,6 +38,10 @@ func (c *Investigation) Run(r *investigation.Resources) (investigation.Investiga
 	// We continue with the next step OCPBUG22226 even if the user is banned.
 	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, r.Name)
 	if err != nil {
+		if errors.Is(err, k8sclient.ErrAPIServerUnavailable) {
+			return result, r.PdClient.EscalateIncidentWithNote("CAD was unable to access cluster's kube-api. Please investigate manually.")
+		}
+
 		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
 	}
 	defer func() {

pkg/k8s/client.go

@@ -4,23 +4,29 @@ import (
 	"fmt"
 	"os"
 
-	configv1 "github.com/openshift/api/config/v1"
 	"github.com/openshift/backplane-cli/pkg/cli/config"
 	bpremediation "github.com/openshift/backplane-cli/pkg/remediation"
 	"github.com/openshift/configuration-anomaly-detection/pkg/ocm"
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+// New returns a Kubernetes client for the given cluster scoped to a given remediation's permissions.
 func New(clusterID string, ocmClient ocm.Client, remediation string) (client.Client, error) {
 	backplaneURL := os.Getenv("BACKPLANE_URL")
 	if backplaneURL == "" {
 		return nil, fmt.Errorf("could not create new k8sclient: missing environment variable BACKPLANE_URL")
 	}
 
-	cfg, err := bpremediation.CreateRemediationWithConn(config.BackplaneConfiguration{URL: backplaneURL}, ocmClient.GetConnection(), clusterID, remediation)
+	cfg, err := bpremediation.CreateRemediationWithConn(
+		config.BackplaneConfiguration{URL: backplaneURL},
+		ocmClient.GetConnection(),
+		clusterID,
+		remediation,
+	)
 	if err != nil {
+		if isAPIServerUnavailable(err) {
+			return nil, fmt.Errorf("%w: %w", ErrAPIServerUnavailable, err)
+		}
 		return nil, err
 	}
 
@@ -32,26 +38,17 @@ func New(clusterID string, ocmClient ocm.Client, remediation string) (client.Cli
 	return client.New(cfg, client.Options{Scheme: scheme})
 }
 
+// Cleanup removes the remediation created for the cluster.
 func Cleanup(clusterID string, ocmClient ocm.Client, remediation string) error {
 	backplaneURL := os.Getenv("BACKPLANE_URL")
 	if backplaneURL == "" {
-		return fmt.Errorf("could not create new k8sclient: missing environment variable BACKPLANE_URL")
+		return fmt.Errorf("could not clean up k8sclient: missing environment variable BACKPLANE_URL")
 	}
-	return bpremediation.DeleteRemediationWithConn(config.BackplaneConfiguration{URL: backplaneURL}, ocmClient.GetConnection(), clusterID, remediation)
-}
-
-// Initialize all apis we need in CAD
-func initScheme() (*runtime.Scheme, error) {
-	scheme := runtime.NewScheme()
 
-	// Add corev1 to scheme for core k8s
-	if err := corev1.AddToScheme(scheme); err != nil {
-		return nil, fmt.Errorf("unable to add corev1 scheme: %w", err)
-	}
-
-	// Add config.openshift.io/v1 to scheme for clusteroperator
-	if err := configv1.Install(scheme); err != nil {
-		return nil, fmt.Errorf("unable to add openshift/api/config scheme: %w", err)
-	}
-	return scheme, nil
+	return bpremediation.DeleteRemediationWithConn(
+		config.BackplaneConfiguration{URL: backplaneURL},
+		ocmClient.GetConnection(),
+		clusterID,
+		remediation,
+	)
 }

pkg/k8s/errors.go

@@ -0,0 +1,14 @@
+package k8sclient
+
+import (
+	"errors"
+	"strings"
+)
+
+var ErrAPIServerUnavailable = errors.New("kubernetes API server unavailable")
+
+// isAPIServerUnavailable detects common symptoms of an unreachable API server.
+func isAPIServerUnavailable(err error) bool {
+	errStr := err.Error()
+	return strings.Contains(errStr, "The cluster could be down or under heavy load")
+}

pkg/k8s/errors_test.go

@@ -0,0 +1,37 @@
+package k8sclient
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestIsAPIServerUnavailable(t *testing.T) {
+	tests := []struct {
+		name     string
+		err      error
+		expected bool
+	}{
+		{
+			name: "Cluster down message present",
+			err: errors.New(`Error: Internal error occurred: failed calling webhook "namespace.operator.tekton.dev": failed to call webhook: Post "https://tekton-operator-proxy-webhook.openshift-pipelines.svc:443/namespace-validation?timeout=10s": context deadline exceeded
+	The cluster could be down or under heavy load
+	`),
+			expected: true,
+		},
+		{
+			name:     "Unrelated error message",
+			err:      errors.New("some other error occurred"),
+			expected: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.err == nil && isAPIServerUnavailable(tt.err) {
+				t.Errorf("Expected false for nil error, but got true")
+			} else if tt.err != nil && isAPIServerUnavailable(tt.err) != tt.expected {
+				t.Errorf("For test '%s', expected %v, got %v", tt.name, tt.expected, !tt.expected)
+			}
+		})
+	}
+}

pkg/k8s/scheme.go

@@ -0,0 +1,24 @@
+package k8sclient
+
+import (
+	"fmt"
+
+	configv1 "github.com/openshift/api/config/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+// initScheme initializes the runtime scheme with required APIs.
+func initScheme() (*runtime.Scheme, error) {
+	scheme := runtime.NewScheme()
+
+	if err := corev1.AddToScheme(scheme); err != nil {
+		return nil, fmt.Errorf("unable to add corev1 scheme: %w", err)
+	}
+
+	if err := configv1.Install(scheme); err != nil {
+		return nil, fmt.Errorf("unable to add config.openshift.io/v1 scheme: %w", err)
+	}
+
+	return scheme, nil
+}