Merge pull request #380 from RaphaelBut/add-insights-inv

Add insights operator down investigation

Author: openshift-merge-bot[bot]

pkg/investigations/insightsoperatordown/insightsoperatordown.go

@@ -0,0 +1,133 @@
+package insightsoperatordown
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	investigation "github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"github.com/openshift/configuration-anomaly-detection/pkg/ocm"
+	"k8s.io/apimachinery/pkg/fields"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type Investigation struct{}
+
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New(r.Name, logging.RawLogger)
+
+	user, err := ocm.GetCreatorFromCluster(r.OcmClient.GetConnection(), r.Cluster)
+	if err != nil {
+		notes.AppendWarning("encountered an issue when checking if the cluster owner is banned: %s", err)
+		return result, r.PdClient.EscalateIncidentWithNote(notes.String())
+	}
+
+	if user.Banned() {
+		notes.AppendWarning("User is banned: %s\nBan description: %s\nPlease open a proactive case, so that MCS can resolve the ban or organize a ownership transfer.", user.BanCode(), user.BanDescription())
+	} else {
+		notes.AppendSuccess("User is not banned.")
+	}
+
+	// We continue with the next step OCPBUG22226 even if the user is banned.
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, r.Name)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		deferErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, r.Name)
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		}
+	}()
+
+	coList := &configv1.ClusterOperatorList{}
+	listOptions := &client.ListOptions{FieldSelector: fields.SelectorFromSet(fields.Set{"metadata.name": "insights"})}
+	err = k8scli.List(context.TODO(), coList, listOptions)
+	if err != nil {
+		return result, fmt.Errorf("unable to list insights clusteroperator: %w", err)
+	}
+
+	if len(coList.Items) != 1 {
+		return result, fmt.Errorf("found %d clusteroperators, expected 1", len(coList.Items))
+	}
+	co := coList.Items[0]
+
+	if isOCPBUG22226(&co) {
+		notes.AppendWarning("Found symptom of OCPBUGS-22226. Try deleting the insights operator pod to remediate.\n$ oc -n openshift-insights delete pods -l app=insights-operator --wait=false")
+		return result, r.PdClient.EscalateIncidentWithNote(notes.String())
+	} else {
+		notes.AppendSuccess("Ruled out OCPBUGS-22226")
+	}
+
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		logging.Error("Network verifier ran into an error: %s", err.Error())
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+
+		err = r.PdClient.AddNote(notes.String())
+		if err != nil {
+			// We do not return as we want the alert to be escalated either no matter what.
+			logging.Error("could not add failure reason incident notes")
+		}
+	}
+
+	switch verifierResult {
+	case networkverifier.Failure:
+		logging.Infof("Network verifier reported failure: %s", failureReason)
+		// XXX: metrics.Inc(metrics.ServicelogPrepared, investigationName)
+		result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+		notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s", r.Cluster.ID(), failureReason)
+
+		// In the future, we want to send a service log in this case
+		err = r.PdClient.AddNote(notes.String())
+		if err != nil {
+			logging.Error("could not add issues to incident notes")
+		}
+	case networkverifier.Success:
+		notes.AppendSuccess("Network verifier passed")
+		err = r.PdClient.AddNote(notes.String())
+		if err != nil {
+			logging.Error("could not add passed message to incident notes")
+		}
+	}
+	return result, r.PdClient.EscalateIncidentWithNote(notes.String())
+}
+
+func isOCPBUG22226(co *configv1.ClusterOperator) bool {
+	symptomStatusString := "Failed to pull SCA certs"
+
+	for _, condition := range co.Status.Conditions {
+		if condition.Type == "SCAAvailable" && strings.Contains(condition.Message, symptomStatusString) {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *Investigation) Name() string {
+	return "insightsoperatordown"
+}
+
+func (c *Investigation) Description() string {
+	return "Investigate insights operator down alert"
+}
+
+func (c *Investigation) ShouldInvestigateAlert(alert string) bool {
+	return strings.Contains(alert, "InsightsOperatorDown")
+}
+
+func (c *Investigation) IsExperimental() bool {
+	return false
+}
+
+func (c *Investigation) RequiresAwsClient() bool {
+	return false
+}

pkg/investigations/insightsoperatordown/insightsoperatordown_test.go

@@ -0,0 +1,49 @@
+package insightsoperatordown
+
+import (
+	"testing"
+
+	configv1 "github.com/openshift/api/config/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestIsOCPBUG22226(t *testing.T) {
+	tests := []struct {
+		name     string
+		co       configv1.ClusterOperator
+		expected bool
+	}{
+		{
+			name: "SCA certs pull failure detected",
+			co: configv1.ClusterOperator{
+				ObjectMeta: v1.ObjectMeta{Name: "insights"},
+				Status: configv1.ClusterOperatorStatus{
+					Conditions: []configv1.ClusterOperatorStatusCondition{
+						{Type: "SCAAvailable", Message: "Failed to pull SCA certs"},
+					},
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "No SCA certs pull failure",
+			co: configv1.ClusterOperator{
+				ObjectMeta: v1.ObjectMeta{Name: "insights"},
+				Status: configv1.ClusterOperatorStatus{
+					Conditions: []configv1.ClusterOperatorStatusCondition{
+						{Type: "SCAAvailable", Message: "All systems operational"},
+					},
+				},
+			},
+			expected: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if isOCPBUG22226(&tt.co) != tt.expected {
+				t.Fatalf("expected %v, got %v", tt.expected, !tt.expected)
+			}
+		})
+	}
+}

pkg/investigations/insightsoperatordown/testing/README.md

@@ -0,0 +1,20 @@
+# Testing InsightsOperatorDownSRE
+
+# OCPBUGS-22226
+
+We can induce the symptom of `Failed to pull SCA certs` on a stage cluster by blocking `https://api.stage.openshift.com`
+The provided script creates a Rule Group and associates it with your clusters VPC.
+Requires awscli and backplane
+
+```
+./pkg/investigations/insightsoperatordown/testing/block-api-openshift.sh <cluster-id>
+```
+
+# Banned user
+
+TODO
+
+# Additional Resources
+
+- SOP Link https://github.com/openshift/ops-sop/blob/master/v4/troubleshoot/clusteroperators/insights.md
+- Alert Definition https://github.com/openshift/managed-cluster-config/blob/master/deploy/sre-prometheus/insights/100-sre-insightsoperator.PrometheusRule.yaml

pkg/investigations/insightsoperatordown/testing/block-api-openshift.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -eox pipefail
+AWS_PAGER=""
+$(ocm backplane cloud credentials -o env $1)
+AWS_REGION=$(ocm describe cluster $1 --json | jq -r '.region.id')
+FW_RULE_GROUP_ID=$(aws route53resolver create-firewall-rule-group --name "api stage openshift com"  | jq -r '.FirewallRuleGroup.Id')
+FW_DOMAIN_LIST_ID=$(aws route53resolver create-firewall-domain-list --name "api stage openshift com" | jq -r '.FirewallDomainList.Id')
+aws route53resolver update-firewall-domains --firewall-domain-list-id $FW_DOMAIN_LIST_ID --domains "api.stage.openshift.com" --operation "ADD"
+aws route53resolver create-firewall-rule --firewall-rule-group-id $FW_RULE_GROUP_ID --firewall-domain-list-id $FW_DOMAIN_LIST_ID --priority "1" --action "BLOCK" --block-response "NODATA" --name "api stage openshift com"
+INFRA_ID=$(ocm describe cluster $1 --json | jq -r '.infra_id')
+VPC_ID=$(aws ec2 describe-vpcs --filters "Name=tag-key,Values=kubernetes.io/cluster/$INFRA_ID" | jq -r '.Vpcs[0].VpcId')
+aws route53resolver associate-firewall-rule-group --firewall-rule-group-id $FW_RULE_GROUP_ID --name "rgassoc-$VPC_ID-$FW_RULE_GROUP_ID" --priority "1001" --vpc-id $VPC_ID
+

pkg/investigations/registry.go

@@ -5,6 +5,7 @@ import (
 	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/chgm"
 	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/clustermonitoringerrorbudgetburn"
 	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/cpd"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/insightsoperatordown"
 	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
 )
 
@@ -14,6 +15,7 @@ var availableInvestigations = []investigation.Investigation{
 	&chgm.Investiation{},
 	&clustermonitoringerrorbudgetburn.Investigation{},
 	&cpd.Investigation{},
+	&insightsoperatordown.Investigation{},
 }
 
 // GetInvestigation returns the first Investigation that applies to the given alert title.

pkg/ocm/ocm.go

@@ -12,6 +12,7 @@ import (
 
 	sdk "github.com/openshift-online/ocm-sdk-go"
 
+	amv1 "github.com/openshift-online/ocm-sdk-go/accountsmgmt/v1"
 	cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1"
 	servicelogsv1 "github.com/openshift-online/ocm-sdk-go/servicelogs/v1"
 	awsv1alpha1 "github.com/openshift/aws-account-operator/api/v1alpha1"
@@ -308,3 +309,30 @@ func (c *SdkClient) IsAccessProtected(cluster *cmv1.Cluster) (bool, error) {
 	}
 	return enabled, nil
 }
+
+func GetCreatorFromCluster(ocmConn *sdk.Connection, cluster *cmv1.Cluster) (*amv1.Account, error) {
+	logging.Debugf("Getting subscription from cluster: %s", cluster.ID())
+	cmv1Subscription, ok := cluster.GetSubscription()
+	if !ok {
+		return nil, fmt.Errorf("failed to get subscription from cluster: %s", cluster.ID())
+	}
+	subscriptionResponse, err := ocmConn.AccountsMgmt().V1().Subscriptions().Subscription(cmv1Subscription.ID()).Get().Send()
+	if err != nil {
+		return nil, err
+	}
+
+	subscription, ok := subscriptionResponse.GetBody()
+	if !ok {
+		return nil, errors.New("failed to get subscription")
+	}
+
+	if status := subscription.Status(); status != "Active" {
+		return nil, fmt.Errorf("Expecting status 'Active' found %v\n", status)
+	}
+
+	creator, ok := subscription.GetCreator()
+	if !ok {
+		return nil, errors.New("failed to get creator from subscription")
+	}
+	return creator, nil
+}

test/generate_incident.sh

@@ -7,6 +7,7 @@ declare -A alert_mapping=(
     ["ClusterHasGoneMissing"]="cadtest has gone missing"
     ["ClusterProvisioningDelay"]="ClusterProvisioningDelay -"
     ["ClusterMonitoringErrorBudgetBurnSRE"]="ClusterMonitoringErrorBudgetBurnSRE Critical (1)"
+    ["InsightsOperatorDown"]="InsightsOperatorDown"
 )
 
 # Function to print help message