Add incident.io alerting

Author: corneliusludmann

hack/diff-satellite.sh

@@ -18,6 +18,8 @@ jsonnet -c -J vendor -m diff/jsonnet-tmp \
     clusterName: 'fake-cluster',
     alerting: {
         pagerdutyRoutingKey: 'pd-routing-key',
+        incidentIoURL: 'https://example.com/',
+        incidentIoAuthToken: 'incident-io-token'
         slackOAuthToken: 'fake-key',
 
         ide: {},

installer/pkg/components/alertmanager/configsecret.go

@@ -17,7 +17,7 @@ const queryString = `{{ reReplaceAll "%22" "%5C%22" (index .Alerts 0).GeneratorU
 func configSecret(ctx *common.RenderContext) ([]runtime.Object, error) {
 	var receivers []*config.Receiver
 
-	receivers = append(receivers, criticalReceiver(ctx))
+	receivers = append(receivers, criticalReceivers(ctx)...)
 	receivers = append(receivers, defaultReceivers(ctx)...)
 	receivers = append(receivers, teamSlackReceivers(ctx)...)
 	resolveTimeout, _ := model.ParseDuration("5m")
@@ -59,6 +59,14 @@ func configSecret(ctx *common.RenderContext) ([]runtime.Object, error) {
 func routes(ctx *common.RenderContext) []*config.Route {
 	var routes []*config.Route
 
+	routes = append(routes, &config.Route{
+		Receiver: "criticalReceiverIncidentIO",
+		Match: map[string]string{
+			"severity": "critical",
+		},
+		Continue: true,
+	})
+
 	routes = append(routes, &config.Route{
 		Receiver: "criticalReceiver",
 		Match: map[string]string{
@@ -114,9 +122,11 @@ func inhibitRules() []*config.InhibitRule {
 	return inhibitRules
 }
 
-func criticalReceiver(ctx *common.RenderContext) *config.Receiver {
+func criticalReceivers(ctx *common.RenderContext) []*config.Receiver {
+	var receivers []*config.Receiver
+
 	if ctx.Config.Alerting.PagerDutyRoutingKey != "" {
-		return &config.Receiver{
+		receivers = append(receivers, &config.Receiver{
 			Name: "criticalReceiver",
 			PagerdutyConfigs: []*config.PagerdutyConfig{
 				{
@@ -138,9 +148,28 @@ func criticalReceiver(ctx *common.RenderContext) *config.Receiver {
 					},
 				},
 			},
-		}
-	} else {
-		return &config.Receiver{
+		})
+	}
+
+	if ctx.Config.Alerting.IncidentIoURL != "" && ctx.Config.Alerting.IncidentIoAuthToken != "" {
+		receivers = append(receivers, &config.Receiver{
+			Name: "criticalReceiverIncidentIO",
+			WebhookConfigs: []*config.WebhookConfig{
+				{
+					VSendResolved: common.ToPointer(true),
+					URL:           ctx.Config.Alerting.IncidentIoURL,
+					HTTPConfig: &config.HTTPClientConfig{
+						Authorization: &config.Authorization{
+							Credentials: ctx.Config.Alerting.IncidentIoAuthToken,
+						},
+					},
+				},
+			},
+		})
+	}
+
+	if len(receivers) == 0 {
+		receivers = append(receivers, &config.Receiver{
 			Name: "criticalReceiver",
 			SlackConfigs: []*config.SlackConfig{
 				{
@@ -158,8 +187,10 @@ func criticalReceiver(ctx *common.RenderContext) *config.Receiver {
 					Actions: slackButtons(),
 				},
 			},
-		}
+		})
 	}
+
+	return receivers
 }
 
 func defaultReceivers(ctx *common.RenderContext) []*config.Receiver {

installer/pkg/config/alertmanagerconfig_types.go

@@ -106,7 +106,7 @@ type Receiver struct {
 	OpsgenieConfigs  []*opsgenieConfig  `yaml:"opsgenie_configs,omitempty" json:"opsgenie_configs,omitempty"`
 	PagerdutyConfigs []*PagerdutyConfig `yaml:"pagerduty_configs,omitempty" json:"pagerduty_configs,omitempty"`
 	SlackConfigs     []*SlackConfig     `yaml:"slack_configs,omitempty" json:"slack_configs,omitempty"`
-	WebhookConfigs   []*webhookConfig   `yaml:"webhook_configs,omitempty" json:"webhook_configs,omitempty"`
+	WebhookConfigs   []*WebhookConfig   `yaml:"webhook_configs,omitempty" json:"webhook_configs,omitempty"`
 	WeChatConfigs    []*weChatConfig    `yaml:"wechat_configs,omitempty" json:"wechat_config,omitempty"`
 	EmailConfigs     []*emailConfig     `yaml:"email_configs,omitempty" json:"email_configs,omitempty"`
 	PushoverConfigs  []*pushoverConfig  `yaml:"pushover_configs,omitempty" json:"pushover_configs,omitempty"`
@@ -115,7 +115,7 @@ type Receiver struct {
 	TelegramConfigs  []*telegramConfig  `yaml:"telegram_configs,omitempty" json:"telegram_configs,omitempty"`
 }
 
-type webhookConfig struct {
+type WebhookConfig struct {
 	VSendResolved *bool             `yaml:"send_resolved,omitempty" json:"send_resolved,omitempty"`
 	URL           string            `yaml:"url,omitempty" json:"url,omitempty"`
 	HTTPConfig    *HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"`

installer/pkg/config/config.go

@@ -83,6 +83,8 @@ type Tracing struct {
 
 type Alerting struct {
 	PagerDutyRoutingKey string              `json:"pagerdutyRoutingKey,omitempty"`
+	IncidentIoURL       string              `json:"incidentIoURL,omitempty"`
+	IncidentIoAuthToken string              `json:"incidentIoAuthToken,omitempty"`
 	SlackOAuthToken     string              `json:"slackOAuthToken"`
 	GenericSlackChannel string              `json:"genericSlackChannel"`
 	TeamRoutes          []TeamAlertingRoute `json:"teamRoutes,omitempty"`

monitoring-central/monitoring-central.libsonnet

@@ -112,7 +112,7 @@ local kubePrometheus =
           _config+: {
             alertmanagerClusterLabels: 'cluster',
             alertmanagerNameLabels: 'namespace,pod',
-            alertmanagerCriticalIntegrationsRegEx: 'slack|pagerduty',
+            alertmanagerCriticalIntegrationsRegEx: 'slack|pagerduty|webhook',
           },
         },
       },

monitoring-satellite/manifests/grafana/dashboardDefinitions.yaml

@@ -35765,7 +35765,7 @@ items:
                       "hide": 2,
                       "includeAll": true,
                       "name": "integration",
-                      "query": "label_values(alertmanager_notifications_total{integration=~\"slack|pagerduty\"}, integration)",
+                      "query": "label_values(alertmanager_notifications_total{integration=~\"slack|pagerduty|webhook\"}, integration)",
                       "refresh": 2,
                       "sort": 1,
                       "type": "query"

monitoring-satellite/monitoring-satellite.libsonnet

@@ -30,7 +30,7 @@ local gitpod = import '../components/gitpod/gitpod.libsonnet';
         _config+: {
           alertmanagerClusterLabels: 'cluster',
           alertmanagerNameLabels: 'namespace,pod',
-          alertmanagerCriticalIntegrationsRegEx: 'slack|pagerduty',
+          alertmanagerCriticalIntegrationsRegEx: 'slack|pagerduty|webhook',
         },
       },
     },