Add support to auto resolve jira issue when alert is resolved. (#117)

* Add initial code to support auto resolve feature Signed-off-by: Bhushan Thakur <[email protected]> * use constant Signed-off-by: Bhushan Thakur <[email protected]> * Resolve alert using transition Signed-off-by: Bhushan Thakur <[email protected]> * Update configuration tests Signed-off-by: Bhushan Thakur <[email protected]> * Update example configuration Signed-off-by: Bhushan Thakur <[email protected]> * Update README Signed-off-by: Bhushan Thakur <[email protected]> * move valid config check to receivers section Signed-off-by: Bhushan Thakur <[email protected]> * Use different configuration option for simplicity Signed-off-by: Bhushan Thakur <[email protected]> * remove old config options Signed-off-by: Bhushan Thakur <[email protected]> * add docstring and update readme Signed-off-by: Bhushan Thakur <[email protected]> * reuse AutoResolve struct in tests Signed-off-by: Bhushan Thakur <[email protected]> * add docstring for AutoResolve Signed-off-by: Bhushan Thakur <[email protected]> * Apply suggestions from code review Co-authored-by: Bartlomiej Plotka <[email protected]> Signed-off-by: Bhushan Thakur <[email protected]> * keep the reasoning why defaults is not auto-resolve Signed-off-by: Bhushan Thakur <[email protected]> * fix lint errors Signed-off-by: Bhushan Thakur <[email protected]> * keep newline in configuration yaml Signed-off-by: Bhushan Thakur <[email protected]> * fix readme Signed-off-by: Bhushan Thakur <[email protected]> Co-authored-by: Bartlomiej Plotka <[email protected]>
prometheus-community · Jun 4, 2022 · a0f0e80 · a0f0e80
1 parent 0c7c40f
commit a0f0e80
Show file tree

Hide file tree

Showing 7 changed files with 168 additions and 21 deletions.
diff --git a/README.md b/README.md
@@ -7,7 +7,7 @@
 
 ## Overview
 
-JIRAlert implements Alertmanager's webhook HTTP API and connects to one or more JIRA instances to create highly configurable JIRA issues. One issue is created per distinct group key — as defined by the [`group_by`](https://prometheus.io/docs/alerting/configuration/#<route>) parameter of Alertmanager's `route` configuration section — but not closed when the alert is resolved. The expectation is that a human will look at the issue, take any necessary action, then close it.  If no human interaction is necessary then it should probably not alert in the first place.
+JIRAlert implements Alertmanager's webhook HTTP API and connects to one or more JIRA instances to create highly configurable JIRA issues. One issue is created per distinct group key — as defined by the [`group_by`](https://prometheus.io/docs/alerting/configuration/#<route>) parameter of Alertmanager's `route` configuration section — but not closed when the alert is resolved. The expectation is that a human will look at the issue, take any necessary action, then close it.  If no human interaction is necessary then it should probably not alert in the first place. This behavior however can be modified by setting `auto_resolve` section, which will resolve the jira issue with required state.
 
 If a corresponding JIRA issue already exists but is resolved, it is reopened. A JIRA transition must exist between the resolved state and the reopened state — as defined by `reopen_state` — or reopening will fail. Optionally a "won't fix" resolution — defined by `wont_fix_resolution` — may be defined: a JIRA issue with this resolution will not be reopened by JIRAlert.
 

diff --git a/examples/jiralert.yml b/examples/jiralert.yml
@@ -47,6 +47,10 @@ receivers:
       customfield_10002: {"value": "red"}
       # MultiSelect
       customfield_10003: [{"value": "red"}, {"value": "blue"}, {"value": "green"}]
+    #
+    # Automatically resolve jira issues when alert is resolved. Optional. If declared, ensure state is not an empty string.
+    auto_resolve:
+      state: 'Done' 
 
 # File containing template definitions. Required.
 template: jiralert.tmpl
diff --git a/pkg/alertmanager/alertmanager.go b/pkg/alertmanager/alertmanager.go
@@ -27,6 +27,9 @@ const (
 
 	// AlertFiring is the status value for a firing alert.
 	AlertFiring = "firing"
+
+	// AlertResolved is the status value for a resolved alert.
+	AlertResolved = "resolved"
 )
 
 // Pair is a key/value string pair.

diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -88,6 +88,11 @@ func resolveFilepaths(baseDir string, cfg *Config, logger log.Logger) {
 	cfg.Template = join(cfg.Template)
 }
 
+// AutoResolve is the struct used for defining jira resolution state when alert is resolved.
+type AutoResolve struct {
+	State string `yaml:"state" json:"state"`
+}
+
 // ReceiverConfig is the configuration for one receiver. It has a unique name and includes API access fields (url and
 // auth) and issue fields (required -- e.g. project, issue type -- and optional -- e.g. priority).
 type ReceiverConfig struct {
@@ -116,6 +121,9 @@ type ReceiverConfig struct {
 	// Label copy settings
 	AddGroupLabels bool `yaml:"add_group_labels" json:"add_group_labels"`
 
+	// Flag to auto-resolve opened issue when the alert is resolved.
+	AutoResolve *AutoResolve `yaml:"auto_resolve" json:"auto_resolve"`
+
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline" json:"-"`
 }
@@ -171,6 +179,12 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		return fmt.Errorf("bad auth config in defaults section: user/password and PAT authentication are mutually exclusive")
 	}
 
+	if c.Defaults.AutoResolve != nil {
+		if c.Defaults.AutoResolve.State == "" {
+			return fmt.Errorf("bad config in defaults section: state cannot be empty")
+		}
+	}
+
 	for _, rc := range c.Receivers {
 		if rc.Name == "" {
 			return fmt.Errorf("missing name for receiver %+v", rc)
@@ -251,6 +265,14 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		if rc.WontFixResolution == "" && c.Defaults.WontFixResolution != "" {
 			rc.WontFixResolution = c.Defaults.WontFixResolution
 		}
+		if rc.AutoResolve != nil {
+			if rc.AutoResolve.State == "" {
+				return fmt.Errorf("bad config in receiver %q, 'auto_resolve' was defined with empty 'state' field", rc.Name)
+			}
+		}
+		if rc.AutoResolve == nil && c.Defaults.AutoResolve != nil {
+			rc.AutoResolve = c.Defaults.AutoResolve
+		}
 		if len(c.Defaults.Fields) > 0 {
 			for key, value := range c.Defaults.Fields {
 				if _, ok := rc.Fields[key]; !ok {

diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go
@@ -111,6 +111,8 @@ type receiverTestConfig struct {
 	WontFixResolution string `yaml:"wont_fix_resolution,omitempty"`
 	AddGroupLabels    bool   `yaml:"add_group_labels,omitempty"`
 
+	AutoResolve *AutoResolve `yaml:"auto_resolve" json:"auto_resolve"`
+
 	// TODO(rporres): Add support for these.
 	// Fields            map[string]interface{} `yaml:"fields,omitempty"`
 	// Components        []string               `yaml:"components,omitempty"`
@@ -290,6 +292,7 @@ func TestAuthKeysOverrides(t *testing.T) {
 // No tests for auth keys here. They will be handled separately
 func TestReceiverOverrides(t *testing.T) {
 	fifteenHoursToDuration, err := ParseDuration("15h")
+	autoResolve := AutoResolve{State: "Done"}
 	require.NoError(t, err)
 
 	// We'll override one key at a time and check the value in the receiver.
@@ -308,8 +311,9 @@ func TestReceiverOverrides(t *testing.T) {
 		{"Description", "A nice description", "A nice description"},
 		{"WontFixResolution", "Won't Fix", "Won't Fix"},
 		{"AddGroupLabels", false, false},
+		{"AutoResolve", &AutoResolve{State: "Done"}, &autoResolve},
 	} {
-		optionalFields := []string{"Priority", "Description", "WontFixResolution", "AddGroupLabels"}
+		optionalFields := []string{"Priority", "Description", "WontFixResolution", "AddGroupLabels", "AutoResolve"}
 		defaultsConfig := newReceiverTestConfig(mandatoryReceiverFields(), optionalFields)
 		receiverConfig := newReceiverTestConfig([]string{"Name"}, optionalFields)
 
@@ -361,6 +365,8 @@ func newReceiverTestConfig(mandatory []string, optional []string) *receiverTestC
 		var value reflect.Value
 		if name == "AddGroupLabels" {
 			value = reflect.ValueOf(true)
+		} else if name == "AutoResolve" {
+			value = reflect.ValueOf(&AutoResolve{State: "Done"})
 		} else {
 			value = reflect.ValueOf(name)
 		}
@@ -399,3 +405,41 @@ func mandatoryReceiverFields() []string {
 	return []string{"Name", "APIURL", "User", "Password", "Project",
 		"IssueType", "Summary", "ReopenState", "ReopenDuration"}
 }
+
+func TestAutoResolveConfigReceiver(t *testing.T) {
+	mandatory := mandatoryReceiverFields()
+	minimalReceiverTestConfig := &receiverTestConfig{
+		Name: "test",
+		AutoResolve: &AutoResolve{
+			State: "",
+		},
+	}
+
+	defaultsConfig := newReceiverTestConfig(mandatory, []string{})
+	config := testConfig{
+		Defaults:  defaultsConfig,
+		Receivers: []*receiverTestConfig{minimalReceiverTestConfig},
+		Template:  "jiralert.tmpl",
+	}
+
+	configErrorTestRunner(t, config, "bad config in receiver \"test\", 'auto_resolve' was defined with empty 'state' field")
+
+}
+
+func TestAutoResolveConfigDefault(t *testing.T) {
+	mandatory := mandatoryReceiverFields()
+	minimalReceiverTestConfig := newReceiverTestConfig([]string{"Name"}, []string{"AutoResolve"})
+
+	defaultsConfig := newReceiverTestConfig(mandatory, []string{})
+	defaultsConfig.AutoResolve = &AutoResolve{
+		State: "",
+	}
+	config := testConfig{
+		Defaults:  defaultsConfig,
+		Receivers: []*receiverTestConfig{minimalReceiverTestConfig},
+		Template:  "jiralert.tmpl",
+	}
+
+	configErrorTestRunner(t, config, "bad config in defaults section: state cannot be empty")
+
+}
diff --git a/pkg/notify/notify.go b/pkg/notify/notify.go
@@ -101,6 +101,15 @@ func (r *Receiver) Notify(data *alertmanager.Data, hashJiraLabel bool) (bool, er
 		}
 
 		if len(data.Alerts.Firing()) == 0 {
+			if r.conf.AutoResolve != nil {
+				level.Debug(r.logger).Log("msg", "no firing alert; resolving issue", "key", issue.Key, "label", issueGroupLabel)
+				retry, err := r.resolveIssue(issue.Key)
+				if err != nil {
+					return retry, err
+				}
+				return false, nil
+			}
+
 			level.Debug(r.logger).Log("msg", "no firing alert; summary checked, nothing else to do.", "key", issue.Key, "label", issueGroupLabel)
 			return false, nil
 		}
@@ -343,24 +352,7 @@ func (r *Receiver) updateDescription(issueKey string, description string) (bool,
 }
 
 func (r *Receiver) reopen(issueKey string) (bool, error) {
-	transitions, resp, err := r.client.GetTransitions(issueKey)
-	if err != nil {
-		return handleJiraErrResponse("Issue.GetTransitions", resp, err, r.logger)
-	}
-
-	for _, t := range transitions {
-		if t.Name == r.conf.ReopenState {
-			level.Debug(r.logger).Log("msg", "transition (reopen)", "key", issueKey, "transitionID", t.ID)
-			resp, err = r.client.DoTransition(issueKey, t.ID)
-			if err != nil {
-				return handleJiraErrResponse("Issue.DoTransition", resp, err, r.logger)
-			}
-
-			level.Debug(r.logger).Log("msg", "reopened", "key", issueKey)
-			return false, nil
-		}
-	}
-	return false, errors.Errorf("JIRA state %q does not exist or no transition possible for %s", r.conf.ReopenState, issueKey)
+	return r.doTransition(issueKey, r.conf.ReopenState)
 }
 
 func (r *Receiver) create(issue *jira.Issue) (bool, error) {
@@ -390,3 +382,29 @@ func handleJiraErrResponse(api string, resp *jira.Response, err error, logger lo
 	}
 	return false, errors.Wrapf(err, "JIRA request %s failed", api)
 }
+
+func (r *Receiver) resolveIssue(issueKey string) (bool, error) {
+	return r.doTransition(issueKey, r.conf.AutoResolve.State)
+}
+
+func (r *Receiver) doTransition(issueKey string, transitionState string) (bool, error) {
+	transitions, resp, err := r.client.GetTransitions(issueKey)
+	if err != nil {
+		return handleJiraErrResponse("Issue.GetTransitions", resp, err, r.logger)
+	}
+
+	for _, t := range transitions {
+		if t.Name == transitionState {
+			level.Debug(r.logger).Log("msg", fmt.Sprintf("transition %s", transitionState), "key", issueKey, "transitionID", t.ID)
+			resp, err = r.client.DoTransition(issueKey, t.ID)
+			if err != nil {
+				return handleJiraErrResponse("Issue.DoTransition", resp, err, r.logger)
+			}
+
+			level.Debug(r.logger).Log("msg", transitionState, "key", issueKey)
+			return false, nil
+		}
+	}
+	return false, errors.Errorf("JIRA state %q does not exist or no transition possible for %s", r.conf.ReopenState, issueKey)
+
+}
diff --git a/pkg/notify/notify_test.go b/pkg/notify/notify_test.go
@@ -46,7 +46,7 @@ type fakeJira struct {
 func newTestFakeJira() *fakeJira {
 	return &fakeJira{
 		issuesByKey:     map[string]*jira.Issue{},
-		transitionsByID: map[string]jira.Transition{},
+		transitionsByID: map[string]jira.Transition{"1234": {ID: "1234", Name: "Done"}},
 		keysByQuery:     map[string][]string{},
 	}
 }
@@ -174,6 +174,19 @@ func testReceiverConfig2() *config.ReceiverConfig {
 	}
 }
 
+func testReceiverConfigAutoResolve() *config.ReceiverConfig {
+	reopen := config.Duration(1 * time.Hour)
+	autoResolve := config.AutoResolve{State: "Done"}
+	return &config.ReceiverConfig{
+		Project:           "abc",
+		Summary:           `[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}`,
+		ReopenDuration:    &reopen,
+		ReopenState:       "reopened",
+		WontFixResolution: "won't-fix",
+		AutoResolve:       &autoResolve,
+	}
+}
+
 func TestNotify_JIRAInteraction(t *testing.T) {
 	testNowTime := time.Now()
 
@@ -479,6 +492,49 @@ func TestNotify_JIRAInteraction(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:        "auto resolve alert",
+			inputConfig: testReceiverConfigAutoResolve(),
+			inputAlert: &alertmanager.Data{
+				Alerts: alertmanager.Alerts{
+					{Status: "resolved"},
+				},
+				Status:      alertmanager.AlertResolved,
+				GroupLabels: alertmanager.KV{"a": "b", "c": "d"},
+			},
+			initJira: func(t *testing.T) *fakeJira {
+				f := newTestFakeJira()
+				_, _, err := f.Create(&jira.Issue{
+					ID:  "1",
+					Key: "1",
+					Fields: &jira.IssueFields{
+						Project:     jira.Project{Key: testReceiverConfigAutoResolve().Project},
+						Labels:      []string{"JIRALERT{819ba5ecba4ea5946a8d17d285cb23f3bb6862e08bb602ab08fd231cd8e1a83a1d095b0208a661787e9035f0541817634df5a994d1b5d4200d6c68a7663c97f5}"},
+						Unknowns:    tcontainer.MarshalMap{},
+						Summary:     "[FIRING:2] b d ",
+						Description: "1",
+					},
+				})
+				require.NoError(t, err)
+				return f
+			},
+			expectedJiraIssues: map[string]*jira.Issue{
+				"1": {
+					ID:  "1",
+					Key: "1",
+					Fields: &jira.IssueFields{
+						Project: jira.Project{Key: testReceiverConfigAutoResolve().Project},
+						Labels:  []string{"JIRALERT{819ba5ecba4ea5946a8d17d285cb23f3bb6862e08bb602ab08fd231cd8e1a83a1d095b0208a661787e9035f0541817634df5a994d1b5d4200d6c68a7663c97f5}"},
+						Status: &jira.Status{
+							StatusCategory: jira.StatusCategory{Key: "Done"},
+						},
+						Unknowns:    tcontainer.MarshalMap{},
+						Summary:     "[RESOLVED] b d ", // Title changed.
+						Description: "1",
+					},
+				},
+			},
+		},
 	} {
 		if ok := t.Run(tcase.name, func(t *testing.T) {
 			fakeJira := tcase.initJira(t)