diff --git a/README.md b/README.md index 29a46ef..d1cc468 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ## Overview -JIRAlert implements Alertmanager's webhook HTTP API and connects to one or more JIRA instances to create highly configurable JIRA issues. One issue is created per distinct group key — as defined by the [`group_by`](https://prometheus.io/docs/alerting/configuration/#) parameter of Alertmanager's `route` configuration section — but not closed when the alert is resolved. The expectation is that a human will look at the issue, take any necessary action, then close it. If no human interaction is necessary then it should probably not alert in the first place. +JIRAlert implements Alertmanager's webhook HTTP API and connects to one or more JIRA instances to create highly configurable JIRA issues. One issue is created per distinct group key — as defined by the [`group_by`](https://prometheus.io/docs/alerting/configuration/#) parameter of Alertmanager's `route` configuration section — but not closed when the alert is resolved. The expectation is that a human will look at the issue, take any necessary action, then close it. If no human interaction is necessary then it should probably not alert in the first place. This behavior however can be modified by setting `auto_resolve` section, which will resolve the jira issue with required state. If a corresponding JIRA issue already exists but is resolved, it is reopened. A JIRA transition must exist between the resolved state and the reopened state — as defined by `reopen_state` — or reopening will fail. Optionally a "won't fix" resolution — defined by `wont_fix_resolution` — may be defined: a JIRA issue with this resolution will not be reopened by JIRAlert. diff --git a/examples/jiralert.yml b/examples/jiralert.yml index 1ec7563..568437a 100644 --- a/examples/jiralert.yml +++ b/examples/jiralert.yml @@ -47,6 +47,10 @@ receivers: customfield_10002: {"value": "red"} # MultiSelect customfield_10003: [{"value": "red"}, {"value": "blue"}, {"value": "green"}] + # + # Automatically resolve jira issues when alert is resolved. Optional. If declared, ensure state is not an empty string. + auto_resolve: + state: 'Done' # File containing template definitions. Required. template: jiralert.tmpl diff --git a/pkg/alertmanager/alertmanager.go b/pkg/alertmanager/alertmanager.go index 48c05c0..55217f9 100644 --- a/pkg/alertmanager/alertmanager.go +++ b/pkg/alertmanager/alertmanager.go @@ -27,6 +27,9 @@ const ( // AlertFiring is the status value for a firing alert. AlertFiring = "firing" + + // AlertResolved is the status value for a resolved alert. + AlertResolved = "resolved" ) // Pair is a key/value string pair. diff --git a/pkg/config/config.go b/pkg/config/config.go index 1886de5..af23c68 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -88,6 +88,11 @@ func resolveFilepaths(baseDir string, cfg *Config, logger log.Logger) { cfg.Template = join(cfg.Template) } +// AutoResolve is the struct used for defining jira resolution state when alert is resolved. +type AutoResolve struct { + State string `yaml:"state" json:"state"` +} + // ReceiverConfig is the configuration for one receiver. It has a unique name and includes API access fields (url and // auth) and issue fields (required -- e.g. project, issue type -- and optional -- e.g. priority). type ReceiverConfig struct { @@ -116,6 +121,9 @@ type ReceiverConfig struct { // Label copy settings AddGroupLabels bool `yaml:"add_group_labels" json:"add_group_labels"` + // Flag to auto-resolve opened issue when the alert is resolved. + AutoResolve *AutoResolve `yaml:"auto_resolve" json:"auto_resolve"` + // Catches all undefined fields and must be empty after parsing. XXX map[string]interface{} `yaml:",inline" json:"-"` } @@ -171,6 +179,12 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { return fmt.Errorf("bad auth config in defaults section: user/password and PAT authentication are mutually exclusive") } + if c.Defaults.AutoResolve != nil { + if c.Defaults.AutoResolve.State == "" { + return fmt.Errorf("bad config in defaults section: state cannot be empty") + } + } + for _, rc := range c.Receivers { if rc.Name == "" { return fmt.Errorf("missing name for receiver %+v", rc) @@ -251,6 +265,14 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { if rc.WontFixResolution == "" && c.Defaults.WontFixResolution != "" { rc.WontFixResolution = c.Defaults.WontFixResolution } + if rc.AutoResolve != nil { + if rc.AutoResolve.State == "" { + return fmt.Errorf("bad config in receiver %q, 'auto_resolve' was defined with empty 'state' field", rc.Name) + } + } + if rc.AutoResolve == nil && c.Defaults.AutoResolve != nil { + rc.AutoResolve = c.Defaults.AutoResolve + } if len(c.Defaults.Fields) > 0 { for key, value := range c.Defaults.Fields { if _, ok := rc.Fields[key]; !ok { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 4537fb5..666fae7 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -111,6 +111,8 @@ type receiverTestConfig struct { WontFixResolution string `yaml:"wont_fix_resolution,omitempty"` AddGroupLabels bool `yaml:"add_group_labels,omitempty"` + AutoResolve *AutoResolve `yaml:"auto_resolve" json:"auto_resolve"` + // TODO(rporres): Add support for these. // Fields map[string]interface{} `yaml:"fields,omitempty"` // Components []string `yaml:"components,omitempty"` @@ -290,6 +292,7 @@ func TestAuthKeysOverrides(t *testing.T) { // No tests for auth keys here. They will be handled separately func TestReceiverOverrides(t *testing.T) { fifteenHoursToDuration, err := ParseDuration("15h") + autoResolve := AutoResolve{State: "Done"} require.NoError(t, err) // We'll override one key at a time and check the value in the receiver. @@ -308,8 +311,9 @@ func TestReceiverOverrides(t *testing.T) { {"Description", "A nice description", "A nice description"}, {"WontFixResolution", "Won't Fix", "Won't Fix"}, {"AddGroupLabels", false, false}, + {"AutoResolve", &AutoResolve{State: "Done"}, &autoResolve}, } { - optionalFields := []string{"Priority", "Description", "WontFixResolution", "AddGroupLabels"} + optionalFields := []string{"Priority", "Description", "WontFixResolution", "AddGroupLabels", "AutoResolve"} defaultsConfig := newReceiverTestConfig(mandatoryReceiverFields(), optionalFields) receiverConfig := newReceiverTestConfig([]string{"Name"}, optionalFields) @@ -361,6 +365,8 @@ func newReceiverTestConfig(mandatory []string, optional []string) *receiverTestC var value reflect.Value if name == "AddGroupLabels" { value = reflect.ValueOf(true) + } else if name == "AutoResolve" { + value = reflect.ValueOf(&AutoResolve{State: "Done"}) } else { value = reflect.ValueOf(name) } @@ -399,3 +405,41 @@ func mandatoryReceiverFields() []string { return []string{"Name", "APIURL", "User", "Password", "Project", "IssueType", "Summary", "ReopenState", "ReopenDuration"} } + +func TestAutoResolveConfigReceiver(t *testing.T) { + mandatory := mandatoryReceiverFields() + minimalReceiverTestConfig := &receiverTestConfig{ + Name: "test", + AutoResolve: &AutoResolve{ + State: "", + }, + } + + defaultsConfig := newReceiverTestConfig(mandatory, []string{}) + config := testConfig{ + Defaults: defaultsConfig, + Receivers: []*receiverTestConfig{minimalReceiverTestConfig}, + Template: "jiralert.tmpl", + } + + configErrorTestRunner(t, config, "bad config in receiver \"test\", 'auto_resolve' was defined with empty 'state' field") + +} + +func TestAutoResolveConfigDefault(t *testing.T) { + mandatory := mandatoryReceiverFields() + minimalReceiverTestConfig := newReceiverTestConfig([]string{"Name"}, []string{"AutoResolve"}) + + defaultsConfig := newReceiverTestConfig(mandatory, []string{}) + defaultsConfig.AutoResolve = &AutoResolve{ + State: "", + } + config := testConfig{ + Defaults: defaultsConfig, + Receivers: []*receiverTestConfig{minimalReceiverTestConfig}, + Template: "jiralert.tmpl", + } + + configErrorTestRunner(t, config, "bad config in defaults section: state cannot be empty") + +} diff --git a/pkg/notify/notify.go b/pkg/notify/notify.go index 0cd7da3..b85de41 100644 --- a/pkg/notify/notify.go +++ b/pkg/notify/notify.go @@ -101,6 +101,15 @@ func (r *Receiver) Notify(data *alertmanager.Data, hashJiraLabel bool) (bool, er } if len(data.Alerts.Firing()) == 0 { + if r.conf.AutoResolve != nil { + level.Debug(r.logger).Log("msg", "no firing alert; resolving issue", "key", issue.Key, "label", issueGroupLabel) + retry, err := r.resolveIssue(issue.Key) + if err != nil { + return retry, err + } + return false, nil + } + level.Debug(r.logger).Log("msg", "no firing alert; summary checked, nothing else to do.", "key", issue.Key, "label", issueGroupLabel) return false, nil } @@ -343,24 +352,7 @@ func (r *Receiver) updateDescription(issueKey string, description string) (bool, } func (r *Receiver) reopen(issueKey string) (bool, error) { - transitions, resp, err := r.client.GetTransitions(issueKey) - if err != nil { - return handleJiraErrResponse("Issue.GetTransitions", resp, err, r.logger) - } - - for _, t := range transitions { - if t.Name == r.conf.ReopenState { - level.Debug(r.logger).Log("msg", "transition (reopen)", "key", issueKey, "transitionID", t.ID) - resp, err = r.client.DoTransition(issueKey, t.ID) - if err != nil { - return handleJiraErrResponse("Issue.DoTransition", resp, err, r.logger) - } - - level.Debug(r.logger).Log("msg", "reopened", "key", issueKey) - return false, nil - } - } - return false, errors.Errorf("JIRA state %q does not exist or no transition possible for %s", r.conf.ReopenState, issueKey) + return r.doTransition(issueKey, r.conf.ReopenState) } func (r *Receiver) create(issue *jira.Issue) (bool, error) { @@ -390,3 +382,29 @@ func handleJiraErrResponse(api string, resp *jira.Response, err error, logger lo } return false, errors.Wrapf(err, "JIRA request %s failed", api) } + +func (r *Receiver) resolveIssue(issueKey string) (bool, error) { + return r.doTransition(issueKey, r.conf.AutoResolve.State) +} + +func (r *Receiver) doTransition(issueKey string, transitionState string) (bool, error) { + transitions, resp, err := r.client.GetTransitions(issueKey) + if err != nil { + return handleJiraErrResponse("Issue.GetTransitions", resp, err, r.logger) + } + + for _, t := range transitions { + if t.Name == transitionState { + level.Debug(r.logger).Log("msg", fmt.Sprintf("transition %s", transitionState), "key", issueKey, "transitionID", t.ID) + resp, err = r.client.DoTransition(issueKey, t.ID) + if err != nil { + return handleJiraErrResponse("Issue.DoTransition", resp, err, r.logger) + } + + level.Debug(r.logger).Log("msg", transitionState, "key", issueKey) + return false, nil + } + } + return false, errors.Errorf("JIRA state %q does not exist or no transition possible for %s", r.conf.ReopenState, issueKey) + +} diff --git a/pkg/notify/notify_test.go b/pkg/notify/notify_test.go index a307161..19c10ef 100644 --- a/pkg/notify/notify_test.go +++ b/pkg/notify/notify_test.go @@ -46,7 +46,7 @@ type fakeJira struct { func newTestFakeJira() *fakeJira { return &fakeJira{ issuesByKey: map[string]*jira.Issue{}, - transitionsByID: map[string]jira.Transition{}, + transitionsByID: map[string]jira.Transition{"1234": {ID: "1234", Name: "Done"}}, keysByQuery: map[string][]string{}, } } @@ -174,6 +174,19 @@ func testReceiverConfig2() *config.ReceiverConfig { } } +func testReceiverConfigAutoResolve() *config.ReceiverConfig { + reopen := config.Duration(1 * time.Hour) + autoResolve := config.AutoResolve{State: "Done"} + return &config.ReceiverConfig{ + Project: "abc", + Summary: `[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}`, + ReopenDuration: &reopen, + ReopenState: "reopened", + WontFixResolution: "won't-fix", + AutoResolve: &autoResolve, + } +} + func TestNotify_JIRAInteraction(t *testing.T) { testNowTime := time.Now() @@ -479,6 +492,49 @@ func TestNotify_JIRAInteraction(t *testing.T) { }, }, }, + { + name: "auto resolve alert", + inputConfig: testReceiverConfigAutoResolve(), + inputAlert: &alertmanager.Data{ + Alerts: alertmanager.Alerts{ + {Status: "resolved"}, + }, + Status: alertmanager.AlertResolved, + GroupLabels: alertmanager.KV{"a": "b", "c": "d"}, + }, + initJira: func(t *testing.T) *fakeJira { + f := newTestFakeJira() + _, _, err := f.Create(&jira.Issue{ + ID: "1", + Key: "1", + Fields: &jira.IssueFields{ + Project: jira.Project{Key: testReceiverConfigAutoResolve().Project}, + Labels: []string{"JIRALERT{819ba5ecba4ea5946a8d17d285cb23f3bb6862e08bb602ab08fd231cd8e1a83a1d095b0208a661787e9035f0541817634df5a994d1b5d4200d6c68a7663c97f5}"}, + Unknowns: tcontainer.MarshalMap{}, + Summary: "[FIRING:2] b d ", + Description: "1", + }, + }) + require.NoError(t, err) + return f + }, + expectedJiraIssues: map[string]*jira.Issue{ + "1": { + ID: "1", + Key: "1", + Fields: &jira.IssueFields{ + Project: jira.Project{Key: testReceiverConfigAutoResolve().Project}, + Labels: []string{"JIRALERT{819ba5ecba4ea5946a8d17d285cb23f3bb6862e08bb602ab08fd231cd8e1a83a1d095b0208a661787e9035f0541817634df5a994d1b5d4200d6c68a7663c97f5}"}, + Status: &jira.Status{ + StatusCategory: jira.StatusCategory{Key: "Done"}, + }, + Unknowns: tcontainer.MarshalMap{}, + Summary: "[RESOLVED] b d ", // Title changed. + Description: "1", + }, + }, + }, + }, } { if ok := t.Run(tcase.name, func(t *testing.T) { fakeJira := tcase.initJira(t)