improve check for offline rules (#1013)

* improve check for offline rules

* bug fixes

* update dependencies

* fix error wrap

* fix offline check

* improve readability

cmd/dashboard/controller/service.go

@@ -11,6 +11,7 @@ import (
 	"github.com/jinzhu/copier"
 
 	"github.com/nezhahq/nezha/model"
+	"github.com/nezhahq/nezha/pkg/utils"
 	"github.com/nezhahq/nezha/service/singleton"
 	"gorm.io/gorm"
 )
@@ -31,8 +32,7 @@ func showService(c *gin.Context) (*model.ServiceResponse, error) {
 		stats := singleton.ServiceSentinelShared.CopyStats()
 		var cycleTransferStats map[uint64]model.CycleTransferStats
 		copier.Copy(&cycleTransferStats, singleton.AlertsCycleTransferStatsStore)
-		return []interface {
-		}{
+		return []any{
 			stats, cycleTransferStats,
 		}, nil
 	})
@@ -288,15 +288,12 @@ func updateService(c *gin.Context) (any, error) {
 		return nil, newGormError("%v", err)
 	}
 
-	var skipServers []uint64
-	for k := range m.SkipServers {
-		skipServers = append(skipServers, k)
-	}
+	skipServers := utils.MapKeysToSlice(mf.SkipServers)
 
-	if m.Cover == 0 {
+	if m.Cover == model.ServiceCoverAll {
 		err = singleton.DB.Unscoped().Delete(&model.ServiceHistory{}, "service_id = ? and server_id in (?)", m.ID, skipServers).Error
 	} else {
-		err = singleton.DB.Unscoped().Delete(&model.ServiceHistory{}, "service_id = ? and server_id not in (?)", m.ID, skipServers).Error
+		err = singleton.DB.Unscoped().Delete(&model.ServiceHistory{}, "service_id = ? and server_id not in (?) and server_id > 0", m.ID, skipServers).Error
 	}
 	if err != nil {
 		return nil, err

cmd/dashboard/main.go

@@ -23,6 +23,7 @@ import (
 	"github.com/nezhahq/nezha/cmd/dashboard/controller/waf"
 	"github.com/nezhahq/nezha/cmd/dashboard/rpc"
 	"github.com/nezhahq/nezha/model"
+	"github.com/nezhahq/nezha/pkg/utils"
 	"github.com/nezhahq/nezha/proto"
 	"github.com/nezhahq/nezha/service/singleton"
 )
@@ -154,6 +155,7 @@ func main() {
 	}
 
 	errChan := make(chan error, 2)
+	errHTTPS := errors.New("error from https server")
 
 	if err := graceful.Graceful(func() error {
 		log.Printf("NEZHA>> Dashboard::START ON %s:%d", singleton.Conf.ListenHost, singleton.Conf.ListenPort)
@@ -171,12 +173,16 @@ func main() {
 		log.Println("NEZHA>> Graceful::START")
 		singleton.RecordTransferHourlyUsage()
 		log.Println("NEZHA>> Graceful::END")
-		err := muxServerHTTPS.Shutdown(c)
-		return errors.Join(muxServerHTTP.Shutdown(c), err)
+		var err error
+		if muxServerHTTPS != nil {
+			err = muxServerHTTPS.Shutdown(c)
+		}
+		return errors.Join(muxServerHTTP.Shutdown(c), utils.IfOr(err != nil, utils.NewWrapError(errHTTPS, err), nil))
 	}); err != nil {
 		log.Printf("NEZHA>> ERROR: %v", err)
-		if errors.Unwrap(err) != nil {
-			log.Printf("NEZHA>> ERROR HTTPS: %v", err)
+		var wrapError *utils.WrapError
+		if errors.As(err, &wrapError) {
+			log.Printf("NEZHA>> ERROR HTTPS: %v", wrapError.Unwrap())
 		}
 	}
 

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/knadh/koanf/providers/env v1.0.0
 	github.com/knadh/koanf/providers/file v1.1.2
 	github.com/knadh/koanf/v2 v2.1.2
-	github.com/libdns/cloudflare v0.1.2
+	github.com/libdns/cloudflare v0.1.3
 	github.com/libdns/libdns v0.2.3
 	github.com/miekg/dns v1.1.63
 	github.com/nezhahq/libdns-tencentcloud v0.0.0-20241029120103-889957240fff

go.sum

@@ -99,8 +99,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
-github.com/libdns/cloudflare v0.1.2 h1:RWUqBSojAFpg2O/jzS29DnkCP9oWQj3LmNEU8OulTLs=
-github.com/libdns/cloudflare v0.1.2/go.mod h1:XbvSCSMcxspwpSialM3bq0LsS3/Houy9WYxW8Ok8b6M=
+github.com/libdns/cloudflare v0.1.3 h1:XPFa2f3Mm/3FDNwl9Ki2bfAQJ0Cm5GQB0e8PQVy25Us=
+github.com/libdns/cloudflare v0.1.3/go.mod h1:XbvSCSMcxspwpSialM3bq0LsS3/Houy9WYxW8Ok8b6M=
 github.com/libdns/libdns v0.2.3 h1:ba30K4ObwMGB/QTmqUxf3H4/GmUrCAIkMWejeGl12v8=
 github.com/libdns/libdns v0.2.3/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ=
 github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=

model/alertrule.go

@@ -1,6 +1,8 @@
 package model
 
 import (
+	"slices"
+
 	"github.com/goccy/go-json"
 	"gorm.io/gorm"
 )
@@ -72,40 +74,49 @@ func (r *AlertRule) Snapshot(cycleTransferStats *CycleTransferStats, server *Ser
 }
 
 // Check 传入包含当前报警规则下所有type检查结果 返回报警持续时间与是否通过报警检查(通过则返回true)
-func (r *AlertRule) Check(points [][]bool) (maxDuration int, passed bool) {
+func (r *AlertRule) Check(points [][]bool) (int, bool) {
 	var hasPassedRule bool
+	durations := make([]int, len(r.Rules))
 
-	for ruleId, rule := range r.Rules {
+	for ruleIndex, rule := range r.Rules {
+		fail, duration := 0, int(rule.Duration)
 		if rule.IsTransferDurationRule() {
 			// 循环区间流量报警
-			if maxDuration < 1 {
-				maxDuration = 1
+			if durations[ruleIndex] < 1 {
+				durations[ruleIndex] = 1
 			}
 			if hasPassedRule {
 				continue
 			}
 			// 只要最后一次检查超出了规则范围 就认为检查未通过
-			if len(points) > 0 && points[len(points)-1][ruleId] {
+			if len(points) > 0 && points[len(points)-1][ruleIndex] {
 				hasPassedRule = true
 			}
+		} else if rule.IsOfflineRule() {
+			// 离线报警，检查直到最后一次在线的离线采样点是否大于 duration
+			if hasPassedRule = boundCheck(len(points), duration, hasPassedRule); hasPassedRule {
+				continue
+			}
+			for timeTick := len(points); timeTick >= len(points)-duration; timeTick-- {
+				fail++
+				if points[timeTick][ruleIndex] {
+					hasPassedRule = true
+					break
+				}
+			}
+			durations[ruleIndex] = fail
+			continue
 		} else {
 			// 常规报警
-			duration := int(rule.Duration)
-			if duration > maxDuration {
-				maxDuration = duration
-			}
-			if hasPassedRule {
-				continue
+			if duration > durations[ruleIndex] {
+				durations[ruleIndex] = duration
 			}
-			if len(points) < duration {
-				// 如果采样点数量不足 则认为检查通过
-				hasPassedRule = true
+			if hasPassedRule = boundCheck(len(points), duration, hasPassedRule); hasPassedRule {
 				continue
 			}
-			total, fail := 0, 0
+			total, fail := duration, 0
 			for timeTick := len(points) - duration; timeTick < len(points); timeTick++ {
-				total++
-				if !points[timeTick][ruleId] {
+				if !points[timeTick][ruleIndex] {
 					fail++
 				}
 			}
@@ -117,5 +128,13 @@ func (r *AlertRule) Check(points [][]bool) (maxDuration int, passed bool) {
 	}
 
 	// 仅当所有检查均未通过时 才触发告警
-	return maxDuration, hasPassedRule
+	return slices.Max(durations), hasPassedRule
+}
+
+func boundCheck(length, duration int, passed bool) bool {
+	if passed {
+		return true
+	}
+	// 如果采样点数量不足 则认为检查通过
+	return length < duration
 }

model/rule.go

@@ -179,6 +179,10 @@ func (u *Rule) IsTransferDurationRule() bool {
 	return strings.HasSuffix(u.Type, "_cycle")
 }
 
+func (u *Rule) IsOfflineRule() bool {
+	return u.Type == "offline"
+}
+
 // GetTransferDurationStart 获取周期流量的起始时间
 func (u *Rule) GetTransferDurationStart() time.Time {
 	// Accept uppercase and lowercase

model/service_api.go

@@ -27,8 +27,8 @@ type ServiceResponseItem struct {
 	TotalUp     uint64       `json:"total_up"`
 	TotalDown   uint64       `json:"total_down"`
 	Delay       *[30]float32 `json:"delay,omitempty"`
-	Up          *[30]int     `json:"up,omitempty"`
-	Down        *[30]int     `json:"down,omitempty"`
+	Up          *[30]uint64  `json:"up,omitempty"`
+	Down        *[30]uint64  `json:"down,omitempty"`
 }
 
 func (r ServiceResponseItem) TotalUptime() float32 {

pkg/utils/utils.go

@@ -149,3 +149,19 @@ func ConvertSeq2[KIn, VIn, KOut, VOut any](seq iter.Seq2[KIn, VIn], f func(KIn,
 		}
 	}
 }
+
+type WrapError struct {
+	err, errIn error
+}
+
+func NewWrapError(err, errIn error) error {
+	return &WrapError{err, errIn}
+}
+
+func (e *WrapError) Error() string {
+	return e.err.Error()
+}
+
+func (e *WrapError) Unwrap() error {
+	return e.errIn
+}

service/rpc/nezha.go

@@ -63,11 +63,11 @@ func (s *NezhaHandler) RequestTask(stream pb.NezhaService_RequestTaskServer) err
 				copier.Copy(&curServer, server)
 				if cr.PushSuccessful && result.GetSuccessful() {
 					singleton.NotificationShared.SendNotification(cr.NotificationGroupID, fmt.Sprintf("[%s] %s, %s\n%s", singleton.Localizer.T("Scheduled Task Executed Successfully"),
-						cr.Name, server.Name, result.GetData()), nil, &curServer)
+						cr.Name, server.Name, result.GetData()), "", &curServer)
 				}
 				if !result.GetSuccessful() {
 					singleton.NotificationShared.SendNotification(cr.NotificationGroupID, fmt.Sprintf("[%s] %s, %s\n%s", singleton.Localizer.T("Scheduled Task Executed Failed"),
-						cr.Name, server.Name, result.GetData()), nil, &curServer)
+						cr.Name, server.Name, result.GetData()), "", &curServer)
 				}
 				singleton.DB.Model(cr).Updates(model.Cron{
 					LastExecutedAt: time.Now().Add(time.Second * -1 * time.Duration(result.GetDelay())),
@@ -103,7 +103,7 @@ func (s *NezhaHandler) ReportSystemState(stream pb.NezhaService_ReportSystemStat
 	for {
 		state, err = stream.Recv()
 		if err != nil {
-			log.Printf("NEZHA>> ReportSystemState eror: %v, clientID: %d\n", err, clientID)
+			log.Printf("NEZHA>> ReportSystemState error: %v, clientID: %d\n", err, clientID)
 			return nil
 		}
 		state := model.PB2State(state)
@@ -258,7 +258,7 @@ func (s *NezhaHandler) ReportGeoIP(c context.Context, r *pb.GeoIP) (*pb.GeoIP, e
 				server.Name, singleton.IPDesensitize(server.GeoIP.IP.Join()),
 				singleton.IPDesensitize(joinedIP),
 			),
-			nil)
+			"")
 	}
 
 	// 根据内置数据库查询 IP 地理位置

service/singleton/alertsentinel.go

@@ -186,7 +186,9 @@ func checkStatus() {
 			}
 			// 清理旧数据
 			if max > 0 && max < len(alertsStore[alert.ID][server.ID]) {
-				alertsStore[alert.ID][server.ID] = alertsStore[alert.ID][server.ID][len(alertsStore[alert.ID][server.ID])-max:]
+				index := len(alertsStore[alert.ID][server.ID]) - max
+				clear(alertsStore[alert.ID][server.ID][:index]) // for GC
+				alertsStore[alert.ID][server.ID] = alertsStore[alert.ID][server.ID][index:]
 			}
 		}
 	}

service/singleton/crontask.go

@@ -55,7 +55,7 @@ func NewCronClass() *CronClass {
 	// 向注册错误的计划任务所在通知组发送通知
 	for _, gid := range notificationGroupList {
 		notificationMsgMap[gid].WriteString(Localizer.T("] These tasks will not execute properly. Fix them in the admin dashboard."))
-		NotificationShared.SendNotification(gid, notificationMsgMap[gid].String(), nil)
+		NotificationShared.SendNotification(gid, notificationMsgMap[gid].String(), "")
 	}
 	cronx.Start()
 
@@ -151,7 +151,7 @@ func CronTrigger(cr *model.Cron, triggerServer ...uint64) func() {
 					// 保存当前服务器状态信息
 					curServer := model.Server{}
 					copier.Copy(&curServer, s)
-					NotificationShared.SendNotification(cr.NotificationGroupID, Localizer.Tf("[Task failed] %s: server %s is offline and cannot execute the task", cr.Name, s.Name), nil, &curServer)
+					NotificationShared.SendNotification(cr.NotificationGroupID, Localizer.Tf("[Task failed] %s: server %s is offline and cannot execute the task", cr.Name, s.Name), "", &curServer)
 				}
 			}
 			return
@@ -174,7 +174,7 @@ func CronTrigger(cr *model.Cron, triggerServer ...uint64) func() {
 				// 保存当前服务器状态信息
 				curServer := model.Server{}
 				copier.Copy(&curServer, s)
-				NotificationShared.SendNotification(cr.NotificationGroupID, Localizer.Tf("[Task failed] %s: server %s is offline and cannot execute the task", cr.Name, s.Name), nil, &curServer)
+				NotificationShared.SendNotification(cr.NotificationGroupID, Localizer.Tf("[Task failed] %s: server %s is offline and cannot execute the task", cr.Name, s.Name), "", &curServer)
 			}
 		}
 	}

service/singleton/notification.go

@@ -193,16 +193,16 @@ func (c *NotificationClass) sortList() {
 	c.sortedList = sortedList
 }
 
-func (c *NotificationClass) UnMuteNotification(notificationGroupID uint64, muteLabel *string) {
-	fullMuteLabel := *NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID))
+func (c *NotificationClass) UnMuteNotification(notificationGroupID uint64, muteLabel string) {
+	fullMuteLabel := NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID))
 	Cache.Delete(fullMuteLabel)
 }
 
 // SendNotification 向指定的通知方式组的所有通知方式发送通知
-func (c *NotificationClass) SendNotification(notificationGroupID uint64, desc string, muteLabel *string, ext ...*model.Server) {
-	if muteLabel != nil {
+func (c *NotificationClass) SendNotification(notificationGroupID uint64, desc string, muteLabel string, ext ...*model.Server) {
+	if muteLabel != "" {
 		// 将通知方式组名称加入静音标志
-		muteLabel := *NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID))
+		muteLabel := NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID))
 		// 通知防骚扰策略
 		var flag bool
 		if cacheN, has := Cache.Get(muteLabel); has {
@@ -261,42 +261,34 @@ type _NotificationMuteLabel struct{}
 
 var NotificationMuteLabel _NotificationMuteLabel
 
-func (_NotificationMuteLabel) IPChanged(serverId uint64) *string {
-	label := fmt.Sprintf("bf::ic-%d", serverId)
-	return &label
+func (_NotificationMuteLabel) IPChanged(serverId uint64) string {
+	return fmt.Sprintf("bf::ic-%d", serverId)
 }
 
-func (_NotificationMuteLabel) ServerIncident(alertId uint64, serverId uint64) *string {
-	label := fmt.Sprintf("bf::sei-%d-%d", alertId, serverId)
-	return &label
+func (_NotificationMuteLabel) ServerIncident(alertId uint64, serverId uint64) string {
+	return fmt.Sprintf("bf::sei-%d-%d", alertId, serverId)
 }
 
-func (_NotificationMuteLabel) ServerIncidentResolved(alertId uint64, serverId uint64) *string {
-	label := fmt.Sprintf("bf::seir-%d-%d", alertId, serverId)
-	return &label
+func (_NotificationMuteLabel) ServerIncidentResolved(alertId uint64, serverId uint64) string {
+	return fmt.Sprintf("bf::seir-%d-%d", alertId, serverId)
 }
 
-func (_NotificationMuteLabel) AppendNotificationGroupName(label *string, notificationGroupName string) *string {
-	newLabel := fmt.Sprintf("%s:%s", *label, notificationGroupName)
-	return &newLabel
+func (_NotificationMuteLabel) AppendNotificationGroupName(label string, notificationGroupName string) string {
+	return fmt.Sprintf("%s:%s", label, notificationGroupName)
 }
 
-func (_NotificationMuteLabel) ServiceLatencyMin(serviceId uint64) *string {
-	label := fmt.Sprintf("bf::sln-%d", serviceId)
-	return &label
+func (_NotificationMuteLabel) ServiceLatencyMin(serviceId uint64) string {
+	return fmt.Sprintf("bf::sln-%d", serviceId)
 }
 
-func (_NotificationMuteLabel) ServiceLatencyMax(serviceId uint64) *string {
-	label := fmt.Sprintf("bf::slm-%d", serviceId)
-	return &label
+func (_NotificationMuteLabel) ServiceLatencyMax(serviceId uint64) string {
+	return fmt.Sprintf("bf::slm-%d", serviceId)
 }
 
-func (_NotificationMuteLabel) ServiceStateChanged(serviceId uint64) *string {
-	label := fmt.Sprintf("bf::ssc-%d", serviceId)
-	return &label
+func (_NotificationMuteLabel) ServiceStateChanged(serviceId uint64) string {
+	return fmt.Sprintf("bf::ssc-%d", serviceId)
 }
 
-func (_NotificationMuteLabel) ServiceTLS(serviceId uint64, extraInfo string) *string {
-	label := fmt.Sprintf("bf::stls-%d-%s", serviceId, extraInfo)
-	return &label
+func (_NotificationMuteLabel) ServiceTLS(serviceId uint64, extraInfo string) string {
+	return fmt.Sprintf("bf::stls-%d-%s", serviceId, extraInfo)
 }