feat(alerting): update state if not been updated for 15min

pkg/api/alerting.go

@@ -140,6 +140,7 @@ func GetAlertStates(c *middleware.Context) Response {
 // PUT /api/alerts/events/:id
 func PutAlertState(c *middleware.Context, cmd models.UpdateAlertStateCommand) Response {
 	cmd.AlertId = c.ParamsInt64(":alertId")
+	cmd.OrgId = c.OrgId
 
 	query := models.GetAlertByIdQuery{Id: cmd.AlertId}
 	if err := bus.Dispatch(&query); err != nil {

pkg/models/alert_state.go

@@ -28,6 +28,7 @@ func (this *UpdateAlertStateCommand) IsValidState() bool {
 
 type UpdateAlertStateCommand struct {
 	AlertId  int64  `json:"alertId" binding:"Required"`
+	OrgId    int64  `json:"orgId" binding:"Required"`
 	NewState string `json:"newState" binding:"Required"`
 	Info     string `json:"info"`
 
@@ -42,3 +43,10 @@ type GetAlertsStateQuery struct {
 
 	Result *[]AlertState
 }
+
+type GetLastAlertStateQuery struct {
+	AlertId int64
+	OrgId   int64
+
+	Result *AlertState
+}

pkg/services/alerting/engine.go

@@ -143,14 +143,12 @@ func (e *Engine) resultHandler() {
 }
 
 func (e *Engine) reactToState(result *AlertResult) {
-	query := &m.GetAlertByIdQuery{Id: result.AlertJob.Rule.Id}
-	bus.Dispatch(query)
-
-	if query.Result.ShouldUpdateState(result.State) {
+	if shouldUpdateState(result) {
 		cmd := &m.UpdateAlertStateCommand{
 			AlertId:  result.AlertJob.Rule.Id,
 			NewState: result.State,
 			Info:     result.Description,
+			OrgId:    result.AlertJob.Rule.OrgId,
 		}
 
 		if err := bus.Dispatch(cmd); err != nil {
@@ -161,3 +159,22 @@ func (e *Engine) reactToState(result *AlertResult) {
 		e.notifier.Notify(result)
 	}
 }
+
+func shouldUpdateState(result *AlertResult) bool {
+	query := &m.GetLastAlertStateQuery{
+		AlertId: result.AlertJob.Rule.Id,
+		OrgId:   result.AlertJob.Rule.OrgId,
+	}
+
+	if err := bus.Dispatch(query); err != nil {
+		log.Error2("Failed to read last alert state", "error", err)
+		return false
+	}
+
+	now := time.Now()
+	noEarlierState := query.Result == nil
+	olderThen15Min := query.Result.Created.Before(now.Add(time.Minute * -15))
+	changedState := query.Result.NewState != result.State
+
+	return noEarlierState || changedState || olderThen15Min
+}

pkg/services/sqlstore/alert_state.go

@@ -12,6 +12,23 @@ import (
 func init() {
 	bus.AddHandler("sql", SetNewAlertState)
 	bus.AddHandler("sql", GetAlertStateLogByAlertId)
+	bus.AddHandler("sql", GetLastAlertStateQuery)
+}
+
+func GetLastAlertStateQuery(cmd *m.GetLastAlertStateQuery) error {
+	states := make([]m.AlertState, 0)
+
+	if err := x.Where("alert_id = ? and org_id = ? ", cmd.AlertId, cmd.OrgId).Desc("created").Find(&states); err != nil {
+		return err
+	}
+
+	if len(states) == 0 {
+		cmd.Result = nil
+		return fmt.Errorf("invalid amount of alertstates. Expected 1 got %v", len(states))
+	}
+
+	cmd.Result = &states[0]
+	return nil
 }
 
 func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error {
@@ -30,17 +47,12 @@ func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error {
 			return fmt.Errorf("Could not find alert")
 		}
 
-		if alert.State == cmd.NewState {
-			cmd.Result = &m.Alert{}
-			return nil
-		}
-
 		alert.State = cmd.NewState
 		sess.Id(alert.Id).Update(&alert)
 
 		alertState := m.AlertState{
 			AlertId:  cmd.AlertId,
-			OrgId:    cmd.AlertId,
+			OrgId:    cmd.OrgId,
 			NewState: cmd.NewState,
 			Info:     cmd.Info,
 			Created:  time.Now(),
@@ -54,12 +66,12 @@ func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error {
 }
 
 func GetAlertStateLogByAlertId(cmd *m.GetAlertsStateQuery) error {
-	alertLogs := make([]m.AlertState, 0)
+	states := make([]m.AlertState, 0)
 
-	if err := x.Where("alert_id = ?", cmd.AlertId).Desc("created").Find(&alertLogs); err != nil {
+	if err := x.Where("alert_id = ?", cmd.AlertId).Desc("created").Find(&states); err != nil {
 		return err
 	}
 
-	cmd.Result = &alertLogs
+	cmd.Result = &states
 	return nil
 }