Alerting: Keep last state on no data (#6354)

* feat(alerting): add support to keep last state on no data

closes #6332

* refactoring(alerting PR #6354): added new option type for NoData option so AlertStateType does not have to contain invalid state, #6354

pkg/models/alert.go

@@ -8,6 +8,7 @@ import (
 
 type AlertStateType string
 type AlertSeverityType string
+type NoDataOption string
 
 const (
 	AlertStateNoData    AlertStateType = "no_data"
@@ -17,10 +18,25 @@ const (
 	AlertStateOK        AlertStateType = "ok"
 )
 
+const (
+	NoDataSetNoData   NoDataOption = "no_data"
+	NoDataSetAlerting NoDataOption = "alerting"
+	NoDataSetOK       NoDataOption = "ok"
+	NoDataKeepState   NoDataOption = "keep_state"
+)
+
 func (s AlertStateType) IsValid() bool {
 	return s == AlertStateOK || s == AlertStateNoData || s == AlertStateExecError || s == AlertStatePaused
 }
 
+func (s NoDataOption) IsValid() bool {
+	return s == NoDataSetNoData || s == NoDataSetAlerting || s == NoDataSetOK || s == NoDataKeepState
+}
+
+func (s NoDataOption) ToAlertState() AlertStateType {
+	return AlertStateType(s)
+}
+
 type Alert struct {
 	Id             int64
 	Version        int64

pkg/services/alerting/result_handler.go

@@ -30,34 +30,35 @@ func NewResultHandler() *DefaultResultHandler {
 func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
 	oldState := evalContext.Rule.State
 
-	exeuctionError := ""
+	executionError := ""
 	annotationData := simplejson.New()
 	if evalContext.Error != nil {
 		handler.log.Error("Alert Rule Result Error", "ruleId", evalContext.Rule.Id, "error", evalContext.Error)
 		evalContext.Rule.State = m.AlertStateExecError
-		exeuctionError = evalContext.Error.Error()
-		annotationData.Set("errorMessage", exeuctionError)
+		executionError = evalContext.Error.Error()
+		annotationData.Set("errorMessage", executionError)
 	} else if evalContext.Firing {
 		evalContext.Rule.State = m.AlertStateAlerting
 		annotationData = simplejson.NewFromAny(evalContext.EvalMatches)
 	} else {
-		// handle no data case
 		if evalContext.NoDataFound {
-			evalContext.Rule.State = evalContext.Rule.NoDataState
+			if evalContext.Rule.NoDataState != m.NoDataKeepState {
+				evalContext.Rule.State = evalContext.Rule.NoDataState.ToAlertState()
+			}
 		} else {
 			evalContext.Rule.State = m.AlertStateOK
 		}
 	}
 
 	countStateResult(evalContext.Rule.State)
-	if evalContext.Rule.State != oldState {
+	if handler.shouldUpdateAlertState(evalContext, oldState) {
 		handler.log.Info("New state change", "alertId", evalContext.Rule.Id, "newState", evalContext.Rule.State, "oldState", oldState)
 
 		cmd := &m.SetAlertStateCommand{
 			AlertId:  evalContext.Rule.Id,
 			OrgId:    evalContext.Rule.OrgId,
 			State:    evalContext.Rule.State,
-			Error:    exeuctionError,
+			Error:    executionError,
 			EvalData: annotationData,
 		}
 
@@ -91,6 +92,10 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
 	return nil
 }
 
+func (handler *DefaultResultHandler) shouldUpdateAlertState(evalContext *EvalContext, oldState m.AlertStateType) bool {
+	return evalContext.Rule.State != oldState
+}
+
 func countStateResult(state m.AlertStateType) {
 	switch state {
 	case m.AlertStateAlerting:

pkg/services/alerting/result_handler_test.go

@@ -1,57 +1,29 @@
 package alerting
 
 // import (
+// 	"context"
 // 	"testing"
-// 	"time"
-//
-// 	"github.com/grafana/grafana/pkg/bus"
-// 	m "github.com/grafana/grafana/pkg/models"
-// 	"github.com/grafana/grafana/pkg/services/alerting/alertstates"
 //
+// 	"github.com/grafana/grafana/pkg/models"
 // 	. "github.com/smartystreets/goconvey/convey"
 // )
 //
 // func TestAlertResultHandler(t *testing.T) {
 // 	Convey("Test result Handler", t, func() {
-// 		resultHandler := ResultHandlerImpl{}
-// 		mockResult := &AlertResultContext{
-// 			Triggered: false,
-// 			Rule: &AlertRule{
-// 				Id:    1,
-// 				OrgId 1,
-// 			},
-// 		}
-// 		mockAlertState := &m.AlertState{}
-// 		bus.ClearBusHandlers()
-// 		bus.AddHandler("test", func(query *m.GetLastAlertStateQuery) error {
-// 			query.Result = mockAlertState
-// 			return nil
-// 		})
+//
+// 		handler := NewResultHandler()
+// 		evalContext := NewEvalContext(context.TODO(), &Rule{})
 //
 // 		Convey("Should update", func() {
 //
 // 			Convey("when no earlier alert state", func() {
-// 				mockAlertState = nil
-// 				So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
-// 			})
+// 				oldState := models.AlertStateOK
 //
-// 			Convey("alert state have changed", func() {
-// 				mockAlertState = &m.AlertState{
-// 					State: alertstates.Critical,
-// 				}
-// 				mockResult.Triggered = false
-// 				So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
-// 			})
+// 				evalContext.Rule.State = models.AlertStateAlerting
+// 				evalContext.Rule.NoDataState = models.NoDataKeepState
+// 				evalContext.NoDataFound = true
 //
-// 			Convey("last alert state was 15min ago", func() {
-// 				now := time.Now()
-// 				mockAlertState = &m.AlertState{
-// 					State:   alertstates.Critical,
-// 					Created: now.Add(time.Minute * -30),
-// 				}
-// 				mockResult.Triggered = true
-// 				mockResult.StartTime = time.Now()
-// 				So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
+// 				So(handler.shouldUpdateAlertState(evalContext, oldState), ShouldBeFalse)
 // 			})
 // 		})
 // 	})

pkg/services/alerting/rule.go

@@ -18,7 +18,7 @@ type Rule struct {
 	Frequency     int64
 	Name          string
 	Message       string
-	NoDataState   m.AlertStateType
+	NoDataState   m.NoDataOption
 	State         m.AlertStateType
 	Conditions    []Condition
 	Notifications []int64
@@ -76,7 +76,7 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
 	model.Message = ruleDef.Message
 	model.Frequency = ruleDef.Frequency
 	model.State = ruleDef.State
-	model.NoDataState = m.AlertStateType(ruleDef.Settings.Get("noDataState").MustString("no_data"))
+	model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
 
 	for _, v := range ruleDef.Settings.Get("notifications").MustArray() {
 		jsonModel := simplejson.NewFromAny(v)

public/app/features/alerting/alert_def.ts

@@ -40,6 +40,7 @@ var noDataModes = [
   {text: 'OK', value: 'ok'},
   {text: 'Alerting', value: 'alerting'},
   {text: 'No Data', value: 'no_data'},
+  {text: 'Keep Last', value: 'keep_last'},
 ];
 
 function createReducerPart(model) {