Merge pull request #6420 from utkarshcmu/alerting_initial_state

Alerts not sent when state is changed from "initialized" to "ok"

pkg/api/alerting.go

@@ -264,7 +264,7 @@ func PauseAlert(c *middleware.Context, dto dtos.PauseAlertCommand) Response {
 		return ApiError(500, "", err)
 	}
 
-	var response models.AlertStateType = models.AlertStateNoData
+	var response models.AlertStateType = models.AlertStatePending
 	pausedState := "un paused"
 	if cmd.Paused {
 		response = models.AlertStatePaused

pkg/metrics/metrics.go

@@ -41,6 +41,7 @@ var (
 	M_Alerting_Result_State_Paused       Counter
 	M_Alerting_Result_State_NoData       Counter
 	M_Alerting_Result_State_ExecError    Counter
+	M_Alerting_Result_State_Pending			 Counter
 	M_Alerting_Active_Alerts             Counter
 	M_Alerting_Notification_Sent_Slack   Counter
 	M_Alerting_Notification_Sent_Email   Counter
@@ -102,6 +103,7 @@ func initMetricVars(settings *MetricSettings) {
 	M_Alerting_Result_State_Paused = RegCounter("alerting.result", "state", "paused")
 	M_Alerting_Result_State_NoData = RegCounter("alerting.result", "state", "no_data")
 	M_Alerting_Result_State_ExecError = RegCounter("alerting.result", "state", "exec_error")
+	M_Alerting_Result_State_Pending = RegCounter("alerting.result", "state", "pending")
 
 	M_Alerting_Active_Alerts = RegCounter("alerting.active_alerts")
 	M_Alerting_Notification_Sent_Slack = RegCounter("alerting.notifications_sent", "type", "slack")

pkg/models/alert.go

@@ -11,11 +11,12 @@ type AlertSeverityType string
 type NoDataOption string
 
 const (
-	AlertStateNoData    AlertStateType = "no_data"
-	AlertStateExecError AlertStateType = "execution_error"
-	AlertStatePaused    AlertStateType = "paused"
-	AlertStateAlerting  AlertStateType = "alerting"
-	AlertStateOK        AlertStateType = "ok"
+	AlertStateNoData    	 AlertStateType = "no_data"
+	AlertStateExecError 	 AlertStateType = "execution_error"
+	AlertStatePaused    	 AlertStateType = "paused"
+	AlertStateAlerting     AlertStateType = "alerting"
+	AlertStateOK           AlertStateType = "ok"
+	AlertStatePending      AlertStateType = "pending"
 )
 
 const (
@@ -26,7 +27,7 @@ const (
 )
 
 func (s AlertStateType) IsValid() bool {
-	return s == AlertStateOK || s == AlertStateNoData || s == AlertStateExecError || s == AlertStatePaused
+	return s == AlertStateOK || s == AlertStateNoData || s == AlertStateExecError || s == AlertStatePaused || s == AlertStatePending
 }
 
 func (s NoDataOption) IsValid() bool {

pkg/services/alerting/result_handler.go

@@ -86,7 +86,12 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
 			handler.log.Error("Failed to save annotation for new alert state", "error", err)
 		}
 
-		handler.notifier.Notify(evalContext)
+		if (oldState == m.AlertStatePending) && (evalContext.Rule.State == m.AlertStateOK) {
+			handler.log.Info("Notfication not sent", "oldState", oldState, "newState", evalContext.Rule.State)
+		} else {
+			handler.notifier.Notify(evalContext)
+		}
+
 	}
 
 	return nil
@@ -98,6 +103,8 @@ func (handler *DefaultResultHandler) shouldUpdateAlertState(evalContext *EvalCon
 
 func countStateResult(state m.AlertStateType) {
 	switch state {
+	case m.AlertStatePending:
+		metrics.M_Alerting_Result_State_Pending.Inc(1)
 	case m.AlertStateAlerting:
 		metrics.M_Alerting_Result_State_Alerting.Inc(1)
 	case m.AlertStateOK:

pkg/services/sqlstore/alert.go

@@ -173,7 +173,7 @@ func upsertAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *xor
 		} else {
 			alert.Updated = time.Now()
 			alert.Created = time.Now()
-			alert.State = m.AlertStateNoData
+			alert.State = m.AlertStatePending
 			alert.NewStateDate = time.Now()
 
 			_, err := sess.Insert(alert)
@@ -260,7 +260,7 @@ func PauseAlertRule(cmd *m.PauseAlertCommand) error {
 		if cmd.Paused {
 			newState = m.AlertStatePaused
 		} else {
-			newState = m.AlertStateNoData
+			newState = m.AlertStatePending
 		}
 		alert.State = newState
 

pkg/services/sqlstore/alert_test.go

@@ -47,7 +47,7 @@ func TestAlertingDataAccess(t *testing.T) {
 			So(err2, ShouldBeNil)
 			So(alert.Name, ShouldEqual, "Alerting title")
 			So(alert.Message, ShouldEqual, "Alerting message")
-			So(alert.State, ShouldEqual, "no_data")
+			So(alert.State, ShouldEqual, "pending")
 			So(alert.Frequency, ShouldEqual, 1)
 		})
 
@@ -77,7 +77,7 @@ func TestAlertingDataAccess(t *testing.T) {
 				So(query.Result[0].Name, ShouldEqual, "Name")
 
 				Convey("Alert state should not be updated", func() {
-					So(query.Result[0].State, ShouldEqual, "no_data")
+					So(query.Result[0].State, ShouldEqual, "pending")
 				})
 			})
 

public/app/features/alerting/alert_def.ts

@@ -87,6 +87,13 @@ function getStateDisplayModel(state) {
         stateClass: 'alert-state-paused'
       };
     }
+    case 'pending': {
+      return {
+        text: 'PENDING',
+        iconClass: "fa fa-exclamation",
+        stateClass: 'alert-state-warning'
+      };
+    }
   }
 }