Feature for repeated alerting in grafana

pkg/api/dtos/alerting.go

@@ -21,6 +21,9 @@ type AlertRule struct {
 	ExecutionError string           `json:"executionError"`
 	Url            string           `json:"url"`
 	CanEdit        bool             `json:"canEdit"`
+	NotifyOnce     bool             `json:"notifyOnce"`
+	NotifyEval     uint64           `json:"notifyEval"`
+	NotifyFreq     uint64           `json:"notifyFrequency"`
 }
 
 type AlertNotification struct {

pkg/models/alert.go

@@ -72,6 +72,9 @@ type Alert struct {
 	Silenced       bool
 	ExecutionError string
 	Frequency      int64
+	NotifyOnce     bool
+	NotifyFreq     uint64
+	NotifyEval     uint64
 
 	EvalData     *simplejson.Json
 	NewStateDate time.Time
@@ -95,6 +98,8 @@ func (this *Alert) ContainsUpdates(other *Alert) bool {
 	result := false
 	result = result || this.Name != other.Name
 	result = result || this.Message != other.Message
+	result = result || this.NotifyOnce != other.NotifyOnce
+	result = result || (!other.NotifyOnce && this.NotifyFreq != other.NotifyFreq)
 
 	if this.Settings != nil && other.Settings != nil {
 		json1, err1 := this.Settings.Encode()
@@ -159,6 +164,10 @@ type SetAlertStateCommand struct {
 	Timestamp time.Time
 }
 
+type IncAlertEvalCommand struct {
+	AlertId int64
+}
+
 //Queries
 type GetAlertsQuery struct {
 	OrgId        int64

pkg/services/alerting/extractor.go

@@ -122,6 +122,8 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json,
 			Handler:     jsonAlert.Get("handler").MustInt64(),
 			Message:     jsonAlert.Get("message").MustString(),
 			Frequency:   frequency,
+			NotifyOnce:  jsonAlert.Get("notifyOnce").MustBool(),
+			NotifyFreq:  jsonAlert.Get("notifyFrequency").MustUint64(),
 		}
 
 		for _, condition := range jsonAlert.Get("conditions").MustArray() {

pkg/services/alerting/notifiers/base.go

@@ -32,7 +32,10 @@ func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model
 
 func defaultShouldNotify(context *alerting.EvalContext) bool {
 	// Only notify on state change.
-	if context.PrevAlertState == context.Rule.State {
+	if context.PrevAlertState == context.Rule.State && context.Rule.NotifyOnce {
+		return false
+	}
+	if !context.Rule.NotifyOnce && context.Rule.NotifyEval != 0 {
 		return false
 	}
 	// Do not notify when we become OK for the first time.

pkg/services/alerting/result_handler.go

@@ -88,6 +88,7 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
 		}
 	}
 
+	bus.Dispatch(&m.IncAlertEvalCommand{AlertId: evalContext.Rule.Id})
 	handler.notifier.SendIfNeeded(evalContext)
 
 	return nil

pkg/services/alerting/rule.go

@@ -23,6 +23,9 @@ type Rule struct {
 	State               m.AlertStateType
 	Conditions          []Condition
 	Notifications       []int64
+	NotifyOnce          bool
+	NotifyFreq          uint64
+	NotifyEval          uint64
 }
 
 type ValidationError struct {
@@ -97,6 +100,9 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
 	model.Name = ruleDef.Name
 	model.Message = ruleDef.Message
 	model.Frequency = ruleDef.Frequency
+	model.NotifyOnce = ruleDef.NotifyOnce
+	model.NotifyFreq = ruleDef.NotifyFreq
+	model.NotifyEval = ruleDef.NotifyEval
 	model.State = ruleDef.State
 	model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
 	model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))

pkg/services/sqlstore/alert.go

@@ -22,6 +22,7 @@ func init() {
 	bus.AddHandler("sql", GetAlertStatesForDashboard)
 	bus.AddHandler("sql", PauseAlert)
 	bus.AddHandler("sql", PauseAllAlerts)
+	bus.AddHandler("sql", IncAlertEval)
 }
 
 func GetAlertById(query *m.GetAlertByIdQuery) error {
@@ -188,7 +189,7 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS
 			if alertToUpdate.ContainsUpdates(alert) {
 				alert.Updated = timeNow()
 				alert.State = alertToUpdate.State
-				sess.MustCols("message")
+				sess.MustCols("message", "notify_freq", "notify_once")
 				_, err := sess.Id(alert.Id).Update(alert)
 				if err != nil {
 					return err
@@ -343,3 +344,22 @@ func GetAlertStatesForDashboard(query *m.GetAlertStatesForDashboardQuery) error
 
 	return err
 }
+
+func IncAlertEval(cmd *m.IncAlertEvalCommand) error {
+	return inTransaction(func(sess *DBSession) error {
+		alert := m.Alert{}
+
+		if _, err := sess.Id(cmd.AlertId).Get(&alert); err != nil {
+			return err
+		}
+
+		alert.NotifyEval = (alert.NotifyEval + 1) % alert.NotifyFreq
+
+		sess.MustCols("notify_eval")
+		if _, err := sess.Id(cmd.AlertId).Update(alert); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}

pkg/services/sqlstore/migrations/alert_mig.go

@@ -29,6 +29,9 @@ func addAlertMigrations(mg *Migrator) {
 			{Name: "state_changes", Type: DB_Int, Nullable: false},
 			{Name: "created", Type: DB_DateTime, Nullable: false},
 			{Name: "updated", Type: DB_DateTime, Nullable: false},
+			{Name: "notify_once", Type: DB_Bool, Nullable: false},
+			{Name: "notify_freq", Type: DB_Int, Nullable: false},
+			{Name: "notify_eval", Type: DB_Int, Nullable: false},
 		},
 		Indices: []*Index{
 			{Cols: []string{"org_id", "id"}, Type: IndexType},

public/app/features/alerting/alert_tab_ctrl.ts

@@ -167,6 +167,9 @@ export class AlertTabCtrl {
     alert.noDataState = alert.noDataState || 'no_data';
     alert.executionErrorState = alert.executionErrorState || 'alerting';
     alert.frequency = alert.frequency || '60s';
+    alert.notifyFrequency = alert.notifyFrequency || 10;
+    alert.notifyOnce = alert.notifyOnce == null ? true : alert.notifyOnce;
+    alert.frequency = alert.frequency || '60s';
     alert.handler = alert.handler || 1;
     alert.notifications = alert.notifications || [];
 

public/app/features/alerting/partials/alert_tab.html

@@ -31,6 +31,9 @@
 					<input type="text" class="gf-form-input width-20" ng-model="ctrl.alert.name">
 					<span class="gf-form-label">Evaluate every</span>
 					<input class="gf-form-input max-width-5" type="text" ng-model="ctrl.alert.frequency"></input>
+					<a class="gf-form-label" ng-click="ctrl.alert.notifyOnce = !ctrl.alert.notifyOnce;">{{ ctrl.alert.notifyOnce ? 'Notify on state change' : 'Notify every' }}</a>
+					<input class="gf-form-input max-width-5" type="number" ng-model="ctrl.alert.notifyFrequency" ng-hide="ctrl.alert.notifyOnce"></input>
+					<span class="gf-form-label" ng-hide="ctrl.alert.notifyOnce">evaluations</span>
 				</div>
 			</div>