Merge pull request #13947 from bergquist/alerting_for

Introduce alert debouncing

devenv/dev-dashboards/testdata_alerts.json

@@ -1,250 +1,546 @@
 {
-  "revision": 2,
-  "title": "Alerting with TestData",
-  "tags": [
-    "grafana-test"
-  ],
-  "style": "dark",
-  "timezone": "browser",
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
   "editable": true,
-  "hideControls": false,
-  "sharedCrosshair": false,
-  "rows": [
+  "gnetId": null,
+  "graphTooltip": 0,
+  "links": [],
+  "panels": [
     {
-      "collapse": false,
+      "alert": {
+        "conditions": [
+          {
+            "evaluator": {
+              "params": [
+                60
+              ],
+              "type": "gt"
+            },
+            "query": {
+              "params": [
+                "A",
+                "5m",
+                "now"
+              ]
+            },
+            "reducer": {
+              "params": [],
+              "type": "avg"
+            },
+            "type": "query"
+          }
+        ],
+        "enabled": true,
+        "frequency": "60s",
+        "handler": 1,
+        "name": "TestData - Always OK",
+        "noDataState": "no_data",
+        "notifications": []
+      },
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "gdev-testdata",
       "editable": true,
-      "height": 255.625,
-      "panels": [
+      "error": false,
+      "fill": 1,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 3,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "connected",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "refId": "A",
+          "scenario": "random_walk",
+          "scenarioId": "csv_metric_values",
+          "stringInput": "1,20,90,30,5,0",
+          "target": ""
+        }
+      ],
+      "thresholds": [
+        {
+          "colorMode": "critical",
+          "fill": true,
+          "line": true,
+          "op": "gt",
+          "value": 60
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Always OK",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": "125",
+          "min": "0",
+          "show": true
+        },
         {
-          "alert": {
-            "conditions": [
-              {
-                "evaluator": {
-                  "params": [
-                    60
-                  ],
-                  "type": "gt"
-                },
-                "query": {
-                  "params": [
-                    "A",
-                    "5m",
-                    "now"
-                  ]
-                },
-                "reducer": {
-                  "params": [],
-                  "type": "avg"
-                },
-                "type": "query"
-              }
-            ],
-            "enabled": true,
-            "frequency": "60s",
-            "handler": 1,
-            "name": "TestData - Always OK",
-            "noDataState": "no_data",
-            "notifications": []
-          },
-          "aliasColors": {},
-          "bars": false,
-          "datasource": "gdev-testdata",
-          "editable": true,
-          "error": false,
-          "fill": 1,
-          "id": 3,
-          "isNew": true,
-          "legend": {
-            "avg": false,
-            "current": false,
-            "max": false,
-            "min": false,
-            "show": true,
-            "total": false,
-            "values": false
-          },
-          "lines": true,
-          "linewidth": 2,
-          "links": [],
-          "nullPointMode": "connected",
-          "percentage": false,
-          "pointradius": 5,
-          "points": false,
-          "renderer": "flot",
-          "seriesOverrides": [],
-          "span": 6,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "refId": "A",
-              "scenario": "random_walk",
-              "scenarioId": "csv_metric_values",
-              "stringInput": "1,20,90,30,5,0",
-              "target": ""
-            }
-          ],
-          "thresholds": [
-            {
-              "value": 60,
-              "op": "gt",
-              "fill": true,
-              "line": true,
-              "colorMode": "critical"
-            }
-          ],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Always OK",
-          "tooltip": {
-            "msResolution": false,
-            "shared": true,
-            "sort": 0,
-            "value_type": "cumulative"
-          },
-          "type": "graph",
-          "xaxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "format": "short",
-              "label": "",
-              "logBase": 1,
-              "max": "125",
-              "min": "0",
-              "show": true
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "alert": {
+        "conditions": [
+          {
+            "evaluator": {
+              "params": [
+                177
+              ],
+              "type": "gt"
             },
-            {
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ]
+            "query": {
+              "params": [
+                "A",
+                "5m",
+                "now"
+              ]
+            },
+            "reducer": {
+              "params": [],
+              "type": "avg"
+            },
+            "type": "query"
+          }
+        ],
+        "enabled": true,
+        "executionErrorState": "alerting",
+        "for": "0m",
+        "frequency": "60s",
+        "handler": 1,
+        "name": "TestData - Always Alerting",
+        "noDataState": "no_data",
+        "notifications": []
+      },
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "gdev-testdata",
+      "editable": true,
+      "error": false,
+      "fill": 1,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 4,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "connected",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "refId": "A",
+          "scenario": "random_walk",
+          "scenarioId": "csv_metric_values",
+          "stringInput": "200,445,100,150,200,220,190",
+          "target": ""
+        }
+      ],
+      "thresholds": [
+        {
+          "colorMode": "critical",
+          "fill": true,
+          "line": true,
+          "op": "gt",
+          "value": 177
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Always Alerting",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
         },
         {
-          "alert": {
-            "conditions": [
-              {
-                "evaluator": {
-                  "params": [
-                    177
-                  ],
-                  "type": "gt"
-                },
-                "query": {
-                  "params": [
-                    "A",
-                    "5m",
-                    "now"
-                  ]
-                },
-                "reducer": {
-                  "params": [],
-                  "type": "avg"
-                },
-                "type": "query"
-              }
-            ],
-            "enabled": true,
-            "frequency": "60s",
-            "handler": 1,
-            "name": "TestData - Always Alerting",
-            "noDataState": "no_data",
-            "notifications": []
-          },
-          "aliasColors": {},
-          "bars": false,
-          "datasource": "gdev-testdata",
-          "editable": true,
-          "error": false,
-          "fill": 1,
-          "id": 4,
-          "isNew": true,
-          "legend": {
-            "avg": false,
-            "current": false,
-            "max": false,
-            "min": false,
-            "show": true,
-            "total": false,
-            "values": false
-          },
-          "lines": true,
-          "linewidth": 2,
-          "links": [],
-          "nullPointMode": "connected",
-          "percentage": false,
-          "pointradius": 5,
-          "points": false,
-          "renderer": "flot",
-          "seriesOverrides": [],
-          "span": 6,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "refId": "A",
-              "scenario": "random_walk",
-              "scenarioId": "csv_metric_values",
-              "stringInput": "200,445,100,150,200,220,190",
-              "target": ""
-            }
-          ],
-          "thresholds": [
-            {
-              "colorMode": "critical",
-              "fill": true,
-              "line": true,
-              "op": "gt",
-              "value": 177
-            }
-          ],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Always Alerting",
-          "tooltip": {
-            "msResolution": false,
-            "shared": true,
-            "sort": 0,
-            "value_type": "cumulative"
-          },
-          "type": "graph",
-          "xaxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "format": "short",
-              "label": "",
-              "logBase": 1,
-              "max": null,
-              "min": "0",
-              "show": true
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "alert": {
+        "conditions": [
+          {
+            "evaluator": {
+              "params": [
+                1
+              ],
+              "type": "gt"
             },
-            {
-              "format": "short",
-              "label": "",
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ]
+            "operator": {
+              "type": "and"
+            },
+            "query": {
+              "params": [
+                "A",
+                "15m",
+                "now"
+              ]
+            },
+            "reducer": {
+              "params": [],
+              "type": "avg"
+            },
+            "type": "query"
+          }
+        ],
+        "executionErrorState": "alerting",
+        "for": "5m",
+        "frequency": "1m",
+        "handler": 1,
+        "name": "TestData - No data",
+        "noDataState": "no_data",
+        "notifications": []
+      },
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "gdev-testdata",
+      "editable": true,
+      "error": false,
+      "fill": 1,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 7
+      },
+      "id": 5,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "connected",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "refId": "A",
+          "scenario": "random_walk",
+          "scenarioId": "no_data_points",
+          "stringInput": "",
+          "target": ""
         }
       ],
-      "title": "New row"
+      "thresholds": [
+        {
+          "colorMode": "critical",
+          "fill": true,
+          "line": true,
+          "op": "gt",
+          "value": 1
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "No data",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "alert": {
+        "conditions": [
+          {
+            "evaluator": {
+              "params": [
+                177
+              ],
+              "type": "gt"
+            },
+            "operator": {
+              "type": "and"
+            },
+            "query": {
+              "params": [
+                "A",
+                "15m",
+                "now"
+              ]
+            },
+            "reducer": {
+              "params": [],
+              "type": "avg"
+            },
+            "type": "query"
+          }
+        ],
+        "executionErrorState": "alerting",
+        "for": "1m",
+        "frequency": "1m",
+        "handler": 1,
+        "name": "TestData - Always Alerting with For",
+        "noDataState": "no_data",
+        "notifications": []
+      },
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "gdev-testdata",
+      "editable": true,
+      "error": false,
+      "fill": 1,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 7
+      },
+      "id": 6,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "connected",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "refId": "A",
+          "scenario": "random_walk",
+          "scenarioId": "csv_metric_values",
+          "stringInput": "200,445,100,150,200,220,190",
+          "target": ""
+        }
+      ],
+      "thresholds": [
+        {
+          "colorMode": "critical",
+          "fill": true,
+          "line": true,
+          "op": "gt",
+          "value": 177
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Always Alerting with For",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
     }
   ],
+  "revision": 2,
+  "schemaVersion": 16,
+  "style": "dark",
+  "tags": [
+    "grafana-test"
+  ],
+  "templating": {
+    "list": []
+  },
   "time": {
     "from": "now-6h",
     "to": "now"
@@ -274,14 +570,8 @@
       "30d"
     ]
   },
-  "templating": {
-    "list": []
-  },
-  "annotations": {
-    "list": []
-  },
-  "schemaVersion": 13,
-  "version": 4,
-  "links": [],
-  "gnetId": null
-}
+  "timezone": "browser",
+  "title": "Alerting with TestData",
+  "uid": "7MeksYbmk",
+  "version": 1
+}

devenv/docker/ha_test/docker-compose.yaml

@@ -9,7 +9,7 @@ services:
       - /var/run/docker.sock:/tmp/docker.sock:ro
 
   db:
-    image: mysql
+    image: mysql:5.6
     environment:
       MYSQL_ROOT_PASSWORD: rootpass
       MYSQL_DATABASE: grafana

devenv/docker/ha_test/grafana/provisioning/alerts.jsonnet

@@ -39,6 +39,7 @@ local alertDashboardTemplate = {
         "executionErrorState": "alerting",
         "frequency": "10s",
         "handler": 1,
+        "for": "1m",
         "name": "bulk alerting",
         "noDataState": "no_data",
         "notifications": [

docs/sources/alerting/rules.md

@@ -39,7 +39,7 @@ Currently alerting supports a limited form of high availability. Since v4.2.0 of
 
 ## Rule Config
 
-{{< imgbox max-width="40%" img="/img/docs/v4/alerting_conditions.png" caption="Alerting Conditions" >}}
+
 
 Currently only the graph panel supports alert rules but this will be added to the **Singlestat** and **Table**
 panels as well in a future release.
@@ -48,6 +48,16 @@ panels as well in a future release.
 
 Here you can specify the name of the alert rule and how often the scheduler should evaluate the alert rule.
 
+### For
+
+> This setting is available in Grafana 5.4 and above.
+
+If an alert rule has a configured `For` and the query violates the configured threshold it will first go from `OK` to `Pending`. Going from `OK` to `Pending` Grafana will not send any notifications. Once the alert rule has been firing for more than `For` duration, it will change to `Alerting` and send alert notifications. 
+
+Typically, it's always a good idea to use this setting since its often worse to get false positive than wait a few minutes before the alert notification triggers.
+
+{{< imgbox max-width="40%" img="/img/docs/v4/alerting_conditions.png" caption="Alerting Conditions" >}}
+
 ### Conditions
 
 Currently the only condition type that exists is a `Query` condition that allows you to
@@ -57,11 +67,11 @@ specify a query letter, time range and an aggregation function.
 ### Query condition example
 
 ```sql
-avg() OF query(A, 5m, now) IS BELOW 14
+avg() OF query(A, 15m, now) IS BELOW 14
 ```
 
 - `avg()` Controls how the values for **each** series should be reduced to a value that can be compared against the threshold. Click on the function to change it to another aggregation function.
-- `query(A, 5m, now)`  The letter defines what query to execute from the **Metrics** tab. The second two parameters define the time range, `5m, now` means 5 minutes ago to now. You can also do `10m, now-2m` to define a time range that will be 10 minutes ago to 2 minutes ago. This is useful if you want to ignore the last 2 minutes of data.
+- `query(A, 15m, now)`  The letter defines what query to execute from the **Metrics** tab. The second two parameters define the time range, `15m, now` means 5 minutes ago to now. You can also do `10m, now-2m` to define a time range that will be 10 minutes ago to 2 minutes ago. This is useful if you want to ignore the last 2 minutes of data.
 - `IS BELOW 14`  Defines the type of threshold and the threshold value.  You can click on `IS BELOW` to change the type of threshold.
 
 The query used in an alert rule cannot contain any template variables. Currently we only support `AND` and `OR` operators between conditions and they are executed serially.

pkg/api/alerting.go

@@ -295,7 +295,7 @@ func PauseAlert(c *m.ReqContext, dto dtos.PauseAlertCommand) Response {
 		return Error(500, "", err)
 	}
 
-	var response m.AlertStateType = m.AlertStatePending
+	var response m.AlertStateType = m.AlertStateUnknown
 	pausedState := "un-paused"
 	if cmd.Paused {
 		response = m.AlertStatePaused

pkg/models/alert.go

@@ -19,6 +19,7 @@ const (
 	AlertStateAlerting AlertStateType = "alerting"
 	AlertStateOK       AlertStateType = "ok"
 	AlertStatePending  AlertStateType = "pending"
+	AlertStateUnknown  AlertStateType = "unknown"
 )
 
 const (
@@ -39,7 +40,12 @@ var (
 )
 
 func (s AlertStateType) IsValid() bool {
-	return s == AlertStateOK || s == AlertStateNoData || s == AlertStatePaused || s == AlertStatePending
+	return s == AlertStateOK ||
+		s == AlertStateNoData ||
+		s == AlertStatePaused ||
+		s == AlertStatePending ||
+		s == AlertStateAlerting ||
+		s == AlertStateUnknown
 }
 
 func (s NoDataOption) IsValid() bool {
@@ -66,12 +72,13 @@ type Alert struct {
 	PanelId        int64
 	Name           string
 	Message        string
-	Severity       string
+	Severity       string //Unused
 	State          AlertStateType
-	Handler        int64
+	Handler        int64 //Unused
 	Silenced       bool
 	ExecutionError string
 	Frequency      int64
+	For            time.Duration
 
 	EvalData     *simplejson.Json
 	NewStateDate time.Time

pkg/services/alerting/eval_context.go

@@ -68,8 +68,13 @@ func (c *EvalContext) GetStateModel() *StateDescription {
 			Color: "#D63232",
 			Text:  "Alerting",
 		}
+	case m.AlertStateUnknown:
+		return &StateDescription{
+			Color: "#888888",
+			Text:  "Unknown",
+		}
 	default:
-		panic("Unknown rule state " + c.Rule.State)
+		panic("Unknown rule state for alert " + c.Rule.State)
 	}
 }
 
@@ -113,7 +118,26 @@ func (c *EvalContext) GetRuleUrl() (string, error) {
 	return fmt.Sprintf(urlFormat, m.GetFullDashboardUrl(ref.Uid, ref.Slug), c.Rule.PanelId, c.Rule.OrgId), nil
 }
 
+// GetNewState returns the new state from the alert rule evaluation
 func (c *EvalContext) GetNewState() m.AlertStateType {
+	ns := getNewStateInternal(c)
+	if ns != m.AlertStateAlerting || c.Rule.For == 0 {
+		return ns
+	}
+
+	since := time.Now().Sub(c.Rule.LastStateChange)
+	if c.PrevAlertState == m.AlertStatePending && since > c.Rule.For {
+		return m.AlertStateAlerting
+	}
+
+	if c.PrevAlertState == m.AlertStateAlerting {
+		return m.AlertStateAlerting
+	}
+
+	return m.AlertStatePending
+}
+
+func getNewStateInternal(c *EvalContext) m.AlertStateType {
 	if c.Error != nil {
 		c.log.Error("Alert Rule Result Error",
 			"ruleId", c.Rule.Id,
@@ -125,11 +149,13 @@ func (c *EvalContext) GetNewState() m.AlertStateType {
 			return c.PrevAlertState
 		}
 		return c.Rule.ExecutionErrorState.ToAlertState()
+	}
 
-	} else if c.Firing {
+	if c.Firing {
 		return m.AlertStateAlerting
+	}
 
-	} else if c.NoDataFound {
+	if c.NoDataFound {
 		c.log.Info("Alert Rule returned no data",
 			"ruleId", c.Rule.Id,
 			"name", c.Rule.Name,

pkg/services/alerting/eval_context_test.go

@@ -2,11 +2,11 @@ package alerting
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"testing"
+	"time"
 
 	"github.com/grafana/grafana/pkg/models"
-	. "github.com/smartystreets/goconvey/convey"
 )
 
 func TestStateIsUpdatedWhenNeeded(t *testing.T) {
@@ -31,71 +31,177 @@ func TestStateIsUpdatedWhenNeeded(t *testing.T) {
 	})
 }
 
-func TestAlertingEvalContext(t *testing.T) {
-	Convey("Should compute and replace properly new rule state", t, func() {
+func TestGetStateFromEvalContext(t *testing.T) {
+	tcs := []struct {
+		name     string
+		expected models.AlertStateType
+		applyFn  func(ec *EvalContext)
+		focus    bool
+	}{
+		{
+			name:     "ok -> alerting",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.Firing = true
+				ec.PrevAlertState = models.AlertStateOK
+			},
+		},
+		{
+			name:     "ok -> error(alerting)",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Error = errors.New("test error")
+				ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
+			},
+		},
+		{
+			name:     "ok -> pending. since its been firing for less than FOR",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Firing = true
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
+				ec.Rule.For = time.Minute * 5
+			},
+		},
+		{
+			name:     "ok -> pending. since it has to be pending longer than FOR and prev state is ok",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Firing = true
+				ec.Rule.LastStateChange = time.Now().Add(-(time.Hour * 5))
+				ec.Rule.For = time.Minute * 2
+			},
+		},
+		{
+			name:     "pending -> alerting. since its been firing for more than FOR and prev state is pending",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Firing = true
+				ec.Rule.LastStateChange = time.Now().Add(-(time.Hour * 5))
+				ec.Rule.For = time.Minute * 2
+			},
+		},
+		{
+			name:     "alerting -> alerting. should not update regardless of FOR",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateAlerting
+				ec.Firing = true
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
+				ec.Rule.For = time.Minute * 2
+			},
+		},
+		{
+			name:     "ok -> ok. should not update regardless of FOR",
+			expected: models.AlertStateOK,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
+				ec.Rule.For = time.Minute * 2
+			},
+		},
+		{
+			name:     "ok -> error(keep_last)",
+			expected: models.AlertStateOK,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Error = errors.New("test error")
+				ec.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
+			},
+		},
+		{
+			name:     "pending -> error(keep_last)",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Error = errors.New("test error")
+				ec.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
+			},
+		},
+		{
+			name:     "ok -> no_data(alerting)",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Rule.NoDataState = models.NoDataSetAlerting
+				ec.NoDataFound = true
+			},
+		},
+		{
+			name:     "ok -> no_data(keep_last)",
+			expected: models.AlertStateOK,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Rule.NoDataState = models.NoDataKeepState
+				ec.NoDataFound = true
+			},
+		},
+		{
+			name:     "pending -> no_data(keep_last)",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Rule.NoDataState = models.NoDataKeepState
+				ec.NoDataFound = true
+			},
+		},
+		{
+			name:     "pending -> no_data(alerting) with for duration have not passed",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Rule.NoDataState = models.NoDataSetAlerting
+				ec.NoDataFound = true
+				ec.Rule.For = time.Minute * 5
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
+			},
+		},
+		{
+			name:     "pending -> no_data(alerting) should set alerting since time passed FOR",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Rule.NoDataState = models.NoDataSetAlerting
+				ec.NoDataFound = true
+				ec.Rule.For = time.Minute * 2
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
+			},
+		},
+		{
+			name:     "pending -> error(alerting) with for duration have not passed ",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
+				ec.Error = errors.New("test error")
+				ec.Rule.For = time.Minute * 5
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
+			},
+		},
+		{
+			name:     "pending -> error(alerting) should set alerting since time passed FOR",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
+				ec.Error = errors.New("test error")
+				ec.Rule.For = time.Minute * 2
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
+			},
+		},
+	}
+
+	for _, tc := range tcs {
 		ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
-		dummieError := fmt.Errorf("dummie error")
 
-		Convey("ok -> alerting", func() {
-			ctx.PrevAlertState = models.AlertStateOK
-			ctx.Firing = true
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
-		})
-
-		Convey("ok -> error(alerting)", func() {
-			ctx.PrevAlertState = models.AlertStateOK
-			ctx.Error = dummieError
-			ctx.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
-		})
-
-		Convey("ok -> error(keep_last)", func() {
-			ctx.PrevAlertState = models.AlertStateOK
-			ctx.Error = dummieError
-			ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
-		})
-
-		Convey("pending -> error(keep_last)", func() {
-			ctx.PrevAlertState = models.AlertStatePending
-			ctx.Error = dummieError
-			ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
-		})
-
-		Convey("ok -> no_data(alerting)", func() {
-			ctx.PrevAlertState = models.AlertStateOK
-			ctx.Rule.NoDataState = models.NoDataSetAlerting
-			ctx.NoDataFound = true
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
-		})
-
-		Convey("ok -> no_data(keep_last)", func() {
-			ctx.PrevAlertState = models.AlertStateOK
-			ctx.Rule.NoDataState = models.NoDataKeepState
-			ctx.NoDataFound = true
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
-		})
-
-		Convey("pending -> no_data(keep_last)", func() {
-			ctx.PrevAlertState = models.AlertStatePending
-			ctx.Rule.NoDataState = models.NoDataKeepState
-			ctx.NoDataFound = true
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
-		})
-	})
+		tc.applyFn(ctx)
+		have := ctx.GetNewState()
+		if have != tc.expected {
+			t.Errorf("failed: %s \n expected '%s' have '%s'\n", tc.name, tc.expected, string(have))
+		}
+	}
 }

pkg/services/alerting/extractor.go

@@ -2,8 +2,8 @@ package alerting
 
 import (
 	"errors"
-
 	"fmt"
+	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/components/simplejson"
@@ -115,6 +115,15 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json,
 			return nil, ValidationError{Reason: "Could not parse frequency"}
 		}
 
+		rawFor := jsonAlert.Get("for").MustString()
+		var forValue time.Duration
+		if rawFor != "" {
+			forValue, err = time.ParseDuration(rawFor)
+			if err != nil {
+				return nil, ValidationError{Reason: "Could not parse for"}
+			}
+		}
+
 		alert := &m.Alert{
 			DashboardId: e.Dash.Id,
 			OrgId:       e.OrgID,
@@ -124,6 +133,7 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json,
 			Handler:     jsonAlert.Get("handler").MustInt64(),
 			Message:     jsonAlert.Get("message").MustString(),
 			Frequency:   frequency,
+			For:         forValue,
 		}
 
 		for _, condition := range jsonAlert.Get("conditions").MustArray() {

pkg/services/alerting/extractor_test.go

@@ -3,6 +3,7 @@ package alerting
 import (
 	"io/ioutil"
 	"testing"
+	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/components/simplejson"
@@ -46,7 +47,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 			return nil
 		})
 
-		json, err := ioutil.ReadFile("./test-data/graphite-alert.json")
+		json, err := ioutil.ReadFile("./testdata/graphite-alert.json")
 		So(err, ShouldBeNil)
 
 		Convey("Extractor should not modify the original json", func() {
@@ -118,6 +119,11 @@ func TestAlertRuleExtraction(t *testing.T) {
 					So(alerts[1].PanelId, ShouldEqual, 4)
 				})
 
+				Convey("should extract for param", func() {
+					So(alerts[0].For, ShouldEqual, time.Minute*2)
+					So(alerts[1].For, ShouldEqual, time.Duration(0))
+				})
+
 				Convey("should extract name and desc", func() {
 					So(alerts[0].Name, ShouldEqual, "name1")
 					So(alerts[0].Message, ShouldEqual, "desc1")
@@ -140,7 +146,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Panels missing id should return error", func() {
-			panelWithoutId, err := ioutil.ReadFile("./test-data/panels-missing-id.json")
+			panelWithoutId, err := ioutil.ReadFile("./testdata/panels-missing-id.json")
 			So(err, ShouldBeNil)
 
 			dashJson, err := simplejson.NewJson(panelWithoutId)
@@ -156,7 +162,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Panel with id set to zero should return error", func() {
-			panelWithIdZero, err := ioutil.ReadFile("./test-data/panel-with-id-0.json")
+			panelWithIdZero, err := ioutil.ReadFile("./testdata/panel-with-id-0.json")
 			So(err, ShouldBeNil)
 
 			dashJson, err := simplejson.NewJson(panelWithIdZero)
@@ -172,7 +178,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Parse alerts from dashboard without rows", func() {
-			json, err := ioutil.ReadFile("./test-data/v5-dashboard.json")
+			json, err := ioutil.ReadFile("./testdata/v5-dashboard.json")
 			So(err, ShouldBeNil)
 
 			dashJson, err := simplejson.NewJson(json)
@@ -192,7 +198,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Parse and validate dashboard containing influxdb alert", func() {
-			json, err := ioutil.ReadFile("./test-data/influxdb-alert.json")
+			json, err := ioutil.ReadFile("./testdata/influxdb-alert.json")
 			So(err, ShouldBeNil)
 
 			dashJson, err := simplejson.NewJson(json)
@@ -221,7 +227,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Should be able to extract collapsed panels", func() {
-			json, err := ioutil.ReadFile("./test-data/collapsed-panels.json")
+			json, err := ioutil.ReadFile("./testdata/collapsed-panels.json")
 			So(err, ShouldBeNil)
 
 			dashJson, err := simplejson.NewJson(json)
@@ -242,7 +248,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Parse and validate dashboard without id and containing an alert", func() {
-			json, err := ioutil.ReadFile("./test-data/dash-without-id.json")
+			json, err := ioutil.ReadFile("./testdata/dash-without-id.json")
 			So(err, ShouldBeNil)
 
 			dashJSON, err := simplejson.NewJson(json)

pkg/services/alerting/notifiers/alertmanager_test.go

@@ -1,13 +1,60 @@
 package notifiers
 
 import (
+	"context"
 	"testing"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/log"
 	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting"
 	. "github.com/smartystreets/goconvey/convey"
 )
 
+func TestWhenAlertManagerShouldNotify(t *testing.T) {
+	tcs := []struct {
+		prevState m.AlertStateType
+		newState  m.AlertStateType
+
+		expect bool
+	}{
+		{
+			prevState: m.AlertStatePending,
+			newState:  m.AlertStateOK,
+			expect:    false,
+		},
+		{
+			prevState: m.AlertStateAlerting,
+			newState:  m.AlertStateOK,
+			expect:    true,
+		},
+		{
+			prevState: m.AlertStateOK,
+			newState:  m.AlertStatePending,
+			expect:    false,
+		},
+		{
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStatePending,
+			expect:    false,
+		},
+	}
+
+	for _, tc := range tcs {
+		am := &AlertmanagerNotifier{log: log.New("test.logger")}
+		evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{
+			State: tc.prevState,
+		})
+
+		evalContext.Rule.State = tc.newState
+
+		res := am.ShouldNotify(context.TODO(), evalContext, &m.AlertNotificationState{})
+		if res != tc.expect {
+			t.Errorf("got %v expected %v", res, tc.expect)
+		}
+	}
+}
+
 func TestAlertmanagerNotifier(t *testing.T) {
 	Convey("Alertmanager notifier tests", t, func() {
 

pkg/services/alerting/notifiers/base.go

@@ -67,6 +67,16 @@ func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalC
 	}
 
 	// Do not notify when we become OK for the first time.
+	if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStateOK {
+		return false
+	}
+
+	// Do not notify when we become OK for the first time.
+	if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStatePending {
+		return false
+	}
+
+	// Do not notify when we become OK from pending
 	if context.PrevAlertState == models.AlertStatePending && context.Rule.State == models.AlertStateOK {
 		return false
 	}

pkg/services/alerting/notifiers/base_test.go

@@ -29,7 +29,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStatePending,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -38,7 +37,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateAlerting,
 			prevState:    m.AlertStateOK,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: true,
 		},
@@ -47,7 +45,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStatePending,
 			prevState:    m.AlertStateOK,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -56,7 +53,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStateOK,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -65,7 +61,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStateOK,
 			sendReminder: true,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -74,7 +69,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStateAlerting,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: true,
 		},
@@ -94,7 +88,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			prevState:    m.AlertStateAlerting,
 			frequency:    time.Minute * 10,
 			sendReminder: true,
-			state:        &m.AlertNotificationState{},
 
 			expect: true,
 		},
@@ -132,6 +125,27 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			prevState: m.AlertStateOK,
 			state:     &m.AlertNotificationState{State: m.AlertNotificationStatePending, UpdatedAt: tnow.Add(-2 * time.Minute).Unix()},
 
+			expect: true,
+		},
+		{
+			name:      "unknown -> ok",
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStateOK,
+
+			expect: false,
+		},
+		{
+			name:      "unknown -> pending",
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStatePending,
+
+			expect: false,
+		},
+		{
+			name:      "unknown -> alerting",
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStateAlerting,
+
 			expect: true,
 		},
 	}
@@ -141,6 +155,10 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			State: tc.prevState,
 		})
 
+		if tc.state == nil {
+			tc.state = &m.AlertNotificationState{}
+		}
+
 		evalContext.Rule.State = tc.newState
 		nb := &NotifierBase{SendReminder: tc.sendReminder, Frequency: tc.frequency}
 

pkg/services/alerting/result_handler.go

@@ -73,6 +73,9 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
 			// when two servers are raising. This makes sure that the server
 			// with the last state change always sends a notification.
 			evalContext.Rule.StateChanges = cmd.Result.StateChanges
+
+			// Update the last state change of the alert rule in memory
+			evalContext.Rule.LastStateChange = time.Now()
 		}
 
 		// save annotation

pkg/services/alerting/rule.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"regexp"
 	"strconv"
+	"time"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 
@@ -18,6 +19,8 @@ type Rule struct {
 	Frequency           int64
 	Name                string
 	Message             string
+	LastStateChange     time.Time
+	For                 time.Duration
 	NoDataState         m.NoDataOption
 	ExecutionErrorState m.ExecutionErrorOption
 	State               m.AlertStateType
@@ -100,6 +103,8 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
 	model.Message = ruleDef.Message
 	model.Frequency = ruleDef.Frequency
 	model.State = ruleDef.State
+	model.LastStateChange = ruleDef.NewStateDate
+	model.For = ruleDef.For
 	model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
 	model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
 	model.StateChanges = ruleDef.StateChanges

pkg/services/alerting/test-data/graphite-alert.json → pkg/services/alerting/testdata/graphite-alert.json

@@ -23,6 +23,7 @@
           "message": "desc1",
           "handler": 1,
           "frequency": "60s",
+          "for": "2m",
           "conditions": [
           {
             "type": "query",

pkg/services/sqlstore/alert.go

@@ -193,7 +193,8 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS
 			if alertToUpdate.ContainsUpdates(alert) {
 				alert.Updated = timeNow()
 				alert.State = alertToUpdate.State
-				sess.MustCols("message")
+				sess.MustCols("message", "for")
+
 				_, err := sess.ID(alert.Id).Update(alert)
 				if err != nil {
 					return err
@@ -204,7 +205,7 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS
 		} else {
 			alert.Updated = timeNow()
 			alert.Created = timeNow()
-			alert.State = m.AlertStatePending
+			alert.State = m.AlertStateUnknown
 			alert.NewStateDate = timeNow()
 
 			_, err := sess.Insert(alert)
@@ -299,7 +300,7 @@ func PauseAlert(cmd *m.PauseAlertCommand) error {
 			params = append(params, string(m.AlertStatePaused))
 			params = append(params, timeNow())
 		} else {
-			params = append(params, string(m.AlertStatePending))
+			params = append(params, string(m.AlertStateUnknown))
 			params = append(params, timeNow())
 		}
 
@@ -323,7 +324,7 @@ func PauseAllAlerts(cmd *m.PauseAllAlertCommand) error {
 		if cmd.Paused {
 			newState = string(m.AlertStatePaused)
 		} else {
-			newState = string(m.AlertStatePending)
+			newState = string(m.AlertStateUnknown)
 		}
 
 		res, err := sess.Exec(`UPDATE alert SET state = ?, new_state_date = ?`, newState, timeNow())

pkg/services/sqlstore/alert_test.go

@@ -109,7 +109,7 @@ func TestAlertingDataAccess(t *testing.T) {
 			So(alert.DashboardId, ShouldEqual, testDash.Id)
 			So(alert.PanelId, ShouldEqual, 1)
 			So(alert.Name, ShouldEqual, "Alerting title")
-			So(alert.State, ShouldEqual, "pending")
+			So(alert.State, ShouldEqual, m.AlertStateUnknown)
 			So(alert.NewStateDate, ShouldNotBeNil)
 			So(alert.EvalData, ShouldNotBeNil)
 			So(alert.EvalData.Get("test").MustString(), ShouldEqual, "test")
@@ -154,7 +154,7 @@ func TestAlertingDataAccess(t *testing.T) {
 				So(query.Result[0].Name, ShouldEqual, "Name")
 
 				Convey("Alert state should not be updated", func() {
-					So(query.Result[0].State, ShouldEqual, "pending")
+					So(query.Result[0].State, ShouldEqual, m.AlertStateUnknown)
 				})
 			})
 

pkg/services/sqlstore/migrations/alert_mig.go

@@ -133,4 +133,8 @@ func addAlertMigrations(mg *Migrator) {
 	mg.AddMigration("create alert_notification_state table v1", NewAddTableMigration(alert_notification_state))
 	mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id",
 		NewAddIndexMigration(alert_notification_state, alert_notification_state.Indices[0]))
+
+	mg.AddMigration("Add for to alert table", NewAddColumnMigration(alertV1, &Column{
+		Name: "for", Type: DB_BigInt, Nullable: true,
+	}))
 }

public/app/core/utils/colors.ts

@@ -7,6 +7,7 @@ export const DEFAULT_ANNOTATION_COLOR = 'rgba(0, 211, 255, 1)';
 export const OK_COLOR = 'rgba(11, 237, 50, 1)';
 export const ALERTING_COLOR = 'rgba(237, 46, 24, 1)';
 export const NO_DATA_COLOR = 'rgba(150, 150, 150, 1)';
+export const PENDING_COLOR = 'rgba(247, 149, 32, 1)';
 export const REGION_FILL_ALPHA = 0.09;
 
 const colors = [

public/app/features/alerting/AlertRuleList.tsx

@@ -29,6 +29,7 @@ export class AlertRuleList extends PureComponent<Props, any> {
     { text: 'Alerting', value: 'alerting' },
     { text: 'No Data', value: 'no_data' },
     { text: 'Paused', value: 'paused' },
+    { text: 'Pending', value: 'pending' },
   ];
 
   componentDidMount() {

public/app/features/alerting/AlertTabCtrl.ts

@@ -169,6 +169,7 @@ export class AlertTabCtrl {
     alert.frequency = alert.frequency || '1m';
     alert.handler = alert.handler || 1;
     alert.notifications = alert.notifications || [];
+    alert.for = alert.for || '0m';
 
     const defaultName = this.panel.title + ' alert';
     alert.name = alert.name || defaultName;
@@ -217,7 +218,7 @@ export class AlertTabCtrl {
   buildDefaultCondition() {
     return {
       type: 'query',
-      query: { params: ['A', '15m', 'now'] },
+      query: { params: ['A', '5m', 'now'] },
       reducer: { type: 'avg', params: [] },
       evaluator: { type: 'gt', params: [null] },
       operator: { type: 'and' },
@@ -354,6 +355,7 @@ export class AlertTabCtrl {
   enable() {
     this.panel.alert = {};
     this.initModel();
+    this.panel.alert.for = '5m'; //default value for new alerts. for existing alerts we use 0m to avoid breaking changes
   }
 
   evaluatorParamsChanged() {

public/app/features/alerting/__snapshots__/AlertRuleList.test.tsx.snap

@@ -81,6 +81,12 @@ exports[`Render should render alert rules 1`] = `
             >
               Paused
             </option>
+            <option
+              key="pending"
+              value="pending"
+            >
+              Pending
+            </option>
           </select>
         </div>
       </div>
@@ -230,6 +236,12 @@ exports[`Render should render component 1`] = `
             >
               Paused
             </option>
+            <option
+              key="pending"
+              value="pending"
+            >
+              Pending
+            </option>
           </select>
         </div>
       </div>

public/app/features/alerting/partials/alert_tab.html

@@ -1,147 +1,159 @@
 <div class="edit-tab-with-sidemenu" ng-if="ctrl.alert">
-	<aside class="edit-sidemenu-aside">
-		<ul class="edit-sidemenu">
-			<li ng-class="{active: ctrl.subTabIndex === 0}">
-				<a ng-click="ctrl.changeTabIndex(0)">Alert Config</a>
-			</li>
-			<li ng-class="{active: ctrl.subTabIndex === 1}">
-				<a ng-click="ctrl.changeTabIndex(1)">
-					Notifications <span class="muted">({{ctrl.alertNotifications.length}})</span>
-				</a>
-			</li>
-			<li ng-class="{active: ctrl.subTabIndex === 2}">
-				<a ng-click="ctrl.changeTabIndex(2)">State history</a>
-			</li>
+  <aside class="edit-sidemenu-aside">
+    <ul class="edit-sidemenu">
+      <li ng-class="{active: ctrl.subTabIndex === 0}">
+        <a ng-click="ctrl.changeTabIndex(0)">Alert Config</a>
+      </li>
+      <li ng-class="{active: ctrl.subTabIndex === 1}">
+        <a ng-click="ctrl.changeTabIndex(1)">
+          Notifications <span class="muted">({{ctrl.alertNotifications.length}})</span>
+        </a>
+      </li>
+      <li ng-class="{active: ctrl.subTabIndex === 2}">
+        <a ng-click="ctrl.changeTabIndex(2)">State history</a>
+      </li>
       <li>
-				<a ng-click="ctrl.delete()">Delete</a>
-			</li>
-		</ul>
-	</aside>
+        <a ng-click="ctrl.delete()">Delete</a>
+      </li>
+    </ul>
+  </aside>
 
-	<div class="edit-tab-content">
-		<div ng-if="ctrl.subTabIndex === 0">
-			<div class="alert alert-error m-b-2" ng-show="ctrl.error">
-				<i class="fa fa-warning"></i> {{ctrl.error}}
-			</div>
+  <div class="edit-tab-content">
+    <div ng-if="ctrl.subTabIndex === 0">
+      <div class="alert alert-error m-b-2" ng-show="ctrl.error">
+        <i class="fa fa-warning"></i> {{ctrl.error}}
+      </div>
 
-			<div class="gf-form-group">
-				<h5 class="section-heading">Alert Config</h5>
-				<div class="gf-form">
-					<span class="gf-form-label width-6">Name</span>
-					<input type="text" class="gf-form-input width-20" ng-model="ctrl.alert.name">
-					<span class="gf-form-label">Evaluate every</span>
-					<input class="gf-form-input max-width-5" type="text" ng-model="ctrl.alert.frequency"></input>
-				</div>
-			</div>
+      <div class="gf-form-group">
+        <h5 class="section-heading">Alert Config</h5>
+        <div class="gf-form">
+          <span class="gf-form-label width-6">Name</span>
+          <input type="text" class="gf-form-input width-20" ng-model="ctrl.alert.name">
+        </div>
+        <div class="gf-form-inline">
+          <div class="gf-form">
+            <span class="gf-form-label width-9">Evaluate every</span>
+            <input class="gf-form-input max-width-6" type="text" ng-model="ctrl.alert.frequency">
+          </div>
+          <div class="gf-form max-width-11">
+            <label class="gf-form-label width-5">For</label>
+            <input type="text" class="gf-form-input max-width-6" ng-model="ctrl.alert.for" spellcheck='false' placeholder="5m">
+            <info-popover mode="right-absolute">
+                If an alert rule has a configured For and the query violates the configured threshold it will first go from OK to Pending. 
+                Going from OK to Pending Grafana will not send any notifications. Once the alert rule has been firing for more than For duration, it will change to Alerting and send alert notifications. 
+            </info-popover>
+          </div>
+        </div>
+      </div>
 
-			<div class="gf-form-group">
-				<h5 class="section-heading">Conditions</h5>
-				<div class="gf-form-inline" ng-repeat="conditionModel in ctrl.conditionModels">
-					<div class="gf-form">
-						<metric-segment-model css-class="query-keyword width-5" ng-if="$index" property="conditionModel.operator.type" options="ctrl.evalOperators" custom="false"></metric-segment-model>
-						<span class="gf-form-label query-keyword width-5" ng-if="$index===0">WHEN</span>
-					</div>
-          			<div class="gf-form">
-						<query-part-editor class="gf-form-label query-part width-9" part="conditionModel.reducerPart" handle-event="ctrl.handleReducerPartEvent(conditionModel, $event)">
-						</query-part-editor>
-            			<span class="gf-form-label query-keyword">OF</span>
-					</div>
-					<div class="gf-form">
-						<query-part-editor class="gf-form-label query-part" part="conditionModel.queryPart" handle-event="ctrl.handleQueryPartEvent(conditionModel, $event)">
-						</query-part-editor>
-					</div>
-					<div class="gf-form">
-						<metric-segment-model property="conditionModel.evaluator.type" options="ctrl.evalFunctions" custom="false" css-class="query-keyword" on-change="ctrl.evaluatorTypeChanged(conditionModel.evaluator)"></metric-segment-model>
-						<input class="gf-form-input max-width-9" type="number" step="any" ng-hide="conditionModel.evaluator.params.length === 0" ng-model="conditionModel.evaluator.params[0]" ng-change="ctrl.evaluatorParamsChanged()"></input>
-            			<label class="gf-form-label query-keyword" ng-show="conditionModel.evaluator.params.length === 2">TO</label>
-            			<input class="gf-form-input max-width-9" type="number" step="any" ng-if="conditionModel.evaluator.params.length === 2" ng-model="conditionModel.evaluator.params[1]" ng-change="ctrl.evaluatorParamsChanged()"></input>
-					</div>
-					<div class="gf-form">
-						<label class="gf-form-label">
-							<a class="pointer" tabindex="1" ng-click="ctrl.removeCondition($index)">
-								<i class="fa fa-trash"></i>
-							</a>
-						</label>
-					</div>
-				</div>
+      <div class="gf-form-group">
+        <h5 class="section-heading">Conditions</h5>
+        <div class="gf-form-inline" ng-repeat="conditionModel in ctrl.conditionModels">
+          <div class="gf-form">
+            <metric-segment-model css-class="query-keyword width-5" ng-if="$index" property="conditionModel.operator.type" options="ctrl.evalOperators" custom="false"></metric-segment-model>
+            <span class="gf-form-label query-keyword width-5" ng-if="$index===0">WHEN</span>
+          </div>
+                <div class="gf-form">
+            <query-part-editor class="gf-form-label query-part width-9" part="conditionModel.reducerPart" handle-event="ctrl.handleReducerPartEvent(conditionModel, $event)">
+            </query-part-editor>
+                  <span class="gf-form-label query-keyword">OF</span>
+          </div>
+          <div class="gf-form">
+            <query-part-editor class="gf-form-label query-part" part="conditionModel.queryPart" handle-event="ctrl.handleQueryPartEvent(conditionModel, $event)">
+            </query-part-editor>
+          </div>
+          <div class="gf-form">
+            <metric-segment-model property="conditionModel.evaluator.type" options="ctrl.evalFunctions" custom="false" css-class="query-keyword" on-change="ctrl.evaluatorTypeChanged(conditionModel.evaluator)"></metric-segment-model>
+            <input class="gf-form-input max-width-9" type="number" step="any" ng-hide="conditionModel.evaluator.params.length === 0" ng-model="conditionModel.evaluator.params[0]" ng-change="ctrl.evaluatorParamsChanged()"></input>
+                  <label class="gf-form-label query-keyword" ng-show="conditionModel.evaluator.params.length === 2">TO</label>
+                  <input class="gf-form-input max-width-9" type="number" step="any" ng-if="conditionModel.evaluator.params.length === 2" ng-model="conditionModel.evaluator.params[1]" ng-change="ctrl.evaluatorParamsChanged()"></input>
+          </div>
+          <div class="gf-form">
+            <label class="gf-form-label">
+              <a class="pointer" tabindex="1" ng-click="ctrl.removeCondition($index)">
+                <i class="fa fa-trash"></i>
+              </a>
+            </label>
+          </div>
+        </div>
 
-				<div class="gf-form">
-					<label class="gf-form-label dropdown">
-						<a class="pointer dropdown-toggle" data-toggle="dropdown">
-							<i class="fa fa-plus"></i>
-						</a>
-						<ul class="dropdown-menu" role="menu">
-							<li ng-repeat="ct in ctrl.conditionTypes" role="menuitem">
-								<a ng-click="ctrl.addCondition(ct.value);">{{ct.text}}</a>
-							</li>
-						</ul>
-					</label>
-				</div>
-			</div>
+        <div class="gf-form">
+          <label class="gf-form-label dropdown">
+            <a class="pointer dropdown-toggle" data-toggle="dropdown">
+              <i class="fa fa-plus"></i>
+            </a>
+            <ul class="dropdown-menu" role="menu">
+              <li ng-repeat="ct in ctrl.conditionTypes" role="menuitem">
+                <a ng-click="ctrl.addCondition(ct.value);">{{ct.text}}</a>
+              </li>
+            </ul>
+          </label>
+        </div>
+      </div>
 
-			<div class="gf-form-group">
-				<div class="gf-form">
-          			<span class="gf-form-label width-18">If no data or all values are null</span>
-          			<span class="gf-form-label query-keyword">SET STATE TO</span>
-					<div class="gf-form-select-wrapper">
-						<select class="gf-form-input" ng-model="ctrl.alert.noDataState" ng-options="f.value as f.text for f in ctrl.noDataModes">
-						</select>
-					</div>
-				</div>
+      <div class="gf-form-group">
+        <div class="gf-form">
+                <span class="gf-form-label width-18">If no data or all values are null</span>
+                <span class="gf-form-label query-keyword">SET STATE TO</span>
+          <div class="gf-form-select-wrapper">
+            <select class="gf-form-input" ng-model="ctrl.alert.noDataState" ng-options="f.value as f.text for f in ctrl.noDataModes">
+            </select>
+          </div>
+        </div>
 
-				<div class="gf-form">
-          			<span class="gf-form-label width-18">If execution error or timeout</span>
-          			<span class="gf-form-label query-keyword">SET STATE TO</span>
-					<div class="gf-form-select-wrapper">
-						<select class="gf-form-input" ng-model="ctrl.alert.executionErrorState" ng-options="f.value as f.text for f in ctrl.executionErrorModes">
-						</select>
-					</div>
-				</div>
+        <div class="gf-form">
+                <span class="gf-form-label width-18">If execution error or timeout</span>
+                <span class="gf-form-label query-keyword">SET STATE TO</span>
+          <div class="gf-form-select-wrapper">
+            <select class="gf-form-input" ng-model="ctrl.alert.executionErrorState" ng-options="f.value as f.text for f in ctrl.executionErrorModes">
+            </select>
+          </div>
+        </div>
 
-				<div class="gf-form-button-row">
-					<button class="btn btn-inverse" ng-click="ctrl.test()">
-						Test Rule
-					</button>
-				</div>
-			</div>
+        <div class="gf-form-button-row">
+          <button class="btn btn-inverse" ng-click="ctrl.test()">
+            Test Rule
+          </button>
+        </div>
+      </div>
 
-			<div class="gf-form-group" ng-if="ctrl.testing">
-				Evaluating rule <i class="fa fa-spinner fa-spin"></i>
-			</div>
+      <div class="gf-form-group" ng-if="ctrl.testing">
+        Evaluating rule <i class="fa fa-spinner fa-spin"></i>
+      </div>
 
-			<div class="gf-form-group" ng-if="ctrl.testResult">
-				<json-tree root-name="result" object="ctrl.testResult" start-expanded="true"></json-tree>
-			</div>
-		</div>
+      <div class="gf-form-group" ng-if="ctrl.testResult">
+        <json-tree root-name="result" object="ctrl.testResult" start-expanded="true"></json-tree>
+      </div>
+    </div>
 
-		<div class="gf-form-group" ng-if="ctrl.subTabIndex === 1">
-			<h5 class="section-heading">Notifications</h5>
-			<div class="gf-form-inline">
-				<div class="gf-form max-width-30">
-					<span class="gf-form-label width-8">Send to</span>
-					<span class="gf-form-label" ng-repeat="nc in ctrl.alertNotifications" ng-style="{'background-color': nc.bgColor }">
-						<i class="{{nc.iconClass}}"></i>&nbsp;{{nc.name}}&nbsp;
-						<i class="fa fa-remove pointer muted" ng-click="ctrl.removeNotification($index)" ng-if="nc.isDefault === false"></i>
-					</span>
-					<metric-segment segment="ctrl.addNotificationSegment" get-options="ctrl.getNotifications()" on-change="ctrl.notificationAdded()"></metric-segment>
-				</div>
-			</div>
-			<div class="gf-form gf-form--v-stretch">
-				<span class="gf-form-label width-8">Message</span>
-				<textarea class="gf-form-input" rows="10" ng-model="ctrl.alert.message"  placeholder="Notification message details..."></textarea>
-			</div>
-		</div>
+    <div class="gf-form-group" ng-if="ctrl.subTabIndex === 1">
+      <h5 class="section-heading">Notifications</h5>
+      <div class="gf-form-inline">
+        <div class="gf-form max-width-30">
+          <span class="gf-form-label width-8">Send to</span>
+          <span class="gf-form-label" ng-repeat="nc in ctrl.alertNotifications" ng-style="{'background-color': nc.bgColor }">
+            <i class="{{nc.iconClass}}"></i>&nbsp;{{nc.name}}&nbsp;
+            <i class="fa fa-remove pointer muted" ng-click="ctrl.removeNotification($index)" ng-if="nc.isDefault === false"></i>
+          </span>
+          <metric-segment segment="ctrl.addNotificationSegment" get-options="ctrl.getNotifications()" on-change="ctrl.notificationAdded()"></metric-segment>
+        </div>
+      </div>
+      <div class="gf-form gf-form--v-stretch">
+        <span class="gf-form-label width-8">Message</span>
+        <textarea class="gf-form-input" rows="10" ng-model="ctrl.alert.message"  placeholder="Notification message details..."></textarea>
+      </div>
+    </div>
 
-		<div class="gf-form-group" style="max-width: 720px;" ng-if="ctrl.subTabIndex === 2">
-			<button class="btn btn-mini btn-danger pull-right" ng-click="ctrl.clearHistory()"><i class="fa fa-trash"></i>&nbsp;Clear history</button>
-      		<h5 class="section-heading" style="whitespace: nowrap">
-				State history <span class="muted small">(last 50 state changes)</span>
-			</h5>
+    <div class="gf-form-group" style="max-width: 720px;" ng-if="ctrl.subTabIndex === 2">
+      <button class="btn btn-mini btn-danger pull-right" ng-click="ctrl.clearHistory()"><i class="fa fa-trash"></i>&nbsp;Clear history</button>
+          <h5 class="section-heading" style="whitespace: nowrap">
+        State history <span class="muted small">(last 50 state changes)</span>
+      </h5>
 
-			<div ng-show="ctrl.alertHistory.length === 0">
-				<br>
-				<i>No state changes recorded</i>
-			</div>
+      <div ng-show="ctrl.alertHistory.length === 0">
+        <br>
+        <i>No state changes recorded</i>
+      </div>
 
       <ol class="alert-rule-list" >
         <li class="alert-rule-item" ng-repeat="al in ctrl.alertHistory">

public/app/features/alerting/state/alertDef.ts

@@ -99,6 +99,13 @@ function getStateDisplayModel(state) {
         stateClass: 'alert-state-warning',
       };
     }
+    case 'unknown': {
+      return {
+        text: 'UNKNOWN',
+        iconClass: 'fa fa-question',
+        stateClass: 'alert-state-paused',
+      };
+    }
   }
 
   throw { message: 'Unknown alert state' };

public/app/features/annotations/annotation_tooltip.ts

@@ -32,7 +32,7 @@ export function annotationTooltipDirective($sanitize, dashboardSrv, contextSrv,
       if (event.alertId) {
         const stateModel = alertDef.getStateDisplayModel(event.newState);
         titleStateClass = stateModel.stateClass;
-        title = `<i class="icon-gf ${stateModel.iconClass}"></i> ${stateModel.text}`;
+        title = `<i class="${stateModel.iconClass}"></i> ${stateModel.text}`;
         text = alertDef.getAlertAnnotationInfo(event);
         if (event.text) {
           text = text + '<br />' + event.text;

public/app/features/annotations/event_manager.ts

@@ -7,6 +7,7 @@ import {
   OK_COLOR,
   ALERTING_COLOR,
   NO_DATA_COLOR,
+  PENDING_COLOR,
   DEFAULT_ANNOTATION_COLOR,
   REGION_FILL_ALPHA,
 } from 'app/core/utils/colors';
@@ -71,6 +72,11 @@ export class EventManager {
         position: 'BOTTOM',
         markerSize: 5,
       },
+      $__pending: {
+        color: PENDING_COLOR,
+        position: 'BOTTOM',
+        markerSize: 5,
+      },
       $__editing: {
         color: DEFAULT_ANNOTATION_COLOR,
         position: 'BOTTOM',

public/app/features/panel/panel_directive.ts

@@ -161,7 +161,11 @@ module.directive('grafanaPanel', ($rootScope, $document, $timeout) => {
             panelContainer.removeClass('panel-alert-state--' + lastAlertState);
           }
 
-          if (ctrl.alertState.state === 'ok' || ctrl.alertState.state === 'alerting') {
+          if (
+            ctrl.alertState.state === 'ok' ||
+            ctrl.alertState.state === 'alerting' ||
+            ctrl.alertState.state === 'pending'
+          ) {
             panelContainer.addClass('panel-alert-state--' + ctrl.alertState.state);
           }
 

public/app/plugins/panel/alertlist/editor.html

@@ -50,5 +50,6 @@
     <gf-form-switch class="gf-form" label="No data" label-class="width-10" checked="ctrl.stateFilter['no_data']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
     <gf-form-switch class="gf-form" label="Execution error" label-class="width-10" checked="ctrl.stateFilter['execution_error']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
     <gf-form-switch class="gf-form" label="Alerting" label-class="width-10" checked="ctrl.stateFilter['alerting']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
+    <gf-form-switch class="gf-form" label="Pending" label-class="width-10" checked="ctrl.stateFilter['pending']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
   </div>
 </div>

public/sass/pages/_alerting.scss

@@ -66,6 +66,13 @@
       content: '\e611';
     }
   }
+
+  &--pending {
+    .panel-alert-icon:before {
+      color: $warn;
+      content: '\e611';
+    }
+  }
 }
 
 @keyframes alerting-panel {