Added Pagerduty basic alerting

pkg/metrics/metrics.go

@@ -9,42 +9,44 @@ func init() {
 }
 
 var (
-	M_Instance_Start                     Counter
-	M_Page_Status_200                    Counter
-	M_Page_Status_500                    Counter
-	M_Page_Status_404                    Counter
-	M_Page_Status_Unknown                Counter
-	M_Api_Status_200                     Counter
-	M_Api_Status_404                     Counter
-	M_Api_Status_500                     Counter
-	M_Api_Status_Unknown                 Counter
-	M_Proxy_Status_200                   Counter
-	M_Proxy_Status_404                   Counter
-	M_Proxy_Status_500                   Counter
-	M_Proxy_Status_Unknown               Counter
-	M_Api_User_SignUpStarted             Counter
-	M_Api_User_SignUpCompleted           Counter
-	M_Api_User_SignUpInvite              Counter
-	M_Api_Dashboard_Save                 Timer
-	M_Api_Dashboard_Get                  Timer
-	M_Api_Dashboard_Search               Timer
-	M_Api_Admin_User_Create              Counter
-	M_Api_Login_Post                     Counter
-	M_Api_Login_OAuth                    Counter
-	M_Api_Org_Create                     Counter
-	M_Api_Dashboard_Snapshot_Create      Counter
-	M_Api_Dashboard_Snapshot_External    Counter
-	M_Api_Dashboard_Snapshot_Get         Counter
-	M_Models_Dashboard_Insert            Counter
-	M_Alerting_Result_State_Alerting     Counter
-	M_Alerting_Result_State_Ok           Counter
-	M_Alerting_Result_State_Paused       Counter
-	M_Alerting_Result_State_NoData       Counter
-	M_Alerting_Result_State_Pending      Counter
-	M_Alerting_Active_Alerts             Counter
-	M_Alerting_Notification_Sent_Slack   Counter
-	M_Alerting_Notification_Sent_Email   Counter
-	M_Alerting_Notification_Sent_Webhook Counter
+	M_Instance_Start                     		Counter
+	M_Page_Status_200                    		Counter
+	M_Page_Status_500                    		Counter
+	M_Page_Status_404                    		Counter
+	M_Page_Status_Unknown                		Counter
+	M_Api_Status_200                     		Counter
+	M_Api_Status_404                     		Counter
+	M_Api_Status_500                     		Counter
+	M_Api_Status_Unknown                 		Counter
+	M_Proxy_Status_200                   		Counter
+	M_Proxy_Status_404                   		Counter
+	M_Proxy_Status_500                   		Counter
+	M_Proxy_Status_Unknown               		Counter
+	M_Api_User_SignUpStarted             		Counter
+	M_Api_User_SignUpCompleted           		Counter
+	M_Api_User_SignUpInvite              		Counter
+	M_Api_Dashboard_Save                 		Timer
+	M_Api_Dashboard_Get                  		Timer
+	M_Api_Dashboard_Search               		Timer
+	M_Api_Admin_User_Create              		Counter
+	M_Api_Login_Post                     		Counter
+	M_Api_Login_OAuth                    		Counter
+	M_Api_Org_Create                     		Counter
+	M_Api_Dashboard_Snapshot_Create      		Counter
+	M_Api_Dashboard_Snapshot_External    		Counter
+	M_Api_Dashboard_Snapshot_Get         		Counter
+	M_Models_Dashboard_Insert            		Counter
+	M_Alerting_Result_State_Alerting     		Counter
+	M_Alerting_Result_State_Ok           		Counter
+	M_Alerting_Result_State_Paused       		Counter
+	M_Alerting_Result_State_NoData       		Counter
+	M_Alerting_Result_State_Pending      		Counter
+	M_Alerting_Active_Alerts             		Counter
+	M_Alerting_Notification_Sent_Slack   		Counter
+	M_Alerting_Notification_Sent_Email   		Counter
+	M_Alerting_Notification_Sent_Webhook 		Counter
+	M_Alerting_Notification_Sent_PagerDuty	Counter
+
 
 	// Timers
 	M_DataSource_ProxyReq_Timer Timer
@@ -107,6 +109,7 @@ func initMetricVars(settings *MetricSettings) {
 	M_Alerting_Notification_Sent_Slack = RegCounter("alerting.notifications_sent", "type", "slack")
 	M_Alerting_Notification_Sent_Email = RegCounter("alerting.notifications_sent", "type", "email")
 	M_Alerting_Notification_Sent_Webhook = RegCounter("alerting.notifications_sent", "type", "webhook")
+	M_Alerting_Notification_Sent_PagerDuty = RegCounter("alerting.notifications_sent", "type", "pagerduty")
 
 	// Timers
 	M_DataSource_ProxyReq_Timer = RegTimer("api.dataproxy.request.all")

pkg/services/alerting/notifiers/pagerduty.go

@@ -0,0 +1,83 @@
+package notifiers
+
+import (
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/log"
+	"github.com/grafana/grafana/pkg/metrics"
+	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting"
+)
+
+func init() {
+	alerting.RegisterNotifier("pagerduty", NewPagerdutyNotifier)
+}
+
+func NewPagerdutyNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
+	key := model.Settings.Get("integrationKey").MustString()
+	if key == "" {
+		return nil, alerting.ValidationError{Reason: "Could not find integration key property in settings"}
+	}
+
+	return &PagerdutyNotifier{
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		Key:          key,
+		log:          log.New("alerting.notifier.pagerduty"),
+	}, nil
+}
+
+type PagerdutyNotifier struct {
+	NotifierBase
+	Key        string
+	log        log.Logger
+}
+
+func (this *PagerdutyNotifier) Notify(evalContext *alerting.EvalContext) error {
+	this.log.Info("Notifying Pagerduty")
+	metrics.M_Alerting_Notification_Sent_PagerDuty.Inc(1)
+
+		if evalContext.Rule.State == m.AlertStateAlerting {
+
+		// Pagerduty Events API URL
+		pgEventsUrl := "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
+
+		bodyJSON := simplejson.New()
+		bodyJSON.Set("service_key", this.Key)
+		bodyJSON.Set("description", evalContext.Rule.Name + "-" + evalContext.Rule.Message)
+		bodyJSON.Set("client", "Grafana")
+		bodyJSON.Set("event_type", "trigger")
+		
+		ruleUrl, err := evalContext.GetRuleUrl()
+		if err != nil {
+			this.log.Error("Failed get rule link", "error", err)
+			return err
+		}
+		bodyJSON.Set("client_url", ruleUrl)
+
+		if evalContext.ImagePublicUrl != "" {
+			var contexts []interface{}
+			imageJSON := simplejson.New()
+			imageJSON.Set("type", "image")
+			imageJSON.Set("src", evalContext.ImagePublicUrl)
+			contexts[0] = imageJSON
+			bodyJSON.Set("contexts", contexts)
+		}
+
+		body, _ := bodyJSON.MarshalJSON()
+
+		cmd := &m.SendWebhook{
+			Url:        pgEventsUrl,
+			Body:       string(body),
+			HttpMethod: "POST",
+		}
+
+		if err := bus.Dispatch(cmd); err != nil {
+			this.log.Error("Failed to send notification to Pagerduty", "error", err, "body", string(body))
+		}
+
+	} else {
+		this.log.Info("Not sending a trigger to Pagerduty", "state", evalContext.Rule.State)
+	}
+
+	return nil
+}

public/app/features/alerting/alert_tab_ctrl.ts

@@ -90,6 +90,7 @@ export class AlertTabCtrl {
       case "email": return "fa fa-envelope";
       case "slack": return "fa fa-slack";
       case "webhook": return "fa fa-cubes";
+      case "pagerduty": return "fa fa-bullhorn";
     }
   }
 

public/app/features/alerting/partials/notification_edit.html

@@ -19,7 +19,7 @@
       <div class="gf-form">
         <span class="gf-form-label width-12">Type</span>
         <div class="gf-form-select-wrapper width-15">
-          <select class="gf-form-input" ng-model="ctrl.model.type" ng-options="t for t in ['webhook', 'email', 'slack']" ng-change="ctrl.typeChanged(notification, $index)">
+          <select class="gf-form-input" ng-model="ctrl.model.type" ng-options="t for t in ['webhook', 'email', 'slack', 'pagerduty']" ng-change="ctrl.typeChanged(notification, $index)">
           </select>
         </div>
       </div>
@@ -97,6 +97,41 @@
       </div>
     </div>
 
+    <div class="gf-form-group" ng-if="ctrl.model.type === 'pagerduty'">
+      <h3 class="page-heading">Pagerduty settings</h3>
+      <div class="gf-form">
+        <span class="gf-form-label width-12">Integration Key</span>
+        <input type="text" required class="gf-form-input max-width-30" ng-model="ctrl.model.settings.integrationKey" placeholder="Pagerduty integeration Key"></input>
+      </div>
+      <div class="gf-form">
+        <gf-form-switch
+           class="gf-form"
+           label="Alert on No Data"
+           label-class="width-12"
+           checked="ctrl.model.settings.alertOnNoData"
+           tooltip="Trigger incident on No Data">
+        </gf-form-switch>
+      </div>
+      <div class="gf-form">
+        <gf-form-switch
+           class="gf-form"
+           label="Alert on Exec Error"
+           label-class="width-12"
+           checked="ctrl.model.settings.alertOnExecError"
+           tooltip="Trigger incident on Execution Error">
+        </gf-form-switch>
+      </div>
+      <div class="gf-form">
+        <gf-form-switch
+           class="gf-form"
+           label="Auto resolve on OK"
+           label-class="width-12"
+           checked="ctrl.model.settings.resolveOnOK"
+           tooltip="Resolve incident on OK">
+        </gf-form-switch>
+      </div>
+    </div>
+
     <div class="gf-form-group">
       <div class="gf-form-inline">
         <div class="gf-form width-6">