feat(metrics): change active alerts to guage

pkg/api/metrics.go

@@ -69,6 +69,10 @@ func GetInternalMetrics(c *middleware.Context) Response {
 		metricName := m.Name() + m.StringifyTags()
 
 		switch metric := m.(type) {
+		case metrics.Gauge:
+			resp[metricName] = map[string]interface{}{
+				"value": metric.Value(),
+			}
 		case metrics.Counter:
 			resp[metricName] = map[string]interface{}{
 				"count": metric.Count(),

pkg/metrics/metrics.go

@@ -9,54 +9,53 @@ func init() {
 }
 
 var (
-	M_Instance_Start                     		Counter
-	M_Page_Status_200                    		Counter
-	M_Page_Status_500                    		Counter
-	M_Page_Status_404                    		Counter
-	M_Page_Status_Unknown                		Counter
-	M_Api_Status_200                     		Counter
-	M_Api_Status_404                     		Counter
-	M_Api_Status_500                     		Counter
-	M_Api_Status_Unknown                 		Counter
-	M_Proxy_Status_200                   		Counter
-	M_Proxy_Status_404                   		Counter
-	M_Proxy_Status_500                   		Counter
-	M_Proxy_Status_Unknown               		Counter
-	M_Api_User_SignUpStarted             		Counter
-	M_Api_User_SignUpCompleted           		Counter
-	M_Api_User_SignUpInvite              		Counter
-	M_Api_Dashboard_Save                 		Timer
-	M_Api_Dashboard_Get                  		Timer
-	M_Api_Dashboard_Search               		Timer
-	M_Api_Admin_User_Create              		Counter
-	M_Api_Login_Post                     		Counter
-	M_Api_Login_OAuth                    		Counter
-	M_Api_Org_Create                     		Counter
-	M_Api_Dashboard_Snapshot_Create      		Counter
-	M_Api_Dashboard_Snapshot_External    		Counter
-	M_Api_Dashboard_Snapshot_Get         		Counter
-	M_Models_Dashboard_Insert            		Counter
-	M_Alerting_Result_State_Alerting     		Counter
-	M_Alerting_Result_State_Ok           		Counter
-	M_Alerting_Result_State_Paused       		Counter
-	M_Alerting_Result_State_NoData       		Counter
-	M_Alerting_Result_State_Pending      		Counter
-	M_Alerting_Active_Alerts             		Counter
-	M_Alerting_Notification_Sent_Slack   		Counter
-	M_Alerting_Notification_Sent_Email   		Counter
-	M_Alerting_Notification_Sent_Webhook 		Counter
-	M_Alerting_Notification_Sent_PagerDuty	Counter
-
+	M_Instance_Start                       Counter
+	M_Page_Status_200                      Counter
+	M_Page_Status_500                      Counter
+	M_Page_Status_404                      Counter
+	M_Page_Status_Unknown                  Counter
+	M_Api_Status_200                       Counter
+	M_Api_Status_404                       Counter
+	M_Api_Status_500                       Counter
+	M_Api_Status_Unknown                   Counter
+	M_Proxy_Status_200                     Counter
+	M_Proxy_Status_404                     Counter
+	M_Proxy_Status_500                     Counter
+	M_Proxy_Status_Unknown                 Counter
+	M_Api_User_SignUpStarted               Counter
+	M_Api_User_SignUpCompleted             Counter
+	M_Api_User_SignUpInvite                Counter
+	M_Api_Dashboard_Save                   Timer
+	M_Api_Dashboard_Get                    Timer
+	M_Api_Dashboard_Search                 Timer
+	M_Api_Admin_User_Create                Counter
+	M_Api_Login_Post                       Counter
+	M_Api_Login_OAuth                      Counter
+	M_Api_Org_Create                       Counter
+	M_Api_Dashboard_Snapshot_Create        Counter
+	M_Api_Dashboard_Snapshot_External      Counter
+	M_Api_Dashboard_Snapshot_Get           Counter
+	M_Models_Dashboard_Insert              Counter
+	M_Alerting_Result_State_Alerting       Counter
+	M_Alerting_Result_State_Ok             Counter
+	M_Alerting_Result_State_Paused         Counter
+	M_Alerting_Result_State_NoData         Counter
+	M_Alerting_Result_State_Pending        Counter
+	M_Alerting_Notification_Sent_Slack     Counter
+	M_Alerting_Notification_Sent_Email     Counter
+	M_Alerting_Notification_Sent_Webhook   Counter
+	M_Alerting_Notification_Sent_PagerDuty Counter
 
 	// Timers
 	M_DataSource_ProxyReq_Timer Timer
 	M_Alerting_Exeuction_Time   Timer
 
 	// StatTotals
-	M_StatTotal_Dashboards Gauge
-	M_StatTotal_Users      Gauge
-	M_StatTotal_Orgs       Gauge
-	M_StatTotal_Playlists  Gauge
+	M_Alerting_Active_Alerts Gauge
+	M_StatTotal_Dashboards   Gauge
+	M_StatTotal_Users        Gauge
+	M_StatTotal_Orgs         Gauge
+	M_StatTotal_Playlists    Gauge
 )
 
 func initMetricVars(settings *MetricSettings) {
@@ -105,7 +104,6 @@ func initMetricVars(settings *MetricSettings) {
 	M_Alerting_Result_State_NoData = RegCounter("alerting.result", "state", "no_data")
 	M_Alerting_Result_State_Pending = RegCounter("alerting.result", "state", "pending")
 
-	M_Alerting_Active_Alerts = RegCounter("alerting.active_alerts")
 	M_Alerting_Notification_Sent_Slack = RegCounter("alerting.notifications_sent", "type", "slack")
 	M_Alerting_Notification_Sent_Email = RegCounter("alerting.notifications_sent", "type", "email")
 	M_Alerting_Notification_Sent_Webhook = RegCounter("alerting.notifications_sent", "type", "webhook")
@@ -116,6 +114,7 @@ func initMetricVars(settings *MetricSettings) {
 	M_Alerting_Exeuction_Time = RegTimer("alerting.execution_time")
 
 	// StatTotals
+	M_Alerting_Active_Alerts = RegGauge("alerting.active_alerts")
 	M_StatTotal_Dashboards = RegGauge("stat_totals", "stat", "dashboards")
 	M_StatTotal_Users = RegGauge("stat_totals", "stat", "users")
 	M_StatTotal_Orgs = RegGauge("stat_totals", "stat", "orgs")

pkg/services/alerting/reader.go

@@ -59,7 +59,7 @@ func (arr *DefaultRuleReader) Fetch() []*Rule {
 		}
 	}
 
-	metrics.M_Alerting_Active_Alerts.Inc(int64(len(res)))
+	metrics.M_Alerting_Active_Alerts.Update(int64(len(res)))
 	return res
 }