Merge pull request #3736 from mtanda/cloudwatch_custom_metrics_alpha

(cloudwatch) custom metrics support

pkg/api/cloudwatch/metrics.go

@@ -3,7 +3,14 @@ package cloudwatch
 import (
 	"encoding/json"
 	"sort"
+	"strings"
+	"sync"
+	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awsutil"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/cloudwatch"
 	"github.com/grafana/grafana/pkg/middleware"
 	"github.com/grafana/grafana/pkg/util"
 )
@@ -11,6 +18,14 @@ import (
 var metricsMap map[string][]string
 var dimensionsMap map[string][]string
 
+type CustomMetricsCache struct {
+	Expire time.Time
+	Cache  []string
+}
+
+var customMetricsMetricsMap map[string]map[string]map[string]*CustomMetricsCache
+var customMetricsDimensionsMap map[string]map[string]map[string]*CustomMetricsCache
+
 func init() {
 	metricsMap = map[string][]string{
 		"AWS/AutoScaling": {"GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity", "GroupInServiceInstances", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"},
@@ -85,6 +100,9 @@ func init() {
 		"AWS/WAF":              {"Rule", "WebACL"},
 		"AWS/WorkSpaces":       {"DirectoryId", "WorkspaceId"},
 	}
+
+	customMetricsMetricsMap = make(map[string]map[string]map[string]*CustomMetricsCache)
+	customMetricsDimensionsMap = make(map[string]map[string]map[string]*CustomMetricsCache)
 }
 
 // Whenever this list is updated, frontend list should also be updated.
@@ -127,10 +145,19 @@ func handleGetMetrics(req *cwRequest, c *middleware.Context) {
 
 	json.Unmarshal(req.Body, reqParam)
 
-	namespaceMetrics, exists := metricsMap[reqParam.Parameters.Namespace]
-	if !exists {
-		c.JsonApiErr(404, "Unable to find namespace "+reqParam.Parameters.Namespace, nil)
-		return
+	var namespaceMetrics []string
+	if !isCustomMetrics(reqParam.Parameters.Namespace) {
+		var exists bool
+		if namespaceMetrics, exists = metricsMap[reqParam.Parameters.Namespace]; !exists {
+			c.JsonApiErr(404, "Unable to find namespace "+reqParam.Parameters.Namespace, nil)
+			return
+		}
+	} else {
+		var err error
+		if namespaceMetrics, err = getMetricsForCustomMetrics(req.Region, reqParam.Parameters.Namespace, req.DataSource.Database, getAllMetrics); err != nil {
+			c.JsonApiErr(500, "Unable to call AWS API", err)
+			return
+		}
 	}
 	sort.Sort(sort.StringSlice(namespaceMetrics))
 
@@ -151,10 +178,19 @@ func handleGetDimensions(req *cwRequest, c *middleware.Context) {
 
 	json.Unmarshal(req.Body, reqParam)
 
-	dimensionValues, exists := dimensionsMap[reqParam.Parameters.Namespace]
-	if !exists {
-		c.JsonApiErr(404, "Unable to find dimension "+reqParam.Parameters.Namespace, nil)
-		return
+	var dimensionValues []string
+	if !isCustomMetrics(reqParam.Parameters.Namespace) {
+		var exists bool
+		if dimensionValues, exists = dimensionsMap[reqParam.Parameters.Namespace]; !exists {
+			c.JsonApiErr(404, "Unable to find dimension "+reqParam.Parameters.Namespace, nil)
+			return
+		}
+	} else {
+		var err error
+		if dimensionValues, err = getDimensionsForCustomMetrics(req.Region, reqParam.Parameters.Namespace, req.DataSource.Database, getAllMetrics); err != nil {
+			c.JsonApiErr(500, "Unable to call AWS API", err)
+			return
+		}
 	}
 	sort.Sort(sort.StringSlice(dimensionValues))
 
@@ -165,3 +201,122 @@ func handleGetDimensions(req *cwRequest, c *middleware.Context) {
 
 	c.JSON(200, result)
 }
+
+func getAllMetrics(region string, namespace string, database string) (cloudwatch.ListMetricsOutput, error) {
+	cfg := &aws.Config{
+		Region:      aws.String(region),
+		Credentials: getCredentials(database),
+	}
+
+	svc := cloudwatch.New(session.New(cfg), cfg)
+
+	params := &cloudwatch.ListMetricsInput{
+		Namespace: aws.String(namespace),
+	}
+
+	var resp cloudwatch.ListMetricsOutput
+	err := svc.ListMetricsPages(params,
+		func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool {
+			metrics, _ := awsutil.ValuesAtPath(page, "Metrics")
+			for _, metric := range metrics {
+				resp.Metrics = append(resp.Metrics, metric.(*cloudwatch.Metric))
+			}
+			return !lastPage
+		})
+	if err != nil {
+		return resp, err
+	}
+
+	return resp, nil
+}
+
+var metricsCacheLock sync.Mutex
+
+func getMetricsForCustomMetrics(region string, namespace string, database string, getAllMetrics func(string, string, string) (cloudwatch.ListMetricsOutput, error)) ([]string, error) {
+	result, err := getAllMetrics(region, namespace, database)
+	if err != nil {
+		return []string{}, err
+	}
+
+	metricsCacheLock.Lock()
+	defer metricsCacheLock.Unlock()
+
+	if _, ok := customMetricsMetricsMap[database]; !ok {
+		customMetricsMetricsMap[database] = make(map[string]map[string]*CustomMetricsCache)
+	}
+	if _, ok := customMetricsMetricsMap[database][region]; !ok {
+		customMetricsMetricsMap[database][region] = make(map[string]*CustomMetricsCache)
+	}
+	if _, ok := customMetricsMetricsMap[database][region][namespace]; !ok {
+		customMetricsMetricsMap[database][region][namespace] = &CustomMetricsCache{}
+		customMetricsMetricsMap[database][region][namespace].Cache = make([]string, 0)
+	}
+
+	if customMetricsMetricsMap[database][region][namespace].Expire.After(time.Now()) {
+		return customMetricsMetricsMap[database][region][namespace].Cache, nil
+	}
+	customMetricsMetricsMap[database][region][namespace].Cache = make([]string, 0)
+	customMetricsMetricsMap[database][region][namespace].Expire = time.Now().Add(5 * time.Minute)
+
+	for _, metric := range result.Metrics {
+		if isDuplicate(customMetricsMetricsMap[database][region][namespace].Cache, *metric.MetricName) {
+			continue
+		}
+		customMetricsMetricsMap[database][region][namespace].Cache = append(customMetricsMetricsMap[database][region][namespace].Cache, *metric.MetricName)
+	}
+
+	return customMetricsMetricsMap[database][region][namespace].Cache, nil
+}
+
+var dimensionsCacheLock sync.Mutex
+
+func getDimensionsForCustomMetrics(region string, namespace string, database string, getAllMetrics func(string, string, string) (cloudwatch.ListMetricsOutput, error)) ([]string, error) {
+	result, err := getAllMetrics(region, namespace, database)
+	if err != nil {
+		return []string{}, err
+	}
+
+	dimensionsCacheLock.Lock()
+	defer dimensionsCacheLock.Unlock()
+
+	if _, ok := customMetricsDimensionsMap[database]; !ok {
+		customMetricsDimensionsMap[database] = make(map[string]map[string]*CustomMetricsCache)
+	}
+	if _, ok := customMetricsDimensionsMap[database][region]; !ok {
+		customMetricsDimensionsMap[database][region] = make(map[string]*CustomMetricsCache)
+	}
+	if _, ok := customMetricsDimensionsMap[database][region][namespace]; !ok {
+		customMetricsDimensionsMap[database][region][namespace] = &CustomMetricsCache{}
+		customMetricsDimensionsMap[database][region][namespace].Cache = make([]string, 0)
+	}
+
+	if customMetricsDimensionsMap[database][region][namespace].Expire.After(time.Now()) {
+		return customMetricsDimensionsMap[database][region][namespace].Cache, nil
+	}
+	customMetricsDimensionsMap[database][region][namespace].Cache = make([]string, 0)
+	customMetricsDimensionsMap[database][region][namespace].Expire = time.Now().Add(5 * time.Minute)
+
+	for _, metric := range result.Metrics {
+		for _, dimension := range metric.Dimensions {
+			if isDuplicate(customMetricsDimensionsMap[database][region][namespace].Cache, *dimension.Name) {
+				continue
+			}
+			customMetricsDimensionsMap[database][region][namespace].Cache = append(customMetricsDimensionsMap[database][region][namespace].Cache, *dimension.Name)
+		}
+	}
+
+	return customMetricsDimensionsMap[database][region][namespace].Cache, nil
+}
+
+func isDuplicate(nameList []string, target string) bool {
+	for _, name := range nameList {
+		if name == target {
+			return true
+		}
+	}
+	return false
+}
+
+func isCustomMetrics(namespace string) bool {
+	return strings.Index(namespace, "AWS/") != 0
+}

pkg/api/cloudwatch/metrics_test.go

@@ -0,0 +1,63 @@
+package cloudwatch
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/cloudwatch"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestCloudWatchMetrics(t *testing.T) {
+
+	Convey("When calling getMetricsForCustomMetrics", t, func() {
+		region := "us-east-1"
+		namespace := "Foo"
+		database := "default"
+		f := func(region string, namespace string, database string) (cloudwatch.ListMetricsOutput, error) {
+			return cloudwatch.ListMetricsOutput{
+				Metrics: []*cloudwatch.Metric{
+					{
+						MetricName: aws.String("Test_MetricName"),
+						Dimensions: []*cloudwatch.Dimension{
+							{
+								Name: aws.String("Test_DimensionName"),
+							},
+						},
+					},
+				},
+			}, nil
+		}
+		metrics, _ := getMetricsForCustomMetrics(region, namespace, database, f)
+
+		Convey("Should contain Test_MetricName", func() {
+			So(metrics, ShouldContain, "Test_MetricName")
+		})
+	})
+
+	Convey("When calling getDimensionsForCustomMetrics", t, func() {
+		region := "us-east-1"
+		namespace := "Foo"
+		database := "default"
+		f := func(region string, namespace string, database string) (cloudwatch.ListMetricsOutput, error) {
+			return cloudwatch.ListMetricsOutput{
+				Metrics: []*cloudwatch.Metric{
+					{
+						MetricName: aws.String("Test_MetricName"),
+						Dimensions: []*cloudwatch.Dimension{
+							{
+								Name: aws.String("Test_DimensionName"),
+							},
+						},
+					},
+				},
+			}, nil
+		}
+		dimensionKeys, _ := getDimensionsForCustomMetrics(region, namespace, database, f)
+
+		Convey("Should contain Test_DimensionName", func() {
+			So(dimensionKeys, ShouldContain, "Test_DimensionName")
+		})
+	})
+
+}

public/app/plugins/datasource/cloudwatch/datasource.js

@@ -90,18 +90,20 @@ function (angular, _, moment, dateMath) {
       return this.awsRequest({action: '__GetNamespaces'});
     };
 
-    this.getMetrics = function(namespace) {
+    this.getMetrics = function(namespace, region) {
       return this.awsRequest({
         action: '__GetMetrics',
+        region: region,
         parameters: {
           namespace: templateSrv.replace(namespace)
         }
       });
     };
 
-    this.getDimensionKeys = function(namespace) {
+    this.getDimensionKeys = function(namespace, region) {
       return this.awsRequest({
         action: '__GetDimensions',
+        region: region,
         parameters: {
           namespace: templateSrv.replace(namespace)
         }
@@ -164,14 +166,14 @@ function (angular, _, moment, dateMath) {
         return this.getNamespaces();
       }
 
-      var metricNameQuery = query.match(/^metrics\(([^\)]+?)\)/);
+      var metricNameQuery = query.match(/^metrics\(([^\)]+?)(,\s?([^,]+?))?\)/);
       if (metricNameQuery) {
-        return this.getMetrics(metricNameQuery[1]);
+        return this.getMetrics(metricNameQuery[1], metricNameQuery[3]);
       }
 
-      var dimensionKeysQuery = query.match(/^dimension_keys\(([^\)]+?)\)/);
+      var dimensionKeysQuery = query.match(/^dimension_keys\(([^\)]+?)(,\s?([^,]+?))?\)/);
       if (dimensionKeysQuery) {
-        return this.getDimensionKeys(dimensionKeysQuery[1]);
+        return this.getDimensionKeys(dimensionKeysQuery[1], dimensionKeysQuery[3]);
       }
 
       var dimensionValuesQuery = query.match(/^dimension_values\(([^,]+?),\s?([^,]+?),\s?([^,]+?),\s?([^,]+?)\)/);

public/app/plugins/datasource/cloudwatch/query_parameter_ctrl.js

@@ -102,7 +102,7 @@ function (angular, _) {
       var query = $q.when([]);
 
       if (segment.type === 'key' || segment.type === 'plus-button') {
-        query = $scope.datasource.getDimensionKeys($scope.target.namespace);
+        query = $scope.datasource.getDimensionKeys($scope.target.namespace, $scope.target.region);
       } else if (segment.type === 'value')  {
         var dimensionKey = $scope.dimSegments[$index-2].value;
         query = $scope.datasource.getDimensionValues(target.region, target.namespace, target.metricName, dimensionKey, {});
@@ -160,7 +160,7 @@ function (angular, _) {
     };
 
     $scope.getMetrics = function() {
-      return $scope.datasource.metricFindQuery('metrics(' + $scope.target.namespace + ')')
+      return $scope.datasource.metricFindQuery('metrics(' + $scope.target.namespace + ',' + $scope.target.region + ')')
       .then($scope.transformToSegments(true));
     };