ES Nested type bucket aggregation (#4694) (#4694)

* (elasticsearch): add nested agg (use bucket aggs). fixes #4693

* (elasticsearch): rebased after merge of #6043 refactored from #4527

public/app/plugins/datasource/elasticsearch/bucket_agg.js

@@ -17,6 +17,7 @@ function (angular, _, queryDef) {
         target: "=",
         index: "=",
         onChange: "&",
+        // getNestedKeys: "&",
         getFields: "&",
       }
     };
@@ -52,14 +53,25 @@ function (angular, _, queryDef) {
         case 'date_histogram':
         case 'terms':  {
           delete $scope.agg.query;
+          delete $scope.agg.settings.nested.path;
           $scope.agg.field = 'select field';
           break;
         }
         case 'filters': {
           delete $scope.agg.field;
+          delete $scope.agg.settings.nested.path;
           $scope.agg.query = '*';
           break;
         }
+        case 'nested': {
+          delete $scope.agg.field;
+          delete $scope.agg.query;
+          $scope.agg.settings.nested = {};
+          $scope.agg.settings.nested.path = 'select field (type: nested)';
+          $scope.agg.settings.nested.term = 'select nested term path';
+          $scope.agg.settings.nested.query = 'select query for Nested Term';
+          break;
+        }
         case 'geohash_grid': {
           $scope.agg.settings.precision = 3;
           break;
@@ -79,6 +91,12 @@ function (angular, _, queryDef) {
       var settings = $scope.agg.settings || {};
 
       switch($scope.agg.type) {
+        case 'nested': {
+          if (settingsLinkText === '') {
+            settingsLinkText = 'Options';
+          }
+          break;
+        }
         case 'terms': {
           settings.order = settings.order || "asc";
           settings.size = settings.size || "10";
@@ -170,6 +188,20 @@ function (angular, _, queryDef) {
       }
     };
 
+    $scope.getFieldsNestedPath = function() {
+      return $scope.getFields({$fieldType: 'nested'});
+    };
+
+    $scope.getFieldsNestedTerm = function() {
+      return $scope.getFields({$fieldType: 'string'});
+    };
+
+    // FIX THIS: add a method getNestedKeys to be called here
+    // for getting nested key string ids, based on term/path supplied.
+    // $scope.getFieldsNestedQuery = function() {
+    //   return $scope.getNestedKeys();
+    // };
+
     $scope.getIntervalOptions = function() {
       return $q.when(uiSegmentSrv.transformToSegments(true, 'interval')(queryDef.intervalOptions));
     };

public/app/plugins/datasource/elasticsearch/datasource.js

@@ -290,9 +290,10 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
         }
 
         // transform to array
-        return _.map(fields, function(value) {
+        var resp = _.map(fields, function(value) {
           return value;
         });
+        return resp;
       });
     };
 

public/app/plugins/datasource/elasticsearch/elastic_response.js

@@ -97,6 +97,33 @@ function (_, queryDef) {
     }
   };
 
+  ElasticResponse.prototype.processNestedAggregationDocs = function(esAgg, aggDef, target, seriesList, props) {
+    var metric, y, i, newSeries, bucket, value;
+
+    for (y = 0; y < target.metrics.length; y++) {
+      metric = target.metrics[y];
+      if (metric.hide) {
+        continue;
+      }
+
+      newSeries = { datapoints: [], metric: metric.type, field: metric.field, props: props};
+      for (i = 0; i < esAgg.buckets.length; i++) {
+        bucket = esAgg.buckets[i][aggDef.id]['nested_aggs'];
+        if (bucket !== undefined) {
+          value = bucket[metric.id];
+          if (value !== undefined) {
+            if (value.normalized_value) {
+              newSeries.datapoints.push([value.normalized_value, esAgg.buckets[i].key]);
+            } else {
+              newSeries.datapoints.push([value.value, esAgg.buckets[i].key]);
+            }
+          }
+        }
+      }
+      seriesList.push(newSeries);
+    }
+  };
+
   ElasticResponse.prototype.processAggregationDocs = function(esAgg, aggDef, target, docs, props) {
     var metric, y, i, bucket, metricName, doc;
 
@@ -145,31 +172,39 @@ function (_, queryDef) {
   // This is quite complex
   // neeed to recurise down the nested buckets to build series
   ElasticResponse.prototype.processBuckets = function(aggs, target, seriesList, docs, props, depth) {
-    var bucket, aggDef, esAgg, aggId;
+    var bucket, aggDef, aggDefNested, esAgg, aggId;
     var maxDepth = target.bucketAggs.length-1;
 
+    aggDefNested = _.find(target.bucketAggs, {type: "nested"});
+
     for (aggId in aggs) {
-      aggDef = _.find(target.bucketAggs, {id: aggId});
       esAgg = aggs[aggId];
 
-      if (!aggDef) {
-        continue;
-      }
+      if (aggDefNested) {
+        this.processNestedAggregationDocs(esAgg, aggDefNested, target, seriesList, props);
+      } else {
+        aggDef = _.find(target.bucketAggs, {id: aggId});
 
-      if (depth === maxDepth) {
-        if (aggDef.type === 'date_histogram')  {
-          this.processMetrics(esAgg, target, seriesList, props);
-        } else {
-          this.processAggregationDocs(esAgg, aggDef, target, docs, props);
+        if (!aggDef) {
+          continue;
         }
-      } else {
-        for (var nameIndex in esAgg.buckets) {
-          bucket = esAgg.buckets[nameIndex];
-          props = _.clone(props);
-          if (bucket.key !== void 0) {
-            props[aggDef.field] = bucket.key;
+
+        if (depth === maxDepth) {
+          if (aggDef.type === 'date_histogram')  {
+            this.processMetrics(esAgg, target, seriesList, props);
           } else {
-            props["filter"] = nameIndex;
+            this.processAggregationDocs(esAgg, aggDef, target, docs, props);
+          }
+        } else {
+          for (var nameIndex in esAgg.buckets) {
+            bucket = esAgg.buckets[nameIndex];
+            props = _.clone(props);
+            if (bucket.key) {
+              props[aggDef.field] = bucket.key;
+            } else {
+              props["filter"] = nameIndex;
+            }
+            this.processBuckets(bucket, target, seriesList, docs, props, depth+1);
           }
           if (bucket.key_as_string) {
             props[aggDef.field] = bucket.key_as_string;
@@ -293,7 +328,7 @@ function (_, queryDef) {
     if (err.root_cause && err.root_cause.length > 0 && err.root_cause[0].reason) {
       result.message = err.root_cause[0].reason;
     } else {
-      result.message = err.reason || 'Unkown elatic error response';
+      result.message = err.reason || 'Unknown elastic error response';
     }
 
     if (response.$$config) {
@@ -325,7 +360,6 @@ function (_, queryDef) {
         this.processBuckets(aggregations, target, tmpSeriesList, docs, {}, 0);
         this.trimDatapoints(tmpSeriesList, target);
         this.nameSeries(tmpSeriesList, target);
-
         for (var y = 0; y < tmpSeriesList.length; y++) {
           seriesList.push(tmpSeriesList[y]);
         }

public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html

@@ -7,6 +7,7 @@
 
 		<metric-segment-model property="agg.type" options="bucketAggTypes" on-change="onTypeChanged()" custom="false" css-class="width-10"></metric-segment-model>
 		<metric-segment-model ng-if="agg.field" property="agg.field" get-options="getFieldsInternal()" on-change="onChange()" css-class="width-12"></metric-segment-model>
+		<metric-segment-model ng-if="agg.settings.nested.path" property="agg.settings.nested.path" get-options="getFieldsNestedPath()" on-change="onChange()" css-class="width-12"></metric-segment-model>
 	</div>
 
 	<div class="gf-form gf-form--grow">
@@ -52,6 +53,18 @@
 		</div>
 	</div>
 
+	<div ng-if="agg.type === 'nested'">
+		<div class="gf-form offset-width-7">
+			<label class="gf-form-label width-10">Nested Term</label>
+			<metric-segment-model property="agg.settings.nested.term" get-options="getFieldsNestedTerm()" on-change="onChangeInternal()" css-class="width-12" custom="true"></metric-segment-model>
+		</div>
+
+		<div class="gf-form offset-width-7">
+			<label class="gf-form-label width-10">Query</label>
+			<input type="text" class="gf-form-input max-width-12" ng-model="agg.settings.nested.query" spellcheck='false' placeholder="Nested string match query" ng-blur="onChangeInternal()">
+		</div>
+	</div>
+
 	<div ng-if="agg.type === 'terms'">
 		<div class="gf-form offset-width-7">
 			<label class="gf-form-label width-10">Order</label>

public/app/plugins/datasource/elasticsearch/query_builder.js

@@ -59,6 +59,33 @@ function (queryDef) {
     return queryNode;
   };
 
+  ElasticQueryBuilder.prototype.buildNestedAgg = function(aggDef, queryNode, target) {
+    var metric, y;
+    if (!aggDef.settings) {
+      return queryNode;
+    }
+    var settings = aggDef.settings;
+    queryNode.nested = {};
+    queryNode.nested.path = settings.nested.path;
+
+    queryNode.aggs = {
+      nested_aggs: {
+        filter: {},
+        aggs: {}
+      }
+    };
+    queryNode.aggs.nested_aggs.filter.term = {};
+    queryNode.aggs.nested_aggs.filter.term[settings.nested.term] = settings.nested.query;
+
+    queryNode.aggs.nested_aggs.aggs = {};
+    for (y = 0; y < target.metrics.length; y++) {
+      metric = target.metrics[y];
+      queryNode.aggs.nested_aggs.aggs[metric.id] = {};
+      queryNode.aggs.nested_aggs.aggs[metric.id][metric.type] = {field: metric.field};
+    }
+    return queryNode;
+  };
+
   ElasticQueryBuilder.prototype.getDateHistogramAgg = function(aggDef) {
     var esAgg = {};
     var settings = aggDef.settings || {};
@@ -183,6 +210,7 @@ function (queryDef) {
 
     nestedAggs = query;
 
+    var foundNested = false;
     for (i = 0; i < target.bucketAggs.length; i++) {
       var aggDef = target.bucketAggs[i];
       var esAgg = {};
@@ -204,6 +232,10 @@ function (queryDef) {
           esAgg['geohash_grid'] = {field: aggDef.field, precision: aggDef.settings.precision};
           break;
         }
+        case 'nested': {
+          foundNested = true;
+          this.buildNestedAgg(aggDef, esAgg, target);
+        }
       }
 
       nestedAggs.aggs = nestedAggs.aggs || {};
@@ -211,35 +243,37 @@ function (queryDef) {
       nestedAggs = esAgg;
     }
 
-    nestedAggs.aggs = {};
+    if (!foundNested) {
+      nestedAggs.aggs = {};
 
-    for (i = 0; i < target.metrics.length; i++) {
-      metric = target.metrics[i];
-      if (metric.type === 'count') {
-        continue;
-      }
+      for (i = 0; i < target.metrics.length; i++) {
+        metric = target.metrics[i];
+        if (metric.type === 'count') {
+          continue;
+        }
 
-      var aggField = {};
-      var metricAgg = null;
+        var aggField = {};
+        var metricAgg = null;
 
-      if (queryDef.isPipelineAgg(metric.type)) {
-        if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) {
-          metricAgg = { buckets_path: metric.pipelineAgg };
+        if (queryDef.isPipelineAgg(metric.type)) {
+          if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) {
+            metricAgg = { buckets_path: metric.pipelineAgg };
+          } else {
+            continue;
+          }
         } else {
-          continue;
+          metricAgg = {field: metric.field};
         }
-      } else {
-        metricAgg = {field: metric.field};
-      }
 
-      for (var prop in metric.settings) {
-        if (metric.settings.hasOwnProperty(prop) && metric.settings[prop] !== null) {
-          metricAgg[prop] = metric.settings[prop];
+        for (var prop in metric.settings) {
+          if (metric.settings.hasOwnProperty(prop) && metric.settings[prop] !== null) {
+            metricAgg[prop] = metric.settings[prop];
+          }
         }
-      }
 
-      aggField[metric.type] = metricAgg;
-      nestedAggs.aggs[metric.id] = aggField;
+        aggField[metric.type] = metricAgg;
+        nestedAggs.aggs[metric.id] = aggField;
+      }
     }
 
     return query;

public/app/plugins/datasource/elasticsearch/query_def.js

@@ -22,6 +22,7 @@ function (_) {
     bucketAggTypes: [
       {text: "Terms",           value: 'terms', requiresField: true},
       {text: "Filters",         value: 'filters' },
+      {text: "Nested",          value: 'nested', requiresField: true},
       {text: "Geo Hash Grid",   value: 'geohash_grid', requiresField: true},
       {text: "Date Histogram",  value: 'date_histogram', requiresField: true},
     ],

public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts

@@ -102,6 +102,37 @@ describe('ElasticQueryBuilder', function() {
     expect(firstLevel.aggs["1"].percentiles.percents).to.eql([1,2,3,4]);
   });
 
+  it('with nested aggs', function(){
+    var query = builder.build({
+      metrics: [{type: 'avg', field: 'disk.wr_sec', id: '1'}],
+      bucketAggs: [
+        {type: 'date_histogram', field: '@timestamp', id: '2'},
+        {
+          id: '3',
+          type: 'nested',
+          settings: {
+            nested: {
+              path: 'disk' ,
+              term: 'disk.disk-device',
+              query: 'dev8-0',
+            }
+          }
+        }
+      ],
+    });
+
+    var firstLevel = query.aggs["2"];
+    var secondLevel = firstLevel.aggs["3"];
+    var thirdLevel = secondLevel.aggs["nested_aggs"];
+    var fourthLevel = thirdLevel.aggs["1"];
+
+    expect(firstLevel.date_histogram.field).to.be("@timestamp");
+    expect(secondLevel.nested.path).to.be("disk");
+    expect(Object.keys(thirdLevel.filter.term)[0]).to.be("disk.disk-device");
+    expect(thirdLevel.filter.term["disk.disk-device"]).to.be("dev8-0");
+    expect(fourthLevel.avg.field).to.be("disk.wr_sec");
+  });
+
   it('with filters aggs', function() {
     var query = builder.build({
       metrics: [{type: 'count', id: '1'}],