feat(elasticsearch): raw queries work, more unit tests and polish, #1034

public/app/panels/graph/module.js

@@ -153,8 +153,6 @@ function (angular, app, $, _, kbn, moment, TimeSeries, PanelMeta) {
         return;
       }
 
-      console.log('graph data', results);
-
       $scope.datapointsWarning = false;
       $scope.datapointsCount = 0;
       $scope.datapointsOutside = false;

public/app/plugins/datasource/elasticsearch/datasource.js

@@ -136,8 +136,8 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
 
     ElasticDatasource.prototype.query = function(options) {
       var queryBuilder = new ElasticQueryBuilder();
-      var header = '{"index":"' + this.index + '","search_type":"count","ignore_unavailable":true}'
-      var payload = ""
+      var header = '{"index":"' + this.index + '","search_type":"count","ignore_unavailable":true}';
+      var payload = "";
       var sentTargets = [];
       var timeFrom = this.translateTime(options.range.from);
       var timeTo = this.translateTime(options.range.to);
@@ -155,8 +155,8 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
       });
 
       payload = payload.replace(/\$interval/g, options.interval);
-      payload = payload.replace(/\$rangeFrom/g, this.translateTime(options.range.from));
-      payload = payload.replace(/\$rangeTo/g, this.translateTime(options.range.to));
+      payload = payload.replace(/\$timeFrom/g, this.translateTime(options.range.from));
+      payload = payload.replace(/\$timeTo/g, this.translateTime(options.range.to));
       payload = payload.replace(/\$maxDataPoints/g, options.maxDataPoints);
       payload = templateSrv.replace(payload, options.scopedVars);
 
@@ -175,22 +175,21 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
     // This is quite complex
     // neeed to recurise down the nested buckets to build series
     ElasticDatasource.prototype._processBuckets = function(buckets, target, series, level, parentName, parentTime) {
-      var points = [];
       var groupBy = target.groupByFields[level];
+      var seriesName, time, value, select, i, y, bucket;
 
-      for (var i = 0; i < buckets.length; i++) {
-        var bucket = buckets[i];
+      for (i = 0; i < buckets.length; i++) {
+        bucket = buckets[i];
 
         if (groupBy) {
-          var seriesName = level > 0 ? parentName + ' ' + bucket.key : parentName;
-          var time = parentTime || bucket.key;
-          this._processBuckets(bucket[groupBy.field].buckets, target, series, level+1, seriesName, time)
+          seriesName = level > 0 ? parentName + ' ' + bucket.key : parentName;
+          time = parentTime || bucket.key;
+          this._processBuckets(bucket[groupBy.field].buckets, target, series, level+1, seriesName, time);
         } else {
 
-          for (var y = 0; y < target.select.length; y++) {
-            var select = target.select[y];
-            var seriesName = parentName;
-            var value;
+          for (y = 0; y < target.select.length; y++) {
+            select = target.select[y];
+            seriesName = parentName;
 
             if (level > 0) {
               seriesName +=  ' ' + bucket.key;
@@ -224,20 +223,21 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
 
         var buckets = response.aggregations.histogram.buckets;
         var target = targets[i];
-        var points = [];
-        var querySeries = {}
+        var querySeries = {};
 
         this._processBuckets(buckets, target, querySeries, 0, target.refId);
 
-        _.each(querySeries, function(value) {
-          series.push(value);
-        });
-      };
+        for (var prop in querySeries) {
+          if (querySeries.hasOwnProperty(prop)) {
+            series.push(querySeries[prop]);
+          }
+        }
+      }
 
       return { data: series };
     };
 
-    ElasticDatasource.prototype.metricFindQuery = function(query) {
+    ElasticDatasource.prototype.metricFindQuery = function() {
       var timeFrom = this.translateTime(timeSrv.time.from);
       var timeTo = this.translateTime(timeSrv.time.to);
 
@@ -275,9 +275,9 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
           }
 
           if (hit._source) {
-            for (var field in hit._source) {
-              if (hit._source.hasOwnProperty(field)) {
-                fields[field] = 1;
+            for (var fieldProp in hit._source) {
+              if (hit._source.hasOwnProperty(fieldProp)) {
+                fields[fieldProp] = 1;
               }
             }
           }
@@ -285,16 +285,11 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
 
         fields = _.map(_.keys(fields), function(field) {
           return {text: field};
-        })
-        console.log('metricFindQuery:',  fields);
+        });
+
         return fields;
       });
-      // var d = $q.defer();
-      //
-      // var fieldsQuery = query.match(/^fields\(\)/);
-      // if (fieldsQuery) {
-      //   return d.promise;
-      // }
+
     };
 
     return ElasticDatasource;

public/app/plugins/datasource/elasticsearch/partials/query.editor.html

@@ -55,7 +55,7 @@
 	<div class="clearfix"></div>
 
 	<div style="padding: 10px" ng-if="target.rawQuery">
-		<textarea ng-model="target.rawJson" rows="8" spellcheck="false" style="width: 100%; box-sizing: border-box;"></textarea>
+		<textarea ng-model="target.rawQuery" rows="8" spellcheck="false" style="width: 100%; box-sizing: border-box;" ng-blur="queryUpdated()"></textarea>
 	</div>
 </div>
 

public/app/plugins/datasource/elasticsearch/queryBuilder.js

@@ -1,13 +1,14 @@
 define([
+  "angular"
 ],
-function () {
+function (angular) {
   'use strict';
 
   function ElasticQueryBuilder() { }
 
-  ElasticQueryBuilder.prototype.build = function(target, timeFrom, timeTo) {
+  ElasticQueryBuilder.prototype.build = function(target) {
     if (target.rawQuery) {
-      return angular.fromJson(target.rawJson);
+      return angular.fromJson(target.rawQuery);
     }
 
     var query = {
@@ -26,8 +27,8 @@ function () {
                 {
                   "range": {
                     "@timestamp": {
-                      "gte": timeFrom,
-                      "lte": timeTo
+                      "gte": "$timeFrom",
+                      "lte": "$timeTo"
                     }
                   }
                 }
@@ -48,17 +49,19 @@ function () {
           "field": target.timeField,
           "min_doc_count": 0,
           "extended_bounds": {
-            "min": timeFrom,
-            "max": timeTo
+            "min": "$timeFrom",
+            "max": "$timeTo"
           }
         }
       },
     };
 
     var nestedAggs = query.aggs.histogram;
+    var i;
+
     target.groupByFields = target.groupByFields || [];
 
-    for (var i = 0; i < target.groupByFields.length; i++) {
+    for (i = 0; i < target.groupByFields.length; i++) {
       var field = target.groupByFields[i].field;
       var aggs = {terms: {field: field}};
 
@@ -69,7 +72,7 @@ function () {
 
     nestedAggs.aggs = {};
 
-    for (var i = 0; i < target.select.length; i++) {
+    for (i = 0; i < target.select.length; i++) {
       var select = target.select[i];
       if (select.field) {
         var aggField = {};
@@ -79,7 +82,6 @@ function () {
       }
     }
 
-    console.log(angular.toJson(query, true));
     return query;
   };
 

public/app/plugins/datasource/elasticsearch/queryCtrl.js

@@ -10,17 +10,10 @@ function (angular, _, ElasticQueryBuilder) {
 
   module.controller('ElasticQueryCtrl', function($scope, $timeout, uiSegmentSrv, templateSrv, $q) {
 
-    $scope.functionList = ['count', 'min', 'max', 'total', 'mean'];
-
-    $scope.functionMenu = _.map($scope.functionList, function(func) {
-      return { text: func, click: "changeFunction('" + func + "');" };
-    });
-
     $scope.init = function() {
-      $scope.queryBuilder = new ElasticQueryBuilder(target);
-
       var target = $scope.target;
-      target.function = target.function || 'mean';
+      if (!target) { return; }
+
       target.timeField = target.timeField || '@timestamp';
       target.select = target.select || [{ agg: 'count' }];
       target.groupByFields = target.groupByFields || [];
@@ -36,6 +29,9 @@ function (angular, _, ElasticQueryBuilder) {
       $scope.removeSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove select --'});
       $scope.resetSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- reset --'});
       $scope.removeGroupBySegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove group by --'});
+
+      $scope.queryBuilder = new ElasticQueryBuilder(target);
+      $scope.rawQueryOld = angular.toJson($scope.queryBuilder.build($scope.target), true);
     };
 
     $scope.initSelectSegments = function() {
@@ -63,9 +59,11 @@ function (angular, _, ElasticQueryBuilder) {
           uiSegmentSrv.newSegment({value: 'max',   type: 'agg', reqField: true}),
           uiSegmentSrv.newSegment({value: 'avg',   type: 'agg', reqField: true}),
         ];
+        // if we have other selects and this is not a plus button add remove option
         if (segment.type !== 'plus-button' && $scope.selectSegments.length > 3) {
           options.splice(0, 0, angular.copy($scope.removeSelectSegment));
         }
+        // revert option is to reset the selectSegments if they become fucked
         if (index === 0 && $scope.selectSegments.length > 2) {
           options.splice(0, 0, angular.copy($scope.resetSelectSegment));
         }
@@ -82,20 +80,22 @@ function (angular, _, ElasticQueryBuilder) {
       if (segment.value === $scope.resetSelectSegment.value) {
         $scope.target.select = [{ agg: 'count' }];
         $scope.initSelectSegments();
-        $scope.get_data();
+        $scope.queryUpdated();
         return;
       }
 
+      var nextSegment, removeCount;
+
       // remove this select field
       if (segment.value === $scope.removeSelectSegment.value) {
-        var nextSegment = $scope.selectSegments[index + 1];
-        var remove = 2;
+        nextSegment = $scope.selectSegments[index + 1];
+        removeCount = 2;
         if (nextSegment && nextSegment.type === 'field') {
-          remove += 1;
+          removeCount += 1;
         }
-        $scope.selectSegments.splice(Math.max(index-1, 0), remove);
+        $scope.selectSegments.splice(Math.max(index-1, 0), removeCount);
         $scope.rebuildTargetSelects();
-        $scope.get_data();
+        $scope.queryUpdated();
         return;
       }
 
@@ -107,7 +107,7 @@ function (angular, _, ElasticQueryBuilder) {
       }
 
       if (segment.type === 'agg')  {
-        var nextSegment = $scope.selectSegments[index + 1];
+        nextSegment = $scope.selectSegments[index + 1];
 
         if (segment.value === 'count' && nextSegment && nextSegment.type === 'field') {
           $scope.selectSegments.splice(index + 1, 1);
@@ -121,7 +121,7 @@ function (angular, _, ElasticQueryBuilder) {
       }
 
       $scope.rebuildTargetSelects();
-      $scope.get_data();
+      $scope.queryUpdated();
     };
 
     $scope.rebuildTargetSelects = function() {
@@ -139,7 +139,7 @@ function (angular, _, ElasticQueryBuilder) {
 
         if (select.field === 'select field') { continue; }
         $scope.target.select.push(select);
-      };
+      }
     };
 
     $scope.getGroupByFields = function(segment) {
@@ -157,7 +157,7 @@ function (angular, _, ElasticQueryBuilder) {
       if (segment.value === $scope.removeGroupBySegment.value) {
         $scope.target.groupByFields.splice(index, 1);
         $scope.groupBySegments.splice(index, 1);
-        $scope.$parent.get_data();
+        $scope.queryUpdated();
         return;
       }
 
@@ -169,7 +169,15 @@ function (angular, _, ElasticQueryBuilder) {
       segment.fake = false;
 
       $scope.target.groupByFields[index] = {field: segment.value};
-      $scope.$parent.get_data();
+      $scope.queryUpdated();
+    };
+
+    $scope.queryUpdated = function() {
+      var newJson = angular.toJson($scope.queryBuilder.build($scope.target), true);
+      if (newJson !== $scope.oldQueryRaw) {
+        $scope.rawQueryOld = newJson;
+        $scope.get_data();
+      }
     };
 
     $scope.transformToSegments = function(addTemplateVars) {
@@ -194,7 +202,11 @@ function (angular, _, ElasticQueryBuilder) {
     };
 
     $scope.toggleQueryMode = function () {
-      $scope.target.rawQuery = !$scope.target.rawQuery;
+      if ($scope.target.rawQuery) {
+        delete $scope.target.rawQuery;
+      } else {
+        $scope.target.rawQuery = $scope.rawQueryOld;
+      }
     };
 
     $scope.init();

public/app/plugins/datasource/influxdb/partials/query.editor.html

@@ -34,9 +34,7 @@
 				{{target.refId}}
 			</li>
 			<li>
-				<a  class="tight-form-item"
-					ng-click="target.hide = !target.hide; get_data();"
-					role="menuitem">
+				<a class="tight-form-item" ng-click="target.hide = !target.hide; get_data();" role="menuitem">
 					<i class="fa fa-eye"></i>
 				</a>
 			</li>

public/test/specs/elasticsearch-querybuilder-specs.js

@@ -11,23 +11,25 @@ define([
       var query = builder.build({
         select: [{agg: 'Count'}],
         groupByFields: [],
-      }, 100, 1000);
+      });
 
-      expect(query.query.filtered.filter.bool.must[0].range["@timestamp"].gte).to.be(100);
-      expect(query.aggs.histogram.date_histogram.extended_bounds.min).to.be(100);
+      expect(query.query.filtered.filter.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom");
+      expect(query.aggs.histogram.date_histogram.extended_bounds.min).to.be("$timeFrom");
     });
 
     it('with select field', function() {
       var builder = new ElasticQueryBuilder();
 
       var query = builder.build({
-        select: [{agg: 'Avg', field: '@value'}],
+        select: [{agg: 'avg', field: '@value'}],
         groupByFields: [],
       }, 100, 1000);
 
-      var aggs = query.aggs.histogram;
+      var aggs = query.aggs.histogram.aggs;
+      expect(aggs["0"].avg.field).to.be("@value");
     });
 
+
   });
 
 });

public/test/specs/elasticsearch-queryctrl-specs.js

@@ -0,0 +1,53 @@
+define([
+  'helpers',
+  'plugins/datasource/elasticsearch/queryCtrl',
+  'services/uiSegmentSrv'
+], function(helpers) {
+  'use strict';
+
+  describe('ElasticQueryCtrl', function() {
+    var ctx = new helpers.ControllerTestContext();
+
+    beforeEach(module('grafana.controllers'));
+    beforeEach(module('grafana.services'));
+    beforeEach(ctx.providePhase());
+    beforeEach(ctx.createControllerPhase('ElasticQueryCtrl'));
+
+    beforeEach(function() {
+      ctx.scope.target = {};
+      ctx.scope.$parent = { get_data: sinon.spy() };
+
+      ctx.scope.datasource = ctx.datasource;
+      ctx.scope.datasource.metricFindQuery = sinon.stub().returns(ctx.$q.when([]));
+    });
+
+    describe('init', function() {
+      beforeEach(function() {
+        ctx.scope.init();
+      });
+
+      it('should init selectSegments', function() {
+        expect(ctx.scope.selectSegments.length).to.be(2);
+      });
+
+      describe('initSelectSegments with 2 selects', function() {
+
+        it('init selectSegments', function() {
+          ctx.scope.target.select = [
+            {agg: 'count'},
+            {agg: 'avg', field: 'value'},
+          ];
+          ctx.scope.initSelectSegments();
+
+          expect(ctx.scope.selectSegments.length).to.be(5);
+          expect(ctx.scope.selectSegments[0].value).to.be('count');
+          expect(ctx.scope.selectSegments[1].value).to.be(' and ');
+          expect(ctx.scope.selectSegments[2].value).to.be('avg');
+          expect(ctx.scope.selectSegments[3].value).to.be('value');
+        });
+      });
+
+    });
+
+  });
+});

public/test/specs/elasticsearch-specs.js

@@ -42,7 +42,7 @@ define([
                 }
               }
             }]
-          })
+          });
         });
 
         it('should return 1 series', function() {
@@ -81,7 +81,7 @@ define([
                 }
               }
             }]
-          })
+          });
         });
 
         it('should return 2 series', function() {
@@ -134,7 +134,7 @@ define([
                 }
               }
             }]
-          })
+          });
         });
 
         it('should return 2 series', function() {
@@ -216,7 +216,7 @@ define([
                 }
               }
             }]
-          })
+          });
         });
 
         it('should return 2 series', function() {

public/test/test-main.js

@@ -154,6 +154,7 @@ require([
     'specs/cloudwatch-datasource-specs',
     'specs/elasticsearch-specs',
     'specs/elasticsearch-querybuilder-specs',
+    'specs/elasticsearch-queryctrl-specs',
   ];
 
   var pluginSpecs = (config.plugins.specs || []).map(function (spec) {