Adding derived type query

src/app/components/kbn.js

@@ -460,7 +460,8 @@ function($, _, moment) {
   };
 
   kbn.colorSteps = function(col,steps) {
-    var _d = 1.6/steps, // distance between steps
+
+    var _d = steps > 5 ? 1.6/steps : 0.3, // distance between steps
       _p = []; // adjustment percentage
 
     // Create a range of numbers between -0.8 and 0.8

src/app/panels/histogram/module.js

@@ -190,14 +190,17 @@ function (angular, app, $, _, kbn, moment, timeSeries) {
       var request = $scope.ejs.Request().indices(dashboard.indices[segment]);
 
       $scope.panel.queries.ids = querySrv.idsByMode($scope.panel.queries);
+
+      var queries = querySrv.getQueryObjs($scope.panel.queries.ids);
+
       // Build the query
-      _.each($scope.panel.queries.ids, function(id) {
+      _.each(queries, function(q) {
         var query = $scope.ejs.FilteredQuery(
-          querySrv.getEjsObj(id),
+          querySrv.toEjsObj(q),
           filterSrv.getBoolFilter(filterSrv.ids)
         );
 
-        var facet = $scope.ejs.DateHistogramFacet(id);
+        var facet = $scope.ejs.DateHistogramFacet(q.id);
 
         if($scope.panel.mode === 'count') {
           facet = facet.field($scope.panel.time_field);
@@ -220,6 +223,7 @@ function (angular, app, $, _, kbn, moment, timeSeries) {
 
       // Populate scope when we have results
       results.then(function(results) {
+
         $scope.panelMeta.loading = false;
         if(segment === 0) {
           $scope.hits = 0;
@@ -233,18 +237,15 @@ function (angular, app, $, _, kbn, moment, timeSeries) {
           return;
         }
 
-        // Convert facet ids to numbers
-        var facetIds = _.map(_.keys(results.facets),function(k){return parseInt(k, 10);});
-
         // Make sure we're still on the same query/queries
-        if($scope.query_id === query_id && _.difference(facetIds, $scope.panel.queries.ids).length === 0) {
+        if($scope.query_id === query_id) {
 
           var i = 0,
             time_series,
             hits;
 
-          _.each($scope.panel.queries.ids, function(id) {
-            var query_results = results.facets[id];
+          _.each(queries, function(q) {
+            var query_results = results.facets[q.id];
             // we need to initialize the data variable on the first run,
             // and when we are working on the first segment of the data.
             if(_.isUndefined($scope.data[i]) || segment === 0) {
@@ -267,7 +268,7 @@ function (angular, app, $, _, kbn, moment, timeSeries) {
               $scope.hits += entry.count; // Entire dataset level hits counter
             });
             $scope.data[i] = {
-              info: querySrv.list[id],
+              info: q,
               time_series: time_series,
               hits: hits
             };

src/app/panels/query/meta.html

@@ -9,20 +9,18 @@
     }
 
   </style>
-  <a class="close" ng-click="render();dismiss();" href="">×</a>
-  <label class="strong small ">Query Alias <button class="btn btn-mini" ng-class="{active:querySrv.list[id].pin}" ng-click="toggle_pin(id);dismiss();" class="pointer"><i class="icon-pushpin"></i></button></label>
-
   <form>
     <input class="input-large input-query-alias" type="text" ng-model="querySrv.list[id].alias" placeholder='Alias...' />
     <div>
       <i ng-repeat="color in querySrv.colors" class="pointer" ng-class="{'icon-circle-blank':querySrv.list[id].color == color,'icon-circle':querySrv.list[id].color != color}" ng-style="{color:color}" ng-click="querySrv.list[id].color = color;render();"> </i>
     </div>
   </form>
+  <select class="input-small" ng-model="querySrv.list[id].type">
+    <option ng-repeat="type in queryTypes|esVersion:'require'">{{type.name}}</option>
+  </select>
 
-  <span>
-    <label class="small">Query type</label>
-    <select ng-change="dismiss();" class="input-small" ng-model="querySrv.list[id].type">
-      <option ng-repeat="type in querySrv.queryTypes|esVersion:'require'">{{type.name}}</option>
-    </select>
-  </span>
+  <div class="pull-right">
+    <button class="btn" ng-class="{active:querySrv.list[id].pin}" ng-click="toggle_pin(id);dismiss();" class="pointer">Pin <i class="icon-pushpin"></i></button>
+    <button class="btn btn-success " ng-click="dashboard.refresh();dismiss();">Apply</button>
+  </div>
 </div>

src/app/panels/query/module.html

@@ -2,7 +2,7 @@
   <div ng-repeat="id in (unPinnedQueries = (querySrv.ids|pinnedQuery:false))" ng-class="{'short-query': unPinnedQueries.length>1}">
     <form class="form-search" style="position:relative;margin-bottom:5px;" ng-submit="refresh()">
       <span class="begin-query">
-        <i class="icon-circle pointer" data-unique="1" bs-popover="'app/panels/query/meta.html'" data-placement="bottomLeft" ng-style="{color: querySrv.list[id].color}"></i>
+        <i class="pointer" ng-class="queryIcon(querySrv.list[id].type)" data-unique="1" bs-popover="'app/panels/query/meta.html'" data-placement="bottomLeft" ng-style="{color: querySrv.list[id].color}"></i>
         <i class="icon-remove-sign pointer remove-query" ng-show="querySrv.ids.length > 1" ng-click="querySrv.remove(id);refresh()"></i>
       </span>
       <span>

src/app/panels/query/module.js

@@ -19,7 +19,7 @@ define([
   var module = angular.module('kibana.panels.query', []);
   app.useModule(module);
 
-  module.controller('query', function($scope, querySrv, $rootScope) {
+  module.controller('query', function($scope, querySrv, $rootScope, dashboard) {
     $scope.panelMeta = {
       status  : "Stable",
       description : "Manage all of the queries on the dashboard. You almost certainly need one of "+
@@ -37,12 +37,20 @@ define([
 
     $scope.querySrv = querySrv;
 
+    // A list of query types for the query config popover
+    $scope.queryTypes = _.map(querySrv.queryTypes, function(v,k) {
+      return {
+        name:k,
+        require:v.require
+      };
+    });
+
     $scope.init = function() {
     };
 
     $scope.refresh = function() {
       update_history(_.pluck($scope.querySrv.list,'query'));
-      $rootScope.$broadcast('refresh');
+      dashboard.refresh();
     };
 
     $scope.render = function() {
@@ -53,6 +61,10 @@ define([
       querySrv.list[id].pin = querySrv.list[id].pin ? false : true;
     };
 
+    $scope.queryIcon = function(type) {
+      return querySrv.queryTypes[type].icon;
+    };
+
     var update_history = function(query) {
       if($scope.panel.remember > 0) {
         $scope.panel.history = _.union(query.reverse(),$scope.panel.history);

src/app/partials/querySelect.html

@@ -16,7 +16,7 @@
     </div>
     <div class="span9 querySelect" ng-show="panel.queries.mode == 'selected'">
       <label class="small">Selected Queries</label>
-      <span ng-style="{'border-color': querySrv.list[id].color}" ng-class="{selected:_.contains(panel.queries.ids,id),unselected:!_.contains(panel.queries.ids,id)}" ng-repeat="id in querySrv.ids" ng-click="panel.queries.ids = _.toggleInOut(panel.queries.ids,id);set_refresh(true);" class="query pointer badge">
+      <span ng-style="{'border-color': querySrv.list[id].color}" ng-class="{selected:_.contains(panel.queries.ids,id),unselected:!_.contains(panel.queries.ids,id)}" ng-repeat="id in querySrv.getIds()" ng-click="panel.queries.ids = _.toggleInOut(panel.queries.ids,id);set_refresh(true);" class="query pointer badge">
         <i class="icon-circle" ng-style="{color: querySrv.list[id].color}"></i>
         <span> {{querySrv.list[id].alias || querySrv.list[id].query}}</span>
       </span>

src/app/services/dashboard.js

@@ -123,7 +123,6 @@ function (angular, $, kbn, _, config, moment, Modernizr) {
     // Since the dashboard is responsible for index computation, we can compute and assign the indices
     // here before telling the panels to refresh
     this.refresh = function() {
-
       if(self.current.index.interval !== 'none') {
         if(filterSrv.idsByType('time').length > 0) {
           var _range = filterSrv.timeRange('last');
@@ -144,12 +143,13 @@ function (angular, $, kbn, _, config, moment, Modernizr) {
                 return false;
               }
             }
-            $rootScope.$broadcast('refresh');
+            // Don't resolve queries until indices are updated
+            querySrv.resolve().then(function(){$rootScope.$broadcast('refresh');});
           });
         } else {
           if(self.current.failover) {
             self.indices = [self.current.index.default];
-            $rootScope.$broadcast('refresh');
+            querySrv.resolve().then(function(){$rootScope.$broadcast('refresh');});
           } else {
             alertSrv.set("No time filter",
               'Timestamped indices are configured without a failover. Waiting for time filter.',
@@ -158,7 +158,7 @@ function (angular, $, kbn, _, config, moment, Modernizr) {
         }
       } else {
         self.indices = [self.current.index.default];
-        $rootScope.$broadcast('refresh');
+        querySrv.resolve().then(function(){$rootScope.$broadcast('refresh');});
       }
     };
 

src/app/services/querySrv.js

@@ -1,14 +1,16 @@
 define([
   'angular',
   'underscore',
-  'config'
+  'config',
+  'kbn'
 ],
-function (angular, _, config) {
+function (angular, _, config, kbn) {
   'use strict';
 
   var module = angular.module('kibana.services');
 
-  module.service('querySrv', function(dashboard, ejsResource) {
+  module.service('querySrv', function(dashboard, ejsResource, filterSrv, $q) {
+
     // Create an object to hold our service state on the dashboard
     dashboard.current.services.query = dashboard.current.services.query || {};
     _.defaults(dashboard.current.services.query,{
@@ -17,6 +19,23 @@ function (angular, _, config) {
       ids : [],
     });
 
+    this.colors = [
+      "#7EB26D","#EAB839","#6ED0E0","#EF843C","#E24D42","#1F78C1","#BA43A9","#705DA0", //1
+      "#508642","#CCA300","#447EBC","#C15C17","#890F02","#0A437C","#6D1F62","#584477", //2
+      "#B7DBAB","#F4D598","#70DBED","#F9BA8F","#F29191","#82B5D8","#E5A8E2","#AEA2E0", //3
+      "#629E51","#E5AC0E","#64B0C8","#E0752D","#BF1B00","#0A50A1","#962D82","#614D93", //4
+      "#9AC48A","#F2C96D","#65C5DB","#F9934E","#EA6460","#5195CE","#D683CE","#806EB7", //5
+      "#3F6833","#967302","#2F575E","#99440A","#58140C","#052B51","#511749","#3F2B5B", //6
+      "#E0F9D7","#FCEACA","#CFFAFF","#F9E2D2","#FCE2DE","#BADFF4","#F9D9F9","#DEDAF7"  //7
+    ];
+
+    // For convenience
+    var ejs = ejsResource(config.elasticsearch);
+    var _q = dashboard.current.services.query;
+
+    // Holds all actual queries, including all resolved abstract queries
+    var resolvedQueries = [];
+
     // Defaults for generic query object
     var _query = {
       alias: '',
@@ -39,40 +58,77 @@ function (angular, _, config) {
       }
     };
 
-    // For convenience
-    var ejs = ejsResource(config.elasticsearch);
-    var _q = dashboard.current.services.query;
-
-    this.colors = [
-      "#7EB26D","#EAB839","#6ED0E0","#EF843C","#E24D42","#1F78C1","#BA43A9","#705DA0", //1
-      "#508642","#CCA300","#447EBC","#C15C17","#890F02","#0A437C","#6D1F62","#584477", //2
-      "#B7DBAB","#F4D598","#70DBED","#F9BA8F","#F29191","#82B5D8","#E5A8E2","#AEA2E0", //3
-      "#629E51","#E5AC0E","#64B0C8","#E0752D","#BF1B00","#0A50A1","#962D82","#614D93", //4
-      "#9AC48A","#F2C96D","#65C5DB","#F9934E","#EA6460","#5195CE","#D683CE","#806EB7", //5
-      "#3F6833","#967302","#2F575E","#99440A","#58140C","#052B51","#511749","#3F2B5B", //6
-      "#E0F9D7","#FCEACA","#CFFAFF","#F9E2D2","#FCE2DE","#BADFF4","#F9D9F9","#DEDAF7"  //7
-    ];
+    // query type meta data that is not stored on the dashboard object
+    this.queryTypes = {
+      lucene: {
+        require:">=0.17.0",
+        icon: "icon-circle",
+        resolve: function(query) {
+          // Simply returns itself
+          var p = $q.defer();
+          p.resolve(_.extend(query,{parent:query.id}));
+          return p.promise;
+        }
+      },
+      regex: {
+        require:">=0.90.3",
+        icon: "icon-circle",
+        resolve: function(query) {
+          // Simply returns itself
+          var p = $q.defer();
+          p.resolve(_.extend(query,{parent:query.id}));
+          return p.promise;
+        }
+      }
+    };
 
-    // Define the query types and the version of elasticsearch they were first available in
-    this.queryTypes = [
-      {name:'lucene',require:">=0.17.0"},
-      {name:'regex',require:">=0.90.3"},
-      {name:'derive',require:">=2.0.0"}
-    ];
+    this.queryTypes.derive = {
+      require:">=0.90.3",
+      icon: "icon-cog",
+      resolve: function(q) {
+        var request = ejs.Request().indices(dashboard.indices);
+        var field = "extension";
+        // Terms mode
+        request = request
+          .facet(ejs.TermsFacet('query')
+            .field(field)
+            .size(10)
+            .facetFilter(ejs.QueryFilter(
+              ejs.FilteredQuery(
+                ejs.QueryStringQuery(q.query || '*'),
+                filterSrv.getBoolFilter(filterSrv.ids)
+                )))).size(0);
 
+        var results = request.doSearch();
+        return results.then(function(data) {
+          var _colors = kbn.colorSteps(q.color,data.facets.query.terms.length);
+          var i = -1;
+          return _.map(data.facets.query.terms,function(t) {
+            ++i;
+            return self.defaults({
+              query  : field+":"+t.term+" AND ("+q.query+")",
+              alias  : t.term + (q.alias ? " ("+q.alias+")" : ""),
+              type   : 'lucene',
+              color  : _colors[i],
+              parent : q.id
+            });
+          });
+        });
+      }
+    };
 
     // Save a reference to this
     var self = this;
 
     this.init = function() {
       _q = dashboard.current.services.query;
+
       self.list = dashboard.current.services.query.list;
       self.ids = dashboard.current.services.query.ids;
 
       // Check each query object, populate its defaults
-      _.each(self.list,function(query,id) {
-        _.defaults(query,_query);
-        query.color = query.color || colorAt(id);
+      _.each(self.list,function(query) {
+        query = self.defaults(query);
       });
 
       if (self.ids.length === 0) {
@@ -90,18 +146,23 @@ function (angular, _, config) {
           return false;
         }
       } else {
-        var _id = query.id || nextId();
-        query.id = _id;
-        query.color = query.color || colorAt(_id);
-        _.defaults(query,_query);
-        _.defaults(query,_dTypes[query.type]);
-
-        self.list[_id] = query;
-        self.ids.push(_id);
-        return _id;
+        // Query must have an id and color already
+        query.id = _.isUndefined(query.id) ? nextId() : query.id;
+        query.color = query.color || colorAt(query.id);
+        // Then it can get defaults
+        query = self.defaults(query);
+        self.list[query.id] = query;
+        self.ids.push(query.id);
+        return query.id;
       }
     };
 
+    this.defaults = function(query) {
+      _.defaults(query,_query);
+      _.defaults(query,_dTypes[query.type]);
+      return query;
+    };
+
     this.remove = function(id) {
       if(!_.isUndefined(self.list[id])) {
         delete self.list[id];
@@ -117,11 +178,6 @@ function (angular, _, config) {
       }
     };
 
-    // This must return an array to correctly resolve compound query types, eg derived
-    this.getEjsObj = function(ids) {
-      return self.toEjsObj(self.list[ids]);
-    };
-
     // In the case of a compound query, such as a derived query, we'd need to
     // return an array of elasticJS objects. Not sure if that is appropriate?
     this.toEjsObj = function (q) {
@@ -132,14 +188,22 @@ function (angular, _, config) {
       case 'regex':
         return ejs.RegexpQuery('_all',q.query);
       default:
-        return _.isUndefined(q.query) ? false : ejs.QueryStringQuery(q.query || '*');
+        return false;
       }
     };
 
-    this.findQuery = function(queryString) {
-      return _.findWhere(self.list,{query:queryString});
+    //
+    this.getQueryObjs = function(ids) {
+      if(_.isUndefined(ids)) {
+        return resolvedQueries;
+      } else {
+        return _.flatten(_.map(ids,function(id) {
+          return _.where(resolvedQueries,{parent:id});
+        }));
+      }
     };
 
+    // BROKEN
     this.idsByMode = function(config) {
       switch(config.mode)
       {
@@ -156,6 +220,25 @@ function (angular, _, config) {
       }
     };
 
+    // This populates the internal query list and returns a promise containing it
+    this.resolve = function() {
+      // Find ids of all abstract queries
+      console.log("keys: "+_.keys(self.list));
+      console.log("ids : "+_.pluck(self.list,'id'));
+      // Get a list of resolvable ids, constrast with total list to get abstract ones
+      return $q.all(_.map(self.ids,function(q) {
+        return self.queryTypes[self.list[q].type].resolve(_.clone(self.list[q])).then(function(data){
+          return data;
+        });
+      })).then(function(data) {
+        resolvedQueries = _.flatten(data);
+        _.each(resolvedQueries,function(q,i) {
+          q.id = i;
+        });
+        return resolvedQueries;
+      });
+    };
+
     var nextId = function() {
       if(_q.idQueue.length > 0) {
         return _q.idQueue.shift();