ES nested fields autocomplete (#6043)

* Re-create PR #4527 from @arcolife:
fixes #4526 - add nested support to fieldname autocomplete; Also:
    - update _mapping to first use given time range
      for deducting index mapping, then fallback to
      today's date based index name

* (elasticsearch): refactor getFields() method.

* (elasticsearch): add tests for getFields() method.

* (elasticsearch): fixed _get() method (tests was broken after @arcolife commit).

public/app/plugins/datasource/elasticsearch/datasource.js

@@ -47,10 +47,19 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
     };
 
     this._get = function(url) {
-      return this._request('GET', this.indexPattern.getIndexForToday() + url).then(function(results) {
-        results.data.$$config = results.config;
-        return results.data;
-      });
+      var range = timeSrv.timeRange();
+      var index_list = this.indexPattern.getIndexList(range.from.valueOf(), range.to.valueOf());
+      if (_.isArray(index_list) && index_list.length) {
+        return this._request('GET', index_list[0] + url).then(function(results) {
+          results.data.$$config = results.config;
+          return results.data;
+        });
+      } else {
+        return this._request('GET', this.indexPattern.getIndexForToday() + url).then(function(results) {
+          results.data.$$config = results.config;
+          return results.data;
+        });
+      }
     };
 
     this._post = function(url, data) {
@@ -210,8 +219,7 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
     }
 
     this.getFields = function(query) {
-      return this._get('/_mapping').then(function(res) {
-        var fields = {};
+      return this._get('/_mapping').then(function(result) {
         var typeMap = {
           'float': 'number',
           'double': 'number',
@@ -219,24 +227,47 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
           'long': 'number',
           'date': 'date',
           'string': 'string',
+          'nested': 'nested'
         };
 
-        for (var indexName in res) {
-          var index = res[indexName];
-          var mappings = index.mappings;
-          if (!mappings) { continue; }
-          for (var typeName in mappings) {
-            var properties = mappings[typeName].properties;
-            for (var field in properties) {
-              var prop = properties[field];
-              if (query.type && typeMap[prop.type] !== query.type) {
-                continue;
-              }
-              if (prop.type && field[0] !== '_') {
-                fields[field] = {text: field, type: prop.type};
+        // Store subfield names: [system, process, cpu, total] -> system.process.cpu.total
+        var fieldNameParts = [];
+        var fields = {};
+        function getFieldsRecursively(obj) {
+          for (var key in obj) {
+            var subObj = obj[key];
+
+            // Check mapping field for nested fields
+            if (subObj.hasOwnProperty('properties')) {
+              fieldNameParts.push(key);
+              getFieldsRecursively(subObj.properties);
+            } else {
+              var fieldName = fieldNameParts.concat(key).join('.');
+
+              // Hide meta-fields and check field type
+              if (key[0] !== '_' &&
+                  (!query.type ||
+                    query.type && typeMap[subObj.type] === query.type)) {
+
+                fields[fieldName] = {
+                  text: fieldName,
+                  type: subObj.type
+                };
               }
             }
           }
+          fieldNameParts.pop();
+        }
+
+        for (var indexName in result) {
+          var index = result[indexName];
+          if (index && index.mappings) {
+            var mappings = index.mappings;
+            for (var typeName in mappings) {
+              var properties = mappings[typeName].properties;
+              getFieldsRecursively(properties);
+            }
+          }
         }
 
         // transform to array

public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts

@@ -1,4 +1,4 @@
-
+import _ from 'lodash';
 import {describe, beforeEach, it, sinon, expect, angularMocks} from 'test/lib/common';
 import moment from 'moment';
 import angular from 'angular';
@@ -112,4 +112,101 @@ describe('ElasticDatasource', function() {
     });
   });
 
+  describe('When getting fields', function() {
+    var requestOptions, parts, header;
+
+    beforeEach(function() {
+      createDatasource({url: 'http://es.com', index: 'metricbeat'});
+
+      ctx.backendSrv.datasourceRequest = function(options) {
+        requestOptions = options;
+        return ctx.$q.when({data: {
+          metricbeat: {
+            mappings: {
+              metricsets: {
+                _all: {},
+                properties: {
+                  '@timestamp': {type: 'date'},
+                  beat: {
+                    properties: {
+                      name: {type: 'string'},
+                      hostname: {type: 'string'},
+                    }
+                  },
+                  system: {
+                    properties: {
+                      cpu: {
+                        properties: {
+                          system: {type: 'float'},
+                          user: {type: 'float'},
+                        }
+                      },
+                      process: {
+                        properties: {
+                          cpu: {
+                            properties: {
+                              total: {type: 'float'}
+                            }
+                          },
+                          name: {type: 'string'},
+                        }
+                      },
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }});
+      };
+    });
+
+    it('should return nested fields', function() {
+      ctx.ds.getFields({
+        find: 'fields',
+        query: '*'
+      }).then((fieldObjects) => {
+        var fields = _.map(fieldObjects, 'text');
+        expect(fields).to.eql([
+          '@timestamp',
+          'beat.name',
+          'beat.hostname',
+          'system.cpu.system',
+          'system.cpu.user',
+          'system.process.cpu.total',
+          'system.process.name'
+        ]);
+      });
+      ctx.$rootScope.$apply();
+    });
+
+    it('should return fields related to query type', function() {
+      ctx.ds.getFields({
+        find: 'fields',
+        query: '*',
+        type: 'number'
+      }).then((fieldObjects) => {
+        var fields = _.map(fieldObjects, 'text');
+        expect(fields).to.eql([
+          'system.cpu.system',
+          'system.cpu.user',
+          'system.process.cpu.total'
+        ]);
+      });
+
+      ctx.ds.getFields({
+        find: 'fields',
+        query: '*',
+        type: 'date'
+      }).then((fieldObjects) => {
+        var fields = _.map(fieldObjects, 'text');
+        expect(fields).to.eql([
+          '@timestamp'
+        ]);
+      });
+
+      ctx.$rootScope.$apply();
+    });
+  });
+
 });