Elasticsearch: allow templating queries to order by doc_count (#18870)

* Elasticsearch Datasource: allow templating queries to order by doc_count

* Elasticsearch Datasource: add tests for doc_count templating queries

docs/sources/features/datasources/elasticsearch.md

@@ -143,6 +143,15 @@ You can use other variables inside the query. Example query definition for a var
 In the above example, we use another variable named `$source` inside the query definition. Whenever you change, via the dropdown, the current value of the ` $source` variable, it will trigger an update of the `$host` variable so it now only contains hostnames filtered by in this case the
 `@source` document property.
 
+These queries by default return results in term order (which can then be sorted alphabetically or numerically as for any variable).
+To produce a list of terms sorted by doc count (a top-N values list), add an `orderBy` property of "doc_count".
+This automatically selects a descending sort; using "asc" with doc_count (a bottom-N list) can be done by setting `order: "asc"` but [is discouraged](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html#search-aggregations-bucket-terms-aggregation-order) as it "increases the error on document counts".
+To keep terms in the doc count order, set the variable's Sort dropdown to **Disabled**; you might alternatively still want to use e.g. **Alphabetical** to re-sort them.
+
+```
+{"find": "terms", "field": "@hostname", "orderBy": "doc_count"}
+```
+
 ### Using variables in queries
 
 There are two syntaxes:

public/app/plugins/datasource/elasticsearch/query_builder.ts

@@ -353,17 +353,32 @@ export class ElasticQueryBuilder {
         terms: {
           field: queryDef.field,
           size: size,
-          order: {
-            _term: 'asc',
-          },
+          order: {},
         },
       },
     };
 
-    if (this.esVersion >= 60) {
-      query.aggs['1'].terms.order = {
-        _key: 'asc',
-      };
+    // Default behaviour is to order results by { _key: asc }
+    // queryDef.order allows selection of asc/desc
+    // queryDef.orderBy allows selection of doc_count ordering (defaults desc)
+
+    const { orderBy = 'key', order = orderBy === 'doc_count' ? 'desc' : 'asc' } = queryDef;
+
+    if (['asc', 'desc'].indexOf(order) < 0) {
+      throw { message: `Invalid query sort order ${order}` };
+    }
+
+    switch (orderBy) {
+      case 'key':
+      case 'term':
+        const keyname = this.esVersion >= 60 ? '_key' : '_term';
+        query.aggs['1'].terms.order[keyname] = order;
+        break;
+      case 'doc_count':
+        query.aggs['1'].terms.order['_count'] = order;
+        break;
+      default:
+        throw { message: `Invalid query sort type ${orderBy}` };
     }
 
     return query;

public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts

@@ -478,20 +478,84 @@ describe('ElasticQueryBuilder', () => {
     expect(query.query.bool.filter[5].bool.must_not.regexp['key6']).toBe('value6');
   });
 
-  it('getTermsQuery should set correct sorting', () => {
+  // terms query ES<6.0 - check ordering for _term and doc_type
+
+  it('getTermsQuery(default case) es<6.0 should set asc sorting on _term', () => {
     const query = builder.getTermsQuery({});
     expect(query.aggs['1'].terms.order._term).toBe('asc');
+    expect(query.aggs['1'].terms.order._key).toBeUndefined();
+    expect(query.aggs['1'].terms.order._count).toBeUndefined();
+  });
+
+  it('getTermsQuery(order:desc) es<6.0 should set desc sorting on _term', () => {
+    const query = builder.getTermsQuery({ order: 'desc' });
+    expect(query.aggs['1'].terms.order._term).toBe('desc');
+    expect(query.aggs['1'].terms.order._key).toBeUndefined();
+    expect(query.aggs['1'].terms.order._count).toBeUndefined();
+  });
+
+  it('getTermsQuery(orderBy:doc_count) es<6.0 should set desc sorting on _count', () => {
+    const query = builder.getTermsQuery({ orderBy: 'doc_count' });
+    expect(query.aggs['1'].terms.order._term).toBeUndefined();
+    expect(query.aggs['1'].terms.order._key).toBeUndefined();
+    expect(query.aggs['1'].terms.order._count).toBe('desc');
+  });
+
+  it('getTermsQuery(orderBy:doc_count, order:asc) es<6.0 should set asc sorting on _count', () => {
+    const query = builder.getTermsQuery({ orderBy: 'doc_count', order: 'asc' });
+    expect(query.aggs['1'].terms.order._term).toBeUndefined();
+    expect(query.aggs['1'].terms.order._key).toBeUndefined();
+    expect(query.aggs['1'].terms.order._count).toBe('asc');
   });
 
-  it('getTermsQuery es6.x should set correct sorting', () => {
+  // terms query ES>=6.0 - check ordering for _key and doc_type
+
+  it('getTermsQuery(default case) es6.x should set asc sorting on _key', () => {
     const builder6x = new ElasticQueryBuilder({
       timeField: '@timestamp',
       esVersion: 60,
     });
     const query = builder6x.getTermsQuery({});
+    expect(query.aggs['1'].terms.order._term).toBeUndefined();
     expect(query.aggs['1'].terms.order._key).toBe('asc');
+    expect(query.aggs['1'].terms.order._count).toBeUndefined();
+  });
+
+  it('getTermsQuery(order:desc) es6.x should set desc sorting on _key', () => {
+    const builder6x = new ElasticQueryBuilder({
+      timeField: '@timestamp',
+      esVersion: 60,
+    });
+    const query = builder6x.getTermsQuery({ order: 'desc' });
+    expect(query.aggs['1'].terms.order._term).toBeUndefined();
+    expect(query.aggs['1'].terms.order._key).toBe('desc');
+    expect(query.aggs['1'].terms.order._count).toBeUndefined();
   });
 
+  it('getTermsQuery(orderBy:doc_count) es6.x should set desc sorting on _count', () => {
+    const builder6x = new ElasticQueryBuilder({
+      timeField: '@timestamp',
+      esVersion: 60,
+    });
+    const query = builder6x.getTermsQuery({ orderBy: 'doc_count' });
+    expect(query.aggs['1'].terms.order._term).toBeUndefined();
+    expect(query.aggs['1'].terms.order._key).toBeUndefined();
+    expect(query.aggs['1'].terms.order._count).toBe('desc');
+  });
+
+  it('getTermsQuery(orderBy:doc_count, order:asc) es6.x should set asc sorting on _count', () => {
+    const builder6x = new ElasticQueryBuilder({
+      timeField: '@timestamp',
+      esVersion: 60,
+    });
+    const query = builder6x.getTermsQuery({ orderBy: 'doc_count', order: 'asc' });
+    expect(query.aggs['1'].terms.order._term).toBeUndefined();
+    expect(query.aggs['1'].terms.order._key).toBeUndefined();
+    expect(query.aggs['1'].terms.order._count).toBe('asc');
+  });
+
+  // Logs query
+
   it('getTermsQuery should request documents and date histogram', () => {
     const query = builder.getLogsQuery({});
     expect(query).toHaveProperty('query.bool.filter');