Merge pull request #12167 from grafana/davkal/ifql-helpers

Query helpers, Annotations, and Template Variables for IFQL datasource

public/app/plugins/datasource/influxdb-ifql/README.md

@@ -14,13 +14,16 @@ Read more about InfluxDB here:
 
 [http://docs.grafana.org/datasources/influxdb/](http://docs.grafana.org/datasources/influxdb/)
 
+## Supported Template Variable Macros:
+
+* List all measurements for a given database: `measurements(database)`
+* List all tags for a given database and measurement: `tags(database, measurement)`
+* List all tag values for a given database, measurement, and tag: `tag_valuess(database, measurement, tag)`
+* List all field keys for a given database and measurement: `field_keys(database, measurement)`
+
 ## Roadmap
 
-- Sync Grafana time ranges with `range()`
-- Template variable expansion
 - Syntax highlighting
 - Tab completion (functions, values)
-- Result helpers (result counts, table previews)
-- Annotations support
 - Alerting integration
 - Explore UI integration

public/app/plugins/datasource/influxdb-ifql/datasource.ts

@@ -2,7 +2,14 @@ import _ from 'lodash';
 
 import * as dateMath from 'app/core/utils/datemath';
 
-import { getTableModelFromResult, getTimeSeriesFromResult, parseResults } from './response_parser';
+import {
+  getAnnotationsFromResult,
+  getTableModelFromResult,
+  getTimeSeriesFromResult,
+  getValuesFromResult,
+  parseResults,
+} from './response_parser';
+import expandMacros from './metric_find_query';
 
 function serializeParams(params) {
   if (!params) {
@@ -54,25 +61,21 @@ export default class InfluxDatasource {
     this.supportMetrics = true;
   }
 
-  prepareQueries(options) {
-    const targets = _.cloneDeep(options.targets);
+  prepareQueryTarget(target, options) {
+    // Replace grafana variables
     const timeFilter = this.getTimeFilter(options);
     options.scopedVars.range = { value: timeFilter };
-
-    // Filter empty queries and replace grafana variables
-    const queryTargets = targets.filter(t => t.query).map(t => {
-      const interpolated = this.templateSrv.replace(t.query, options.scopedVars);
-      return {
-        ...t,
-        query: interpolated,
-      };
-    });
-
-    return queryTargets;
+    const interpolated = this.templateSrv.replace(target.query, options.scopedVars);
+    return {
+      ...target,
+      query: interpolated,
+    };
   }
 
   query(options) {
-    const queryTargets = this.prepareQueries(options);
+    const queryTargets = options.targets
+      .filter(target => target.query)
+      .map(target => this.prepareQueryTarget(target, options));
     if (queryTargets.length === 0) {
       return Promise.resolve({ data: [] });
     }
@@ -81,13 +84,9 @@ export default class InfluxDatasource {
       const { query, resultFormat } = target;
 
       if (resultFormat === 'table') {
-        return (
-          this._seriesQuery(query, options)
-            .then(response => parseResults(response.data))
-            // Keep only first result from each request
-            .then(results => results[0])
-            .then(getTableModelFromResult)
-        );
+        return this._seriesQuery(query, options)
+          .then(response => parseResults(response.data))
+          .then(results => results.map(getTableModelFromResult));
       } else {
         return this._seriesQuery(query, options)
           .then(response => parseResults(response.data))
@@ -108,18 +107,42 @@ export default class InfluxDatasource {
       });
     }
 
-    var timeFilter = this.getTimeFilter({ rangeRaw: options.rangeRaw });
-    var query = options.annotation.query.replace('$timeFilter', timeFilter);
-    query = this.templateSrv.replace(query, null, 'regex');
+    const { query } = options.annotation;
+    const queryOptions = {
+      scopedVars: {},
+      ...options,
+      silent: true,
+    };
+    const target = this.prepareQueryTarget({ query }, queryOptions);
 
-    return {};
+    return this._seriesQuery(target.query, queryOptions).then(response => {
+      const results = parseResults(response.data);
+      if (results.length === 0) {
+        throw { message: 'No results in response from InfluxDB' };
+      }
+      const annotations = _.flatten(results.map(result => getAnnotationsFromResult(result, options.annotation)));
+      return annotations;
+    });
   }
 
   metricFindQuery(query: string, options?: any) {
-    // TODO not implemented
-    var interpolated = this.templateSrv.replace(query, null, 'regex');
-
-    return this._seriesQuery(interpolated, options).then(_.curry(parseResults)(query));
+    const interpreted = expandMacros(query);
+
+    // Use normal querier in silent mode
+    const queryOptions = {
+      rangeRaw: { to: 'now', from: 'now - 1h' },
+      scopedVars: {},
+      ...options,
+      silent: true,
+    };
+    const target = this.prepareQueryTarget({ query: interpreted }, queryOptions);
+    return this._seriesQuery(target.query, queryOptions).then(response => {
+      const results = parseResults(response.data);
+      const values = _.uniq(_.flatten(results.map(getValuesFromResult)));
+      return values
+        .filter(value => value && value[0] !== '_') // Ignore internal fields
+        .map(value => ({ text: value }));
+    });
   }
 
   _seriesQuery(query: string, options?: any) {

public/app/plugins/datasource/influxdb-ifql/metric_find_query.ts

@@ -0,0 +1,63 @@
+// MACROS
+
+// List all measurements for a given database: `measurements(database)`
+const MEASUREMENTS_REGEXP = /^\s*measurements\((.+)\)\s*$/;
+
+// List all tags for a given database and measurement: `tags(database, measurement)`
+const TAGS_REGEXP = /^\s*tags\((.+)\s*,\s*(.+)\)\s*$/;
+
+// List all tag values for a given database, measurement, and tag: `tag_valuess(database, measurement, tag)`
+const TAG_VALUES_REGEXP = /^\s*tag_values\((.+)\s*,\s*(.+)\s*,\s*(.+)\)\s*$/;
+
+// List all field keys for a given database and measurement: `field_keys(database, measurement)`
+const FIELD_KEYS_REGEXP = /^\s*field_keys\((.+)\s*,\s*(.+)\)\s*$/;
+
+export default function expandMacros(query) {
+  const measurementsQuery = query.match(MEASUREMENTS_REGEXP);
+  if (measurementsQuery) {
+    const database = measurementsQuery[1];
+    return `from(db:"${database}")
+    |> range($range)
+    |> group(by:["_measurement"])
+    |> distinct(column:"_measurement")
+    |> group(none:true)`;
+  }
+
+  const tagsQuery = query.match(TAGS_REGEXP);
+  if (tagsQuery) {
+    const database = tagsQuery[1];
+    const measurement = tagsQuery[2];
+    return `from(db:"${database}")
+    |> range($range)
+    |> filter(fn:(r) => r._measurement == "${measurement}")
+    |> keys()`;
+  }
+
+  const tagValuesQuery = query.match(TAG_VALUES_REGEXP);
+  if (tagValuesQuery) {
+    const database = tagValuesQuery[1];
+    const measurement = tagValuesQuery[2];
+    const tag = tagValuesQuery[3];
+    return `from(db:"${database}")
+    |> range($range)
+    |> filter(fn:(r) => r._measurement == "${measurement}")
+    |> group(by:["${tag}"])
+    |> distinct(column:"${tag}")
+    |> group(none:true)`;
+  }
+
+  const fieldKeysQuery = query.match(FIELD_KEYS_REGEXP);
+  if (fieldKeysQuery) {
+    const database = fieldKeysQuery[1];
+    const measurement = fieldKeysQuery[2];
+    return `from(db:"${database}")
+    |> range($range)
+    |> filter(fn:(r) => r._measurement == "${measurement}")
+    |> group(by:["_field"])
+    |> distinct(column:"_field")
+    |> group(none:true)`;
+  }
+
+  // By default return pure query
+  return query;
+}

public/app/plugins/datasource/influxdb-ifql/partials/annotations.editor.html

@@ -1,11 +1,13 @@
-
 <div class="gf-form-group">
 	<div class="gf-form">
-		<input type="text" class="gf-form-input" ng-model='ctrl.annotation.query' placeholder="select text from events where $timeFilter limit 1000"></input>
+		<input type="text" class="gf-form-input" ng-model='ctrl.annotation.query' placeholder='from(db:"telegraf") |> range($range)'></input>
 	</div>
 </div>
 
-<h5 class="section-heading">Field mappings <tip>If your influxdb query returns more than one field you need to specify the column names below. An annotation event is composed of a title, tags, and an additional text field.</tip></h5>
+<h5 class="section-heading">Field mappings
+	<tip>If your influxdb query returns more than one field you need to specify the column names below. An annotation event is composed
+		of a title, tags, and an additional text field.</tip>
+</h5>
 <div class="gf-form-group">
 	<div class="gf-form-inline">
 		<div class="gf-form">
@@ -16,9 +18,5 @@
 			<span class="gf-form-label width-4">Tags</span>
 			<input type="text" class="gf-form-input max-width-10" ng-model='ctrl.annotation.tagsColumn' placeholder=""></input>
 		</div>
-		<div class="gf-form" ng-show="ctrl.annotation.titleColumn">
-			<span class="gf-form-label width-4">Title <em class="muted">(deprecated)</em></span>
-			<input type="text" class="gf-form-input max-width-10" ng-model='ctrl.annotation.titleColumn' placeholder=""></input>
-		</div>
 	</div>
-</div>
+</div>

public/app/plugins/datasource/influxdb-ifql/partials/query.editor.html

@@ -1,8 +1,10 @@
 <query-editor-row query-ctrl="ctrl" can-collapse="true" has-text-edit-mode="true">
 
   <div class="gf-form">
-    <textarea rows="3" class="gf-form-input" ng-model="ctrl.target.query" spellcheck="false" placeholder="IFQL Query" ng-model-onblur
+    <textarea rows="10" class="gf-form-input" ng-model="ctrl.target.query" spellcheck="false" placeholder="IFQL Query" ng-model-onblur
       ng-change="ctrl.refresh()"></textarea>
+    <!-- Result preview -->
+    <textarea rows="10" class="gf-form-input" ng-model="ctrl.dataPreview" readonly></textarea>
   </div>
   <div class="gf-form-inline">
     <div class="gf-form">
@@ -12,9 +14,15 @@
           ng-change="ctrl.refresh()"></select>
       </div>
     </div>
-    <div class="gf-form max-width-25" ng-hide="ctrl.target.resultFormat === 'table'">
-      <label class="gf-form-label query-keyword">ALIAS BY</label>
-      <input type="text" class="gf-form-input" ng-model="ctrl.target.alias" spellcheck='false' placeholder="Naming pattern" ng-blur="ctrl.refresh()">
+    <div class="gf-form" ng-if="ctrl.panelCtrl.loading">
+      <label class="gf-form-label">
+        <i class="fa fa-spinner fa-spin"></i> Loading</label>
+    </div>
+    <div class="gf-form" ng-if="!ctrl.panelCtrl.loading">
+      <label class="gf-form-label">Result tables</label>
+      <input type="text" class="gf-form-input" ng-model="ctrl.resultTableCount" disabled="disabled">
+      <label class="gf-form-label">Result records</label>
+      <input type="text" class="gf-form-input" ng-model="ctrl.resultRecordCount" disabled="disabled">
     </div>
     <div class="gf-form gf-form--grow">
       <div class="gf-form-label gf-form-label--grow"></div>

public/app/plugins/datasource/influxdb-ifql/plugin.json

@@ -4,7 +4,7 @@
   "id": "influxdb-ifql",
   "defaultMatchFormat": "regex values",
   "metrics": true,
-  "annotations": false,
+  "annotations": true,
   "alerting": false,
   "queryOptions": {
     "minInterval": true

public/app/plugins/datasource/influxdb-ifql/query_ctrl.ts

@@ -1,3 +1,4 @@
+import appEvents from 'app/core/app_events';
 import { QueryCtrl } from 'app/plugins/sdk';
 
 function makeDefaultQuery(database) {
@@ -9,18 +10,46 @@ function makeDefaultQuery(database) {
 export class InfluxIfqlQueryCtrl extends QueryCtrl {
   static templateUrl = 'partials/query.editor.html';
 
+  dataPreview: string;
+  resultRecordCount: string;
+  resultTableCount: string;
   resultFormats: any[];
 
   /** @ngInject **/
   constructor($scope, $injector) {
     super($scope, $injector);
 
+    this.resultRecordCount = '';
+    this.resultTableCount = '';
+
     if (this.target.query === undefined) {
       this.target.query = makeDefaultQuery(this.datasource.database);
     }
     this.resultFormats = [{ text: 'Time series', value: 'time_series' }, { text: 'Table', value: 'table' }];
+
+    appEvents.on('ds-request-response', this.onResponseReceived, $scope);
+    this.panelCtrl.events.on('refresh', this.onRefresh, $scope);
+    this.panelCtrl.events.on('data-received', this.onDataReceived, $scope);
   }
 
+  onDataReceived = dataList => {
+    this.resultRecordCount = dataList.reduce((count, model) => {
+      const records = model.type === 'table' ? model.rows.length : model.datapoints.length;
+      return count + records;
+    }, 0);
+    this.resultTableCount = dataList.length;
+  };
+
+  onResponseReceived = response => {
+    this.dataPreview = response.data;
+  };
+
+  onRefresh = () => {
+    this.dataPreview = '';
+    this.resultRecordCount = '';
+    this.resultTableCount = '';
+  };
+
   getCollapsedText() {
     return this.target.query;
   }

public/app/plugins/datasource/influxdb-ifql/response_parser.ts

@@ -1,4 +1,5 @@
 import Papa from 'papaparse';
+import flatten from 'lodash/flatten';
 import groupBy from 'lodash/groupBy';
 
 import TableModel from 'app/core/table_model';
@@ -6,17 +7,25 @@ import TableModel from 'app/core/table_model';
 const filterColumnKeys = key => key && key[0] !== '_' && key !== 'result' && key !== 'table';
 
 const IGNORE_FIELDS_FOR_NAME = ['result', '', 'table'];
+
+export const getTagsFromRecord = record =>
+  Object.keys(record)
+    .filter(key => key[0] !== '_')
+    .filter(key => IGNORE_FIELDS_FOR_NAME.indexOf(key) === -1)
+    .reduce((tags, key) => {
+      tags[key] = record[key];
+      return tags;
+    }, {});
+
 export const getNameFromRecord = record => {
   // Measurement and field
   const metric = [record._measurement, record._field];
 
   // Add tags
-  const tags = Object.keys(record)
-    .filter(key => key[0] !== '_')
-    .filter(key => IGNORE_FIELDS_FOR_NAME.indexOf(key) === -1)
-    .map(key => `${key}=${record[key]}`);
+  const tags = getTagsFromRecord(record);
+  const tagValues = Object.keys(tags).map(key => `${key}=${tags[key]}`);
 
-  return [...metric, ...tags].join(' ');
+  return [...metric, ...tagValues].join(' ');
 };
 
 const parseCSV = (input: string) =>
@@ -36,6 +45,33 @@ export function parseResults(response: string): any[] {
   return response.trim().split(/\n\s*\s/);
 }
 
+export function getAnnotationsFromResult(result: string, options: any) {
+  const data = parseCSV(result);
+  if (data.length === 0) {
+    return [];
+  }
+
+  const annotations = [];
+  const textSelector = options.textCol || '_value';
+  const tagsSelector = options.tagsCol || '';
+  const tagSelection = tagsSelector.split(',').map(t => t.trim());
+
+  data.forEach(record => {
+    // Remove empty values, then split in different tags for comma separated values
+    const tags = getTagsFromRecord(record);
+    const tagValues = flatten(tagSelection.filter(tag => tags[tag]).map(tag => tags[tag].split(',')));
+
+    annotations.push({
+      annotation: options,
+      time: parseTime(record._time),
+      tags: tagValues,
+      text: record[textSelector],
+    });
+  });
+
+  return annotations;
+}
+
 export function getTableModelFromResult(result: string) {
   const data = parseCSV(result);
 
@@ -86,3 +122,8 @@ export function getTimeSeriesFromResult(result: string) {
 
   return seriesList;
 }
+
+export function getValuesFromResult(result: string) {
+  const data = parseCSV(result);
+  return data.map(record => record['_value']);
+}

public/app/plugins/datasource/influxdb-ifql/specs/datasource.jest.ts

@@ -13,41 +13,27 @@ describe('InfluxDB (IFQL)', () => {
     targets: [],
   };
 
-  let queries: any[];
-
-  describe('prepareQueries()', () => {
-    it('filters empty queries', () => {
-      queries = ds.prepareQueries(DEFAULT_OPTIONS);
-      expect(queries.length).toBe(0);
-
-      queries = ds.prepareQueries({
-        ...DEFAULT_OPTIONS,
-        targets: [{ query: '' }],
-      });
-      expect(queries.length).toBe(0);
-    });
+  describe('prepareQueryTarget()', () => {
+    let target: any;
 
     it('replaces $range variable', () => {
-      queries = ds.prepareQueries({
-        ...DEFAULT_OPTIONS,
-        targets: [{ query: 'from(db: "test") |> range($range)' }],
-      });
-      expect(queries.length).toBe(1);
-      expect(queries[0].query).toBe('from(db: "test") |> range(start: -3h)');
+      target = ds.prepareQueryTarget({ query: 'from(db: "test") |> range($range)' }, DEFAULT_OPTIONS);
+      expect(target.query).toBe('from(db: "test") |> range(start: -3h)');
     });
 
     it('replaces $range variable with custom dates', () => {
       const to = moment();
       const from = moment().subtract(1, 'hours');
-      queries = ds.prepareQueries({
-        ...DEFAULT_OPTIONS,
-        rangeRaw: { to, from },
-        targets: [{ query: 'from(db: "test") |> range($range)' }],
-      });
-      expect(queries.length).toBe(1);
+      target = ds.prepareQueryTarget(
+        { query: 'from(db: "test") |> range($range)' },
+        {
+          ...DEFAULT_OPTIONS,
+          rangeRaw: { to, from },
+        }
+      );
       const start = from.toISOString();
       const stop = to.toISOString();
-      expect(queries[0].query).toBe(`from(db: "test") |> range(start: ${start}, stop: ${stop})`);
+      expect(target.query).toBe(`from(db: "test") |> range(start: ${start}, stop: ${stop})`);
     });
   });
 });

public/app/plugins/datasource/influxdb-ifql/specs/metric_find_query.jest.ts

@@ -0,0 +1,43 @@
+import expandMacros from '../metric_find_query';
+
+describe('metric find query', () => {
+  describe('expandMacros()', () => {
+    it('returns a non-macro query unadulterated', () => {
+      const query = 'from(db:"telegraf") |> last()';
+      const result = expandMacros(query);
+      expect(result).toBe(query);
+    });
+
+    it('returns a measurement query for measurements()', () => {
+      const query = ' measurements(mydb) ';
+      const result = expandMacros(query).replace(/\s/g, '');
+      expect(result).toBe(
+        'from(db:"mydb")|>range($range)|>group(by:["_measurement"])|>distinct(column:"_measurement")|>group(none:true)'
+      );
+    });
+
+    it('returns a tags query for tags()', () => {
+      const query = ' tags(mydb , mymetric) ';
+      const result = expandMacros(query).replace(/\s/g, '');
+      expect(result).toBe('from(db:"mydb")|>range($range)|>filter(fn:(r)=>r._measurement=="mymetric")|>keys()');
+    });
+
+    it('returns a tag values query for tag_values()', () => {
+      const query = ' tag_values(mydb , mymetric, mytag) ';
+      const result = expandMacros(query).replace(/\s/g, '');
+      expect(result).toBe(
+        'from(db:"mydb")|>range($range)|>filter(fn:(r)=>r._measurement=="mymetric")' +
+          '|>group(by:["mytag"])|>distinct(column:"mytag")|>group(none:true)'
+      );
+    });
+
+    it('returns a field keys query for field_keys()', () => {
+      const query = ' field_keys(mydb , mymetric) ';
+      const result = expandMacros(query).replace(/\s/g, '');
+      expect(result).toBe(
+        'from(db:"mydb")|>range($range)|>filter(fn:(r)=>r._measurement=="mymetric")' +
+          '|>group(by:["_field"])|>distinct(column:"_field")|>group(none:true)'
+      );
+    });
+  });
+});

public/app/plugins/datasource/influxdb-ifql/specs/response_parser.jest.ts

@@ -1,7 +1,9 @@
 import {
+  getAnnotationsFromResult,
   getNameFromRecord,
   getTableModelFromResult,
   getTimeSeriesFromResult,
+  getValuesFromResult,
   parseResults,
   parseValue,
 } from '../response_parser';
@@ -15,6 +17,17 @@ describe('influxdb ifql response parser', () => {
     });
   });
 
+  describe('getAnnotationsFromResult()', () => {
+    it('expects a list of annotations', () => {
+      const results = parseResults(response);
+      const annotations = getAnnotationsFromResult(results[0], { tagsCol: 'cpu' });
+      expect(annotations.length).toBe(300);
+      expect(annotations[0].tags.length).toBe(1);
+      expect(annotations[0].tags[0]).toBe('cpu-total');
+      expect(annotations[0].text).toBe('0');
+    });
+  });
+
   describe('getTableModelFromResult()', () => {
     it('expects a table model', () => {
       const results = parseResults(response);
@@ -33,6 +46,14 @@ describe('influxdb ifql response parser', () => {
     });
   });
 
+  describe('getValuesFromResult()', () => {
+    it('returns all values from the _value field in the response', () => {
+      const results = parseResults(response);
+      const values = getValuesFromResult(results[0]);
+      expect(values.length).toBe(300);
+    });
+  });
+
   describe('getNameFromRecord()', () => {
     it('expects name based on measurements and tags', () => {
       const record = {