Merge pull request #4138 from utkarshcmu/tsdb-refac2

Opentsdb 2.2 support without breaking 2.1

CHANGELOG.md

@@ -8,6 +8,7 @@
 * **Prometheus**: Prometheus annotation support, closes[#2883](https://github.com/grafana/grafana/pull/2883)
 * **Cli**: New cli tool for downloading and updating plugins
 * **Annotations**: Annotations can now contain links that can be clicked (you can navigate on to annotation popovers), closes [#1588](https://github.com/grafana/grafana/issues/1588)
+* **Opentsdb**: Opentsdb 2.2 filters support, closes[#3077](https://github.com/grafana/grafana/issues/3077)
 
 ### Breaking changes
 * **Plugin API**: Both datasource and panel plugin api (and plugin.json schema) have been updated, requiring an update to plugins. See [plugin api](https://github.com/grafana/grafana/blob/master/public/app/plugins/plugin_api.md) for more info.

docs/sources/datasources/opentsdb.md

@@ -23,6 +23,7 @@ Name | The data source name, important that this is the same as in Grafana v1.x
 Default | Default data source means that it will be pre-selected for new panels.
 Url | The http protocol, ip and port of you opentsdb server (default port is usually 4242)
 Access | Proxy = access via Grafana backend, Direct = access directory from browser.
+Version | Version = opentsdb version, either <=2.1 or 2.2
 
 ## Query editor
 Open a graph in edit mode by click the title.

public/app/plugins/datasource/opentsdb/config_ctrl.ts

@@ -0,0 +1,21 @@
+///<reference path="../../../headers/common.d.ts" />
+
+import angular from 'angular';
+import _ from 'lodash';
+
+export class OpenTsConfigCtrl {
+  static templateUrl = 'public/app/plugins/datasource/opentsdb/partials/config.html';
+  current: any;
+
+  /** @ngInject */
+  constructor($scope) {
+    this.current.jsonData = this.current.jsonData || {};
+    this.current.jsonData.tsdbVersion = this.current.jsonData.tsdbVersion || 1;
+  }
+
+  tsdbVersions = [
+    {name: '<=2.1', value: 1},
+    {name: '2.2', value: 2},
+  ];
+
+}

public/app/plugins/datasource/opentsdb/datasource.js

@@ -14,6 +14,8 @@ function (angular, _, dateMath) {
     this.name = instanceSettings.name;
     this.withCredentials = instanceSettings.withCredentials;
     this.basicAuth = instanceSettings.basicAuth;
+    instanceSettings.jsonData = instanceSettings.jsonData || {};
+    this.tsdbVersion = instanceSettings.jsonData.tsdbVersion || 1;
     this.supportMetrics = true;
     this.tagKeys = {};
 
@@ -39,9 +41,15 @@ function (angular, _, dateMath) {
 
       var groupByTags = {};
       _.each(queries, function(query) {
-        _.each(query.tags, function(val, key) {
-          groupByTags[key] = true;
-        });
+        if (query.filters && query.filters.length > 0) {
+          _.each(query.filters, function(val) {
+            groupByTags[val.tagk] = true;
+          });
+        } else {
+          _.each(query.tags, function(val, key) {
+            groupByTags[key] = true;
+          });
+        }
       });
 
       return this.performTimeSeriesQuery(queries, start, end).then(function(response) {
@@ -88,6 +96,7 @@ function (angular, _, dateMath) {
       // In case the backend is 3rd-party hosted and does not suport OPTIONS, urlencoded requests
       // go as POST rather than OPTIONS+POST
       options.headers = { 'Content-Type': 'application/x-www-form-urlencoded' };
+
       return backendSrv.datasourceRequest(options);
     };
 
@@ -215,7 +224,7 @@ function (angular, _, dateMath) {
     this.getAggregators = function() {
       if (aggregatorsPromise) { return aggregatorsPromise; }
 
-      aggregatorsPromise =  this._get('/api/aggregators').then(function(result) {
+      aggregatorsPromise = this._get('/api/aggregators').then(function(result) {
         if (result.data && _.isArray(result.data)) {
           return result.data.sort();
         }
@@ -224,6 +233,19 @@ function (angular, _, dateMath) {
       return aggregatorsPromise;
     };
 
+    var filterTypesPromise = null;
+    this.getFilterTypes = function() {
+      if (filterTypesPromise) { return filterTypesPromise; }
+
+      filterTypesPromise = this._get('/api/config/filters').then(function(result) {
+        if (result.data) {
+          return Object.keys(result.data).sort();
+        }
+        return [];
+      });
+      return filterTypesPromise;
+    };
+
     function transformMetricData(md, groupByTags, target, options) {
       var metricLabel = createMetricLabel(md, target, groupByTags, options);
       var dps = [];
@@ -307,10 +329,14 @@ function (angular, _, dateMath) {
         }
       }
 
-      query.tags = angular.copy(target.tags);
-      if(query.tags){
-        for(var key in query.tags){
-          query.tags[key] = templateSrv.replace(query.tags[key], options.scopedVars);
+      if (target.filters && target.filters.length > 0) {
+        query.filters = angular.copy(target.filters);
+      } else {
+        query.tags = angular.copy(target.tags);
+        if(query.tags){
+          for(var key in query.tags){
+            query.tags[key] = templateSrv.replace(query.tags[key], options.scopedVars);
+          }
         }
       }
 
@@ -321,11 +347,18 @@ function (angular, _, dateMath) {
       var interpolatedTagValue;
       return _.map(metrics, function(metricData) {
         return _.findIndex(options.targets, function(target) {
-          return target.metric === metricData.metric &&
+          if (target.filters && target.filters.length > 0) {
+            return target.metric === metricData.metric &&
+            _.all(target.filters, function(filter) {
+              return filter.tagk === interpolatedTagValue === "*";
+            });
+          } else {
+            return target.metric === metricData.metric &&
             _.all(target.tags, function(tagV, tagK) {
-            interpolatedTagValue = templateSrv.replace(tagV, options.scopedVars);
-            return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === "*";
-          });
+              interpolatedTagValue = templateSrv.replace(tagV, options.scopedVars);
+              return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === "*";
+            });
+          }
         });
       });
     }

public/app/plugins/datasource/opentsdb/module.ts

@@ -1,9 +1,6 @@
 import {OpenTsDatasource} from './datasource';
 import {OpenTsQueryCtrl} from './query_ctrl';
-
-class OpenTsConfigCtrl {
-  static templateUrl = 'partials/config.html';
-}
+import {OpenTsConfigCtrl} from './config_ctrl';
 
 export {
   OpenTsDatasource as Datasource,

public/app/plugins/datasource/opentsdb/partials/config.html

@@ -1,2 +1,13 @@
 <datasource-http-settings current="ctrl.current"></datasource-http-settings>
 
+<br>
+<h5>Opentsdb settings</h5>
+<div class="gf-form">
+  <span class="gf-form-label width-7">
+    Version
+  </span>
+  <span class="gf-form-select-wrapper">
+    <select class="gf-form-input gf-size-auto" ng-model="ctrl.current.jsonData.tsdbVersion" ng-options="v.value as v.name for v in ctrl.tsdbVersions"></select>
+  </span>
+  <div class="clearfix"></div>
+</div>

public/app/plugins/datasource/opentsdb/partials/query.editor.html

@@ -63,12 +63,11 @@
 			</select>
 		</li>
 
-		<li class="tight-form-item query-keyword" style="width: 59px">
+		<li class="tight-form-item query-keyword" style="width: 59px" ng-if="ctrl.tsdbVersion == 2">
 			Fill
-			<tip>Available since OpenTSDB 2.2</tip>
 		</li>
 
-		<li>
+		<li ng-if="ctrl.tsdbVersion == 2">
 			<select ng-model="ctrl.target.downsampleFillPolicy" class="tight-form-input input-small"
 				ng-options="agg for agg in ctrl.fillPolicies"
 				ng-change="ctrl.targetBlur()">
@@ -83,10 +82,67 @@
 	<div class="clearfix"></div>
 </div>
 
+<div class="tight-form" ng-if="ctrl.tsdbVersion == 2">
+  <ul class="tight-form-list" role="menu">
+    <li class="tight-form-item tight-form-align query-keyword" style="width: 100px">
+      Filters
+      <tip ng-if="ctrl.tsdbVersion == 2">Filters does not work with tags, either of the two will work but not both.</tip>
+    </li>
+    <li ng-repeat="fil in ctrl.target.filters track by $index" class="tight-form-item">
+      {{fil.tagk}}&nbsp;=&nbsp;{{fil.type}}&#40;{{fil.filter}}&#41;&nbsp;&#44&nbsp;groupBy&nbsp;=&nbsp;{{fil.groupBy}}
+      <a ng-click="ctrl.editFilter(fil, $index)">
+        <i class="fa fa-pencil"></i>
+      </a>
+      <a ng-click="ctrl.removeFilter($index)">
+        <i class="fa fa-remove"></i>
+      </a>
+    </li>
+    <li class="tight-form-item query-keyword" ng-hide="ctrl.addFilterMode">
+      <a ng-click="ctrl.addFilter()">
+        <i class="fa fa-plus"></i>
+      </a>
+    </li>
+
+    <li class="query-keyword" ng-show="ctrl.addFilterMode">
+      <input type="text" class="input-small tight-form-input" spellcheck='false'
+      bs-typeahead="ctrl.suggestTagKeys" data-min-length=0 data-items=100
+      ng-model="ctrl.target.currentFilterKey" placeholder="key"></input>
+
+      Type <select ng-model="ctrl.target.currentFilterType"
+      class="tight-form-input input-small"
+      ng-options="filType for filType in ctrl.filterTypes">
+      </select>
+ 
+      <input type="text" class="input-small tight-form-input"
+      spellcheck='false' bs-typeahead="ctrl.suggestTagValues"
+      data-min-length=0 data-items=100 ng-model="ctrl.target.currentFilterValue" placeholder="filter">
+      </input>
+
+      groupBy <editor-checkbox text="" model="ctrl.target.currentFilterGroupBy"></editor-checkbox>
+
+      <a bs-tooltip="ctrl.errors.filters"
+        style="color: rgb(229, 189, 28)"
+        ng-show="ctrl.errors.filters">
+        <i class="fa fa-warning"></i>
+      </a>
+ 
+      <a ng-click="ctrl.addFilter()" ng-hide="ctrl.errors.filters">
+        add filter
+      </a>
+      <a ng-click="ctrl.closeAddFilterMode()">
+        <i class="fa fa-remove"></i>
+      </a>
+
+    </li>
+  </ul>
+  <div class="clearfix"></div>
+</div>
+
 <div class="tight-form">
 	<ul class="tight-form-list" role="menu">
 		<li class="tight-form-item tight-form-align query-keyword" style="width: 100px">
 			Tags
+      <tip ng-if="ctrl.tsdbVersion == 2">Please use filters, tags are deprecated in opentsdb 2.2</tip>
 		</li>
 		<li ng-repeat="(key, value) in ctrl.target.tags track by $index" class="tight-form-item">
 			{{key}}&nbsp;=&nbsp;{{value}}
@@ -113,15 +169,21 @@
 			spellcheck='false' bs-typeahead="ctrl.suggestTagValues"
 			data-min-length=0 data-items=100 ng-model="ctrl.target.currentTagValue" placeholder="value">
 			</input>
-			<a ng-click="ctrl.addTag()">
+
+      <a bs-tooltip="ctrl.errors.tags"
+        style="color: rgb(229, 189, 28)"
+        ng-show="ctrl.errors.tags">
+        <i class="fa fa-warning"></i>
+      </a>
+
+			<a ng-click="ctrl.addTag()" ng-hide="ctrl.errors.tags">
 				add tag
 			</a>
-			<a bs-tooltip="ctrl.errors.tags"
-				style="color: rgb(229, 189, 28)"
-				ng-show="target.errors.tags">
-				<i class="fa fa-warning"></i>
-			</a>
-		</li>
+      <a ng-click="ctrl.closeAddTagMode()">
+        <i class="fa fa-remove"></i>
+      </a>
+		
+    </li>
 	</ul>
 	<div class="clearfix"></div>
 </div>

public/app/plugins/datasource/opentsdb/query_ctrl.ts

@@ -8,6 +8,8 @@ export class OpenTsQueryCtrl extends QueryCtrl {
   static templateUrl = 'partials/query.editor.html';
   aggregators: any;
   fillPolicies: any;
+  filterTypes: any;
+  tsdbVersion: any;
   aggregator: any;
   downsampleInterval: any;
   downsampleAggregator: any;
@@ -17,6 +19,7 @@ export class OpenTsQueryCtrl extends QueryCtrl {
   suggestTagKeys: any;
   suggestTagValues: any;
   addTagMode: boolean;
+  addFilterMode: boolean;
 
   /** @ngInject **/
   constructor($scope, $injector) {
@@ -25,6 +28,9 @@ export class OpenTsQueryCtrl extends QueryCtrl {
     this.errors = this.validateTarget();
     this.aggregators = ['avg', 'sum', 'min', 'max', 'dev', 'zimsum', 'mimmin', 'mimmax'];
     this.fillPolicies = ['none', 'nan', 'null', 'zero'];
+    this.filterTypes = ['wildcard','iliteral_or','not_iliteral_or','not_literal_or','iwildcard','literal_or','regexp'];
+
+    this.tsdbVersion = this.datasource.tsdbVersion;
 
     if (!this.target.aggregator) {
       this.target.aggregator = 'sum';
@@ -39,7 +45,15 @@ export class OpenTsQueryCtrl extends QueryCtrl {
     }
 
     this.datasource.getAggregators().then((aggs) => {
-      this.aggregators = aggs;
+      if (aggs.length !== 0) {
+        this.aggregators = aggs;
+      }
+    });
+
+    this.datasource.getFilterTypes().then((filterTypes) => {
+      if (filterTypes.length !== 0) {
+        this.filterTypes = filterTypes;
+      }
     });
 
     // needs to be defined here as it is called from typeahead
@@ -70,6 +84,11 @@ export class OpenTsQueryCtrl extends QueryCtrl {
   }
 
   addTag() {
+
+    if (this.target.filters && this.target.filters.length > 0) {
+      this.errors.tags = "Please remove filters to use tags, tags and filters are mutually exclusive.";
+    }
+
     if (!this.addTagMode) {
       this.addTagMode = true;
       return;
@@ -103,6 +122,73 @@ export class OpenTsQueryCtrl extends QueryCtrl {
     this.addTag();
   }
 
+  closeAddTagMode() {
+    this.addTagMode = false;
+    return;
+  }
+
+  addFilter() {
+
+    if (this.target.tags && _.size(this.target.tags) > 0) {
+      this.errors.filters = "Please remove tags to use filters, tags and filters are mutually exclusive.";
+    }
+
+    if (!this.addFilterMode) {
+      this.addFilterMode = true;
+      return;
+    }
+
+    if (!this.target.filters) {
+      this.target.filters = [];
+    }
+
+    if (!this.target.currentFilterType) {
+      this.target.currentFilterType = 'iliteral_or';
+    }
+
+    if (!this.target.currentFilterGroupBy) {
+      this.target.currentFilterGroupBy = false;
+    }
+
+    this.errors = this.validateTarget();
+
+    if (!this.errors.filters) {
+      var currentFilter = {
+        type:    this.target.currentFilterType,
+        tagk:     this.target.currentFilterKey,
+        filter:   this.target.currentFilterValue,
+        groupBy: this.target.currentFilterGroupBy
+      };
+      this.target.filters.push(currentFilter);
+      this.target.currentFilterType = 'literal_or';
+      this.target.currentFilterKey = '';
+      this.target.currentFilterValue = '';
+      this.target.currentFilterGroupBy = false;
+      this.targetBlur();
+    }
+
+    this.addFilterMode = false;
+  }
+
+  removeFilter(index) {
+    this.target.filters.splice(index, 1);
+    this.targetBlur();
+  }
+
+  editFilter(fil, index) {
+    this.removeFilter(index);
+    this.target.currentFilterKey = fil.tagk;
+    this.target.currentFilterValue = fil.filter;
+    this.target.currentFilterType = fil.type;
+    this.target.currentFilterGroupBy = fil.groupBy;
+    this.addFilter();
+  }
+
+  closeAddFilterMode() {
+    this.addFilterMode = false;
+    return;
+  }
+
   validateTarget() {
     var errs: any = {};
 

public/app/plugins/datasource/opentsdb/specs/datasource-specs.ts

@@ -4,7 +4,7 @@ import {OpenTsDatasource} from "../datasource";
 
 describe('opentsdb', function() {
   var ctx = new helpers.ServiceTestContext();
-  var instanceSettings = {url: '' };
+  var instanceSettings = {url: '', jsonData: { tsdbVersion: 1 }};
 
   beforeEach(angularMocks.module('grafana.core'));
   beforeEach(angularMocks.module('grafana.services'));

public/app/plugins/datasource/opentsdb/specs/query-ctrl-specs.ts

@@ -0,0 +1,86 @@
+import {describe, beforeEach, it, sinon, expect, angularMocks} from 'test/lib/common';
+import helpers from 'test/specs/helpers';
+import {OpenTsQueryCtrl} from "../query_ctrl";
+
+describe('OpenTsQueryCtrl', function() {
+  var ctx = new helpers.ControllerTestContext();
+
+  beforeEach(angularMocks.module('grafana.core'));
+  beforeEach(angularMocks.module('grafana.services'));
+  beforeEach(ctx.providePhase(['backendSrv','templateSrv']));
+
+  beforeEach(ctx.providePhase());
+  beforeEach(angularMocks.inject(($rootScope, $controller, $q) => {
+    ctx.$q = $q;
+    ctx.scope = $rootScope.$new();
+    ctx.target = {target: ''};
+    ctx.panelCtrl = {panel: {}};
+    ctx.panelCtrl.refresh = sinon.spy();
+    ctx.datasource.getAggregators = sinon.stub().returns(ctx.$q.when([]));
+    ctx.datasource.getFilterTypes = sinon.stub().returns(ctx.$q.when([]));
+
+    ctx.ctrl = $controller(OpenTsQueryCtrl, {$scope: ctx.scope}, {
+      panelCtrl: ctx.panelCtrl,
+      datasource: ctx.datasource,
+      target: ctx.target,
+    });
+    ctx.scope.$digest();
+  }));
+
+  describe('init query_ctrl variables', function() {
+
+    it('filter types should be initialized', function() {
+      expect(ctx.ctrl.filterTypes.length).to.be(7);
+    });
+
+    it('aggregators should be initialized', function() {
+      expect(ctx.ctrl.aggregators.length).to.be(8);
+    });
+
+    it('fill policy options should be initialized', function() {
+      expect(ctx.ctrl.fillPolicies.length).to.be(4);
+    });
+
+  });
+
+  describe('when adding filters and tags', function() {
+
+    it('addTagMode should be false when closed', function() {
+      ctx.ctrl.addTagMode = true;
+      ctx.ctrl.closeAddTagMode();
+      expect(ctx.ctrl.addTagMode).to.be(false);
+    });
+
+    it('addFilterMode should be false when closed', function() {
+      ctx.ctrl.addFilterMode = true;
+      ctx.ctrl.closeAddFilterMode();
+      expect(ctx.ctrl.addFilterMode).to.be(false);
+    });
+
+    it('removing a tag from the tags list', function() {
+      ctx.ctrl.target.tags = {"tagk": "tag_key", "tagk2": "tag_value2"};
+      ctx.ctrl.removeTag("tagk");
+      expect(Object.keys(ctx.ctrl.target.tags).length).to.be(1);
+    });
+
+    it('removing a filter from the filters list', function() {
+      ctx.ctrl.target.filters = [{"tagk": "tag_key", "filter": "tag_value2", "type": "wildcard", "groupBy": true}];
+      ctx.ctrl.removeFilter(0);
+      expect(ctx.ctrl.target.filters.length).to.be(0);
+    });
+
+    it('adding a filter when tags exist should generate error', function() {
+      ctx.ctrl.target.tags = {"tagk": "tag_key", "tagk2": "tag_value2"};
+      ctx.ctrl.addFilter();
+      expect(ctx.ctrl.errors.filters).to.be('Please remove tags to use filters, tags and filters are mutually exclusive.');
+    });
+
+    it('adding a tag when filters exist should generate error', function() {
+      ctx.ctrl.target.filters = [{"tagk": "tag_key", "filter": "tag_value2", "type": "wildcard", "groupBy": true}];
+      ctx.ctrl.addTag();
+      expect(ctx.ctrl.errors.tags).to.be('Please remove filters to use tags, tags and filters are mutually exclusive.');
+    });
+
+  });
+
+});