Merge pull request #9226 from alin-amana/adjust_interval_variable_with_min_step

Prometheus: Rework interval and step computation

public/app/plugins/datasource/prometheus/datasource.ts

@@ -1,15 +1,12 @@
 ///<reference path="../../../headers/common.d.ts" />
 
 import _ from 'lodash';
-import moment from 'moment';
 
 import kbn from 'app/core/utils/kbn';
 import * as dateMath from 'app/core/utils/datemath';
 import PrometheusMetricFindQuery from './metric_find_query';
 import TableModel from 'app/core/table_model';
 
-var durationSplitRegexp = /(\d+)(ms|s|m|h|d|w|M|y)/;
-
 function prometheusSpecialRegexEscape(value) {
   return value.replace(/[\\^$*+?.()|[\]{}]/g, '\\\\$&');
 }
@@ -83,6 +80,7 @@ export class PrometheusDatasource {
     var self = this;
     var start = this.getPrometheusTime(options.range.from, false);
     var end = this.getPrometheusTime(options.range.to, true);
+    var range = Math.ceil(end - start);
 
     var queries = [];
     var activeTargets = [];
@@ -95,18 +93,7 @@ export class PrometheusDatasource {
       }
 
       activeTargets.push(target);
-
-      var query: any = {};
-      query.expr = this.templateSrv.replace(target.expr, options.scopedVars, self.interpolateQueryExpr);
-      query.requestId = options.panelId + target.refId;
-      query.instant = target.instant;
-
-      var interval = this.templateSrv.replace(target.interval, options.scopedVars) || options.interval;
-      var intervalFactor = target.intervalFactor || 1;
-      target.step = query.step = this.calculateInterval(interval, intervalFactor);
-      var range = Math.ceil(end - start);
-      target.step = query.step = this.adjustStep(query.step, this.intervalSeconds(options.interval), range);
-      queries.push(query);
+      queries.push(this.createQuery(target, options, range));
     }
 
     // No valid targets, return the empty result to save a round trip.
@@ -147,13 +134,41 @@ export class PrometheusDatasource {
     });
   }
 
-  adjustStep(step, autoStep, range) {
-    // Prometheus drop query if range/step > 11000
-    // calibrate step if it is too big
-    if (step !== 0 && range / step > 11000) {
-      step = Math.ceil(range / 11000);
+  createQuery(target, options, range) {
+    var query: any = {};
+    query.instant = target.instant;
+
+    var interval = kbn.interval_to_seconds(options.interval);
+    // Minimum interval ("Min step"), if specified for the query. or same as interval otherwise
+    var minInterval = kbn.interval_to_seconds(this.templateSrv.replace(target.interval, options.scopedVars) || options.interval);
+    var intervalFactor = target.intervalFactor || 1;
+    // Adjust the interval to take into account any specified minimum and interval factor plus Prometheus limits
+    var adjustedInterval = this.adjustInterval(interval, minInterval, range, intervalFactor);
+
+    var scopedVars = options.scopedVars;
+    // If the interval was adjusted, make a shallow copy of scopedVars with updated interval vars
+    if (interval !== adjustedInterval) {
+      interval = adjustedInterval;
+      scopedVars = Object.assign({}, options.scopedVars, {
+        "__interval":     {text: interval + "s",  value: interval + "s"},
+        "__interval_ms":  {text: interval * 1000, value: interval * 1000},
+      });
+    }
+    target.step = query.step = interval;
+
+    // Only replace vars in expression after having (possibly) updated interval vars
+    query.expr = this.templateSrv.replace(target.expr, scopedVars, this.interpolateQueryExpr);
+    query.requestId = options.panelId + target.refId;
+    return query;
+  }
+
+  adjustInterval(interval, minInterval, range, intervalFactor) {
+    // Prometheus will drop queries that might return more than 11000 data points.
+    // Calibrate interval if it is too small.
+    if (interval !== 0 && range / intervalFactor / interval > 11000) {
+      interval = Math.ceil(range / intervalFactor / 11000);
     }
-    return Math.max(step, autoStep);
+    return Math.max(interval * intervalFactor, minInterval);
   }
 
   performTimeSeriesQuery(query, start, end) {
@@ -218,7 +233,7 @@ export class PrometheusDatasource {
     var end = this.getPrometheusTime(options.range.to, true);
     var query = {
       expr: interpolated,
-      step: this.adjustStep(kbn.interval_to_seconds(step), 0, Math.ceil(end - start)) + 's'
+      step: this.adjustInterval(kbn.interval_to_seconds(step), 0, Math.ceil(end - start), 1) + 's'
     };
 
     var self = this;
@@ -257,21 +272,6 @@ export class PrometheusDatasource {
     });
   }
 
-  calculateInterval(interval, intervalFactor) {
-    return Math.ceil(this.intervalSeconds(interval) * intervalFactor);
-  }
-
-  intervalSeconds(interval) {
-    var m = interval.match(durationSplitRegexp);
-    var dur = moment.duration(parseInt(m[1]), m[2]);
-    var sec = dur.asSeconds();
-    if (sec < 1) {
-      sec = 1;
-    }
-
-    return sec;
-  }
-
   transformMetricData(md, options, start, end) {
     var dps = [],
       metricLabel = null;

public/app/plugins/datasource/prometheus/specs/datasource_specs.ts

@@ -269,4 +269,311 @@ describe('PrometheusDatasource', function() {
       );
     });
   });
+  describe('The "step" query parameter', function() {
+    var response = {
+      status: "success",
+      data: {
+        resultType: "matrix",
+        result: []
+      }
+    };
+
+    it('should be min interval when greater than auto interval', function() {
+      var query = {
+        // 6 hour range
+        range: { from: moment(1443438674760), to: moment(1443460274760) },
+        targets: [{
+          expr: 'test',
+          interval: '10s'
+        }],
+        interval: '5s'
+      };
+      var urlExpected = 'proxied/api/v1/query_range?query=test' +
+                        '&start=1443438675&end=1443460275&step=10';
+      ctx.$httpBackend.expect('GET', urlExpected).respond(response);
+      ctx.ds.query(query);
+      ctx.$httpBackend.verifyNoOutstandingExpectation();
+    });
+    it('should be auto interval when greater than min interval', function() {
+      var query = {
+        // 6 hour range
+        range: { from: moment(1443438674760), to: moment(1443460274760) },
+        targets: [{
+          expr: 'test',
+          interval: '5s'
+        }],
+        interval: '10s'
+      };
+      var urlExpected = 'proxied/api/v1/query_range?query=test' +
+                        '&start=1443438675&end=1443460275&step=10';
+      ctx.$httpBackend.expect('GET', urlExpected).respond(response);
+      ctx.ds.query(query);
+      ctx.$httpBackend.verifyNoOutstandingExpectation();
+    });
+    it('should result in querying fewer than 11000 data points', function() {
+      var query = {
+        // 6 hour range
+        range: { from: moment(1443438674760), to: moment(1443460274760) },
+        targets: [{ expr: 'test' }],
+        interval: '1s'
+      };
+      var urlExpected = 'proxied/api/v1/query_range?query=test' +
+                        '&start=1443438675&end=1443460275&step=2';
+      ctx.$httpBackend.expect('GET', urlExpected).respond(response);
+      ctx.ds.query(query);
+      ctx.$httpBackend.verifyNoOutstandingExpectation();
+    });
+    it('should not apply min interval when interval * intervalFactor greater', function() {
+      var query = {
+        // 6 hour range
+        range: { from: moment(1443438674760), to: moment(1443460274760) },
+        targets: [{
+          expr: 'test',
+          interval: '10s',
+          intervalFactor: 10
+        }],
+        interval: '5s'
+      };
+      var urlExpected = 'proxied/api/v1/query_range?query=test' +
+                        '&start=1443438675&end=1443460275&step=50';
+      ctx.$httpBackend.expect('GET', urlExpected).respond(response);
+      ctx.ds.query(query);
+      ctx.$httpBackend.verifyNoOutstandingExpectation();
+    });
+    it('should apply min interval when interval * intervalFactor smaller', function() {
+      var query = {
+        // 6 hour range
+        range: { from: moment(1443438674760), to: moment(1443460274760) },
+        targets: [{
+          expr: 'test',
+          interval: '15s',
+          intervalFactor: 2
+        }],
+        interval: '5s'
+      };
+      var urlExpected = 'proxied/api/v1/query_range?query=test' +
+                        '&start=1443438675&end=1443460275&step=15';
+      ctx.$httpBackend.expect('GET', urlExpected).respond(response);
+      ctx.ds.query(query);
+      ctx.$httpBackend.verifyNoOutstandingExpectation();
+    });
+    it('should apply intervalFactor to auto interval when greater', function() {
+      var query = {
+        // 6 hour range
+        range: { from: moment(1443438674760), to: moment(1443460274760) },
+        targets: [{
+          expr: 'test',
+          interval: '5s',
+          intervalFactor: 10
+        }],
+        interval: '10s'
+      };
+      var urlExpected = 'proxied/api/v1/query_range?query=test' +
+                        '&start=1443438675&end=1443460275&step=100';
+      ctx.$httpBackend.expect('GET', urlExpected).respond(response);
+      ctx.ds.query(query);
+      ctx.$httpBackend.verifyNoOutstandingExpectation();
+    });
+    it('should not not be affected by the 11000 data points limit when large enough', function() {
+      var query = {
+        // 1 week range
+        range: { from: moment(1443438674760), to: moment(1444043474760) },
+        targets: [{
+          expr: 'test',
+          intervalFactor: 10
+        }],
+        interval: '10s'
+      };
+      var urlExpected = 'proxied/api/v1/query_range?query=test' +
+                        '&start=1443438675&end=1444043475&step=100';
+      ctx.$httpBackend.expect('GET', urlExpected).respond(response);
+      ctx.ds.query(query);
+      ctx.$httpBackend.verifyNoOutstandingExpectation();
+    });
+    it('should be determined by the 11000 data points limit when too small', function() {
+      var query = {
+        // 1 week range
+        range: { from: moment(1443438674760), to: moment(1444043474760) },
+        targets: [{
+          expr: 'test',
+          intervalFactor: 10
+        }],
+        interval: '5s'
+      };
+      var urlExpected = 'proxied/api/v1/query_range?query=test' +
+                        '&start=1443438675&end=1444043475&step=60';
+      ctx.$httpBackend.expect('GET', urlExpected).respond(response);
+      ctx.ds.query(query);
+      ctx.$httpBackend.verifyNoOutstandingExpectation();
+    });
+  });
+  describe('The __interval and __interval_ms template variables', function() {
+    var response = {
+      status: "success",
+      data: {
+        resultType: "matrix",
+        result: []
+      }
+    };
+
+    it('should be unchanged when auto interval is greater than min interval', function() {
+      var query = {
+        // 6 hour range
+        range: { from: moment(1443438674760), to: moment(1443460274760) },
+        targets: [{
+          expr: 'rate(test[$__interval])',
+          interval: '5s'
+        }],
+        interval: '10s',
+        scopedVars: {
+          "__interval":     {text: "10s",  value: "10s"},
+          "__interval_ms":  {text: 10 * 1000, value: 10 * 1000},
+        }
+      };
+      var urlExpected = 'proxied/api/v1/query_range?query=' +
+                        encodeURIComponent('rate(test[10s])') +
+                        '&start=1443438675&end=1443460275&step=10';
+      ctx.$httpBackend.expect('GET', urlExpected).respond(response);
+      ctx.ds.query(query);
+      ctx.$httpBackend.verifyNoOutstandingExpectation();
+
+      expect(query.scopedVars.__interval.text).to.be("10s");
+      expect(query.scopedVars.__interval.value).to.be("10s");
+      expect(query.scopedVars.__interval_ms.text).to.be(10 * 1000);
+      expect(query.scopedVars.__interval_ms.value).to.be(10 * 1000);
+    });
+    it('should be min interval when it is greater than auto interval', function() {
+      var query = {
+        // 6 hour range
+        range: { from: moment(1443438674760), to: moment(1443460274760) },
+        targets: [{
+          expr: 'rate(test[$__interval])',
+          interval: '10s'
+        }],
+        interval: '5s',
+        scopedVars: {
+          "__interval":     {text: "5s",  value: "5s"},
+          "__interval_ms":  {text: 5 * 1000, value: 5 * 1000},
+        }
+      };
+      var urlExpected = 'proxied/api/v1/query_range?query=' +
+                        encodeURIComponent('rate(test[10s])') +
+                        '&start=1443438675&end=1443460275&step=10';
+      ctx.$httpBackend.expect('GET', urlExpected).respond(response);
+      ctx.ds.query(query);
+      ctx.$httpBackend.verifyNoOutstandingExpectation();
+
+      expect(query.scopedVars.__interval.text).to.be("5s");
+      expect(query.scopedVars.__interval.value).to.be("5s");
+      expect(query.scopedVars.__interval_ms.text).to.be(5 * 1000);
+      expect(query.scopedVars.__interval_ms.value).to.be(5 * 1000);
+    });
+    it('should account for intervalFactor', function() {
+      var query = {
+        // 6 hour range
+        range: { from: moment(1443438674760), to: moment(1443460274760) },
+        targets: [{
+          expr: 'rate(test[$__interval])',
+          interval: '5s',
+          intervalFactor: 10
+        }],
+        interval: '10s',
+        scopedVars: {
+          "__interval":     {text: "10s",  value: "10s"},
+          "__interval_ms":  {text: 10 * 1000, value: 10 * 1000},
+        }
+      };
+      var urlExpected = 'proxied/api/v1/query_range?query=' +
+                        encodeURIComponent('rate(test[100s])') +
+                        '&start=1443438675&end=1443460275&step=100';
+      ctx.$httpBackend.expect('GET', urlExpected).respond(response);
+      ctx.ds.query(query);
+      ctx.$httpBackend.verifyNoOutstandingExpectation();
+
+      expect(query.scopedVars.__interval.text).to.be("10s");
+      expect(query.scopedVars.__interval.value).to.be("10s");
+      expect(query.scopedVars.__interval_ms.text).to.be(10 * 1000);
+      expect(query.scopedVars.__interval_ms.value).to.be(10 * 1000);
+    });
+    it('should be interval * intervalFactor when greater than min interval', function() {
+      var query = {
+        // 6 hour range
+        range: { from: moment(1443438674760), to: moment(1443460274760) },
+        targets: [{
+          expr: 'rate(test[$__interval])',
+          interval: '10s',
+          intervalFactor: 10
+        }],
+        interval: '5s',
+        scopedVars: {
+          "__interval":     {text: "5s",  value: "5s"},
+          "__interval_ms":  {text: 5 * 1000, value: 5 * 1000},
+        }
+      };
+      var urlExpected = 'proxied/api/v1/query_range?query=' +
+                        encodeURIComponent('rate(test[50s])') +
+                        '&start=1443438675&end=1443460275&step=50';
+      ctx.$httpBackend.expect('GET', urlExpected).respond(response);
+      ctx.ds.query(query);
+      ctx.$httpBackend.verifyNoOutstandingExpectation();
+
+      expect(query.scopedVars.__interval.text).to.be("5s");
+      expect(query.scopedVars.__interval.value).to.be("5s");
+      expect(query.scopedVars.__interval_ms.text).to.be(5 * 1000);
+      expect(query.scopedVars.__interval_ms.value).to.be(5 * 1000);
+    });
+    it('should be min interval when greater than interval * intervalFactor', function() {
+      var query = {
+        // 6 hour range
+        range: { from: moment(1443438674760), to: moment(1443460274760) },
+        targets: [{
+          expr: 'rate(test[$__interval])',
+          interval: '15s',
+          intervalFactor: 2
+        }],
+        interval: '5s',
+        scopedVars: {
+          "__interval":     {text: "5s",  value: "5s"},
+          "__interval_ms":  {text: 5 * 1000, value: 5 * 1000},
+        }
+      };
+      var urlExpected = 'proxied/api/v1/query_range?query=' +
+                        encodeURIComponent('rate(test[15s])') +
+                        '&start=1443438675&end=1443460275&step=15';
+      ctx.$httpBackend.expect('GET', urlExpected).respond(response);
+      ctx.ds.query(query);
+      ctx.$httpBackend.verifyNoOutstandingExpectation();
+
+      expect(query.scopedVars.__interval.text).to.be("5s");
+      expect(query.scopedVars.__interval.value).to.be("5s");
+      expect(query.scopedVars.__interval_ms.text).to.be(5 * 1000);
+      expect(query.scopedVars.__interval_ms.value).to.be(5 * 1000);
+    });
+    it('should be determined by the 11000 data points limit, accounting for intervalFactor', function() {
+      var query = {
+        // 1 week range
+        range: { from: moment(1443438674760), to: moment(1444043474760) },
+        targets: [{
+          expr: 'rate(test[$__interval])',
+          intervalFactor: 10
+        }],
+        interval: '5s',
+        scopedVars: {
+          "__interval":     {text: "5s",  value: "5s"},
+          "__interval_ms":  {text: 5 * 1000, value: 5 * 1000},
+        }
+      };
+      var urlExpected = 'proxied/api/v1/query_range?query=' +
+                        encodeURIComponent('rate(test[60s])') +
+                        '&start=1443438675&end=1444043475&step=60';
+      ctx.$httpBackend.expect('GET', urlExpected).respond(response);
+      ctx.ds.query(query);
+      ctx.$httpBackend.verifyNoOutstandingExpectation();
+
+      expect(query.scopedVars.__interval.text).to.be("5s");
+      expect(query.scopedVars.__interval.value).to.be("5s");
+      expect(query.scopedVars.__interval_ms.text).to.be(5 * 1000);
+      expect(query.scopedVars.__interval_ms.value).to.be(5 * 1000);
+    });
+  });
 });