Adhoc-filtering for prometheus dashboards (#13212)

* Basic adhoc-filtering support for prometheus

public/app/features/dashboard/ad_hoc_filters.ts

@@ -59,10 +59,10 @@ export class AdHocFiltersCtrl {
       let promise = null;
 
       if (segment.type !== 'value') {
-        promise = ds.getTagKeys();
+        promise = ds.getTagKeys ? ds.getTagKeys() : Promise.resolve([]);
       } else {
         options.key = this.segments[index - 2].value;
-        promise = ds.getTagValues(options);
+        promise = ds.getTagValues ? ds.getTagValues(options) : Promise.resolve([]);
       }
 
       return promise.then(results => {
@@ -99,7 +99,7 @@ export class AdHocFiltersCtrl {
           this.segments.splice(index, 0, this.uiSegmentSrv.newCondition('AND'));
         }
         this.segments.push(this.uiSegmentSrv.newOperator('='));
-        this.segments.push(this.uiSegmentSrv.newFake('select tag value', 'value', 'query-segment-value'));
+        this.segments.push(this.uiSegmentSrv.newFake('select value', 'value', 'query-segment-value'));
         segment.type = 'key';
         segment.cssClass = 'query-segment-key';
       }

public/app/features/templating/template_srv.ts

@@ -56,11 +56,10 @@ export class TemplateSrv {
         continue;
       }
 
-      if (variable.datasource === datasourceName) {
+      // null is the "default" datasource
+      if (variable.datasource === null || variable.datasource === datasourceName) {
         filters = filters.concat(variable.filters);
-      }
-
-      if (variable.datasource.indexOf('$') === 0) {
+      } else if (variable.datasource.indexOf('$') === 0) {
         if (this.replace(variable.datasource) === datasourceName) {
           filters = filters.concat(variable.filters);
         }

public/app/plugins/datasource/prometheus/add_label_to_query.ts

@@ -0,0 +1,93 @@
+import _ from 'lodash';
+
+const keywords = 'by|without|on|ignoring|group_left|group_right';
+
+// Duplicate from mode-prometheus.js, which can't be used in tests due to global ace not being loaded.
+const builtInWords = [
+  keywords,
+  'count|count_values|min|max|avg|sum|stddev|stdvar|bottomk|topk|quantile',
+  'true|false|null|__name__|job',
+  'abs|absent|ceil|changes|clamp_max|clamp_min|count_scalar|day_of_month|day_of_week|days_in_month|delta|deriv',
+  'drop_common_labels|exp|floor|histogram_quantile|holt_winters|hour|idelta|increase|irate|label_replace|ln|log2',
+  'log10|minute|month|predict_linear|rate|resets|round|scalar|sort|sort_desc|sqrt|time|vector|year|avg_over_time',
+  'min_over_time|max_over_time|sum_over_time|count_over_time|quantile_over_time|stddev_over_time|stdvar_over_time',
+]
+  .join('|')
+  .split('|');
+
+const metricNameRegexp = /([A-Za-z]\w*)\b(?![\(\]{=!",])/g;
+const selectorRegexp = /{([^{]*)}/g;
+
+// addLabelToQuery('foo', 'bar', 'baz') => 'foo{bar="baz"}'
+export function addLabelToQuery(query: string, key: string, value: string, operator?: string): string {
+  if (!key || !value) {
+    throw new Error('Need label to add to query.');
+  }
+
+  // Add empty selectors to bare metric names
+  let previousWord;
+  query = query.replace(metricNameRegexp, (match, word, offset) => {
+    const insideSelector = isPositionInsideChars(query, offset, '{', '}');
+    // Handle "sum by (key) (metric)"
+    const previousWordIsKeyWord = previousWord && keywords.split('|').indexOf(previousWord) > -1;
+    previousWord = word;
+    if (!insideSelector && !previousWordIsKeyWord && builtInWords.indexOf(word) === -1) {
+      return `${word}{}`;
+    }
+    return word;
+  });
+
+  // Adding label to existing selectors
+  let match = selectorRegexp.exec(query);
+  const parts = [];
+  let lastIndex = 0;
+  let suffix = '';
+
+  while (match) {
+    const prefix = query.slice(lastIndex, match.index);
+    const selector = match[1];
+    const selectorWithLabel = addLabelToSelector(selector, key, value, operator);
+    lastIndex = match.index + match[1].length + 2;
+    suffix = query.slice(match.index + match[0].length);
+    parts.push(prefix, '{', selectorWithLabel, '}');
+    match = selectorRegexp.exec(query);
+  }
+
+  parts.push(suffix);
+  return parts.join('');
+}
+
+const labelRegexp = /(\w+)\s*(=|!=|=~|!~)\s*("[^"]*")/g;
+
+function addLabelToSelector(selector: string, labelKey: string, labelValue: string, labelOperator?: string) {
+  const parsedLabels = [];
+
+  // Split selector into labels
+  if (selector) {
+    let match = labelRegexp.exec(selector);
+    while (match) {
+      parsedLabels.push({ key: match[1], operator: match[2], value: match[3] });
+      match = labelRegexp.exec(selector);
+    }
+  }
+
+  // Add new label
+  const operatorForLabelKey = labelOperator || '=';
+  parsedLabels.push({ key: labelKey, operator: operatorForLabelKey, value: `"${labelValue}"` });
+
+  // Sort labels by key and put them together
+  return _.chain(parsedLabels)
+    .compact()
+    .sortBy('key')
+    .map(({ key, operator, value }) => `${key}${operator}${value}`)
+    .value()
+    .join(',');
+}
+
+function isPositionInsideChars(text: string, position: number, openChar: string, closeChar: string) {
+  const nextSelectorStart = text.slice(position).indexOf(openChar);
+  const nextSelectorEnd = text.slice(position).indexOf(closeChar);
+  return nextSelectorEnd > -1 && (nextSelectorStart === -1 || nextSelectorStart > nextSelectorEnd);
+}
+
+export default addLabelToQuery;

public/app/plugins/datasource/prometheus/datasource.ts

@@ -7,6 +7,8 @@ import PrometheusMetricFindQuery from './metric_find_query';
 import { ResultTransformer } from './result_transformer';
 import { BackendSrv } from 'app/core/services/backend_srv';
 
+import addLabelToQuery from './add_label_to_query';
+
 export function alignRange(start, end, step) {
   const alignedEnd = Math.ceil(end / step) * step;
   const alignedStart = Math.floor(start / step) * step;
@@ -16,74 +18,6 @@ export function alignRange(start, end, step) {
   };
 }
 
-const keywords = 'by|without|on|ignoring|group_left|group_right';
-
-// Duplicate from mode-prometheus.js, which can't be used in tests due to global ace not being loaded.
-const builtInWords = [
-  keywords,
-  'count|count_values|min|max|avg|sum|stddev|stdvar|bottomk|topk|quantile',
-  'true|false|null|__name__|job',
-  'abs|absent|ceil|changes|clamp_max|clamp_min|count_scalar|day_of_month|day_of_week|days_in_month|delta|deriv',
-  'drop_common_labels|exp|floor|histogram_quantile|holt_winters|hour|idelta|increase|irate|label_replace|ln|log2',
-  'log10|minute|month|predict_linear|rate|resets|round|scalar|sort|sort_desc|sqrt|time|vector|year|avg_over_time',
-  'min_over_time|max_over_time|sum_over_time|count_over_time|quantile_over_time|stddev_over_time|stdvar_over_time',
-]
-  .join('|')
-  .split('|');
-
-// addLabelToQuery('foo', 'bar', 'baz') => 'foo{bar="baz"}'
-export function addLabelToQuery(query: string, key: string, value: string): string {
-  if (!key || !value) {
-    throw new Error('Need label to add to query.');
-  }
-
-  // Add empty selector to bare metric name
-  let previousWord;
-  query = query.replace(/([A-Za-z]\w*)\b(?![\(\]{=",])/g, (match, word, offset) => {
-    // Check if inside a selector
-    const nextSelectorStart = query.slice(offset).indexOf('{');
-    const nextSelectorEnd = query.slice(offset).indexOf('}');
-    const insideSelector = nextSelectorEnd > -1 && (nextSelectorStart === -1 || nextSelectorStart > nextSelectorEnd);
-    // Handle "sum by (key) (metric)"
-    const previousWordIsKeyWord = previousWord && keywords.split('|').indexOf(previousWord) > -1;
-    previousWord = word;
-    if (!insideSelector && !previousWordIsKeyWord && builtInWords.indexOf(word) === -1) {
-      return `${word}{}`;
-    }
-    return word;
-  });
-
-  // Adding label to existing selectors
-  const selectorRegexp = /{([^{]*)}/g;
-  let match = selectorRegexp.exec(query);
-  const parts = [];
-  let lastIndex = 0;
-  let suffix = '';
-
-  while (match) {
-    const prefix = query.slice(lastIndex, match.index);
-    const selectorParts = match[1].split(',');
-    const labels = selectorParts.reduce((acc, label) => {
-      const labelParts = label.split('=');
-      if (labelParts.length === 2) {
-        acc[labelParts[0]] = labelParts[1];
-      }
-      return acc;
-    }, {});
-    labels[key] = `"${value}"`;
-    const selector = Object.keys(labels)
-      .sort()
-      .map(key => `${key}=${labels[key]}`)
-      .join(',');
-    lastIndex = match.index + match[1].length + 2;
-    suffix = query.slice(match.index + match[0].length);
-    parts.push(prefix, '{', selector, '}');
-    match = selectorRegexp.exec(query);
-  }
-  parts.push(suffix);
-  return parts.join('');
-}
-
 export function determineQueryHints(series: any[], datasource?: any): any[] {
   const hints = series.map((s, i) => {
     const query: string = s.query;
@@ -406,8 +340,21 @@ export class PrometheusDatasource {
     }
     query.step = interval;
 
+    let expr = target.expr;
+
+    // Apply adhoc filters
+    const adhocFilters = this.templateSrv.getAdhocFilters(this.name);
+    expr = adhocFilters.reduce((acc, filter) => {
+      const { key, operator } = filter;
+      let { value } = filter;
+      if (operator === '=~' || operator === '!~') {
+        value = prometheusSpecialRegexEscape(value);
+      }
+      return addLabelToQuery(acc, key, value, operator);
+    }, expr);
+
     // Only replace vars in expression after having (possibly) updated interval vars
-    query.expr = this.templateSrv.replace(target.expr, scopedVars, this.interpolateQueryExpr);
+    query.expr = this.templateSrv.replace(expr, scopedVars, this.interpolateQueryExpr);
     query.requestId = options.panelId + target.refId;
 
     // Align query interval with step

public/app/plugins/datasource/prometheus/specs/add_label_to_query.test.ts

@@ -0,0 +1,42 @@
+import addLabelToQuery from '../add_label_to_query';
+
+describe('addLabelToQuery()', () => {
+  it('should add label to simple query', () => {
+    expect(() => {
+      addLabelToQuery('foo', '', '');
+    }).toThrow();
+    expect(addLabelToQuery('foo', 'bar', 'baz')).toBe('foo{bar="baz"}');
+    expect(addLabelToQuery('foo{}', 'bar', 'baz')).toBe('foo{bar="baz"}');
+    expect(addLabelToQuery('foo{x="yy"}', 'bar', 'baz')).toBe('foo{bar="baz",x="yy"}');
+    expect(addLabelToQuery('metric > 0.001', 'foo', 'bar')).toBe('metric{foo="bar"} > 0.001');
+  });
+
+  it('should add custom operator', () => {
+    expect(addLabelToQuery('foo{}', 'bar', 'baz', '!=')).toBe('foo{bar!="baz"}');
+    expect(addLabelToQuery('foo{x="yy"}', 'bar', 'baz', '!=')).toBe('foo{bar!="baz",x="yy"}');
+  });
+
+  it('should not modify ranges', () => {
+    expect(addLabelToQuery('rate(metric[1m])', 'foo', 'bar')).toBe('rate(metric{foo="bar"}[1m])');
+  });
+
+  it('should detect in-order function use', () => {
+    expect(addLabelToQuery('sum by (xx) (foo)', 'bar', 'baz')).toBe('sum by (xx) (foo{bar="baz"})');
+  });
+
+  it('should handle selectors with punctuation', () => {
+    expect(addLabelToQuery('foo{instance="my-host.com:9100"}', 'bar', 'baz')).toBe(
+      'foo{bar="baz",instance="my-host.com:9100"}'
+    );
+    expect(addLabelToQuery('foo{list="a,b,c"}', 'bar', 'baz')).toBe('foo{bar="baz",list="a,b,c"}');
+  });
+
+  it('should work on arithmetical expressions', () => {
+    expect(addLabelToQuery('foo + foo', 'bar', 'baz')).toBe('foo{bar="baz"} + foo{bar="baz"}');
+    expect(addLabelToQuery('foo{x="yy"} + metric', 'bar', 'baz')).toBe('foo{bar="baz",x="yy"} + metric{bar="baz"}');
+    expect(addLabelToQuery('avg(foo) + sum(xx_yy)', 'bar', 'baz')).toBe('avg(foo{bar="baz"}) + sum(xx_yy{bar="baz"})');
+    expect(addLabelToQuery('foo{x="yy"} * metric{y="zz",a="bb"} * metric2', 'bar', 'baz')).toBe(
+      'foo{bar="baz",x="yy"} * metric{a="bb",bar="baz",y="zz"} * metric2{bar="baz"}'
+    );
+  });
+});

public/app/plugins/datasource/prometheus/specs/datasource.test.ts

@@ -8,11 +8,15 @@ import {
   PrometheusDatasource,
   prometheusSpecialRegexEscape,
   prometheusRegularEscape,
-  addLabelToQuery,
 } from '../datasource';
 
 jest.mock('../metric_find_query');
 
+const DEFAULT_TEMPLATE_SRV_MOCK = {
+  getAdhocFilters: () => [],
+  replace: a => a,
+};
+
 describe('PrometheusDatasource', () => {
   const ctx: any = {};
   const instanceSettings = {
@@ -25,9 +29,8 @@ describe('PrometheusDatasource', () => {
 
   ctx.backendSrvMock = {};
 
-  ctx.templateSrvMock = {
-    replace: a => a,
-  };
+  ctx.templateSrvMock = DEFAULT_TEMPLATE_SRV_MOCK;
+
   ctx.timeSrvMock = {
     timeRange: () => {
       return {
@@ -60,6 +63,37 @@ describe('PrometheusDatasource', () => {
     });
   });
 
+  describe('When using adhoc filters', () => {
+    const DEFAULT_QUERY_EXPRESSION = 'metric{job="foo"} - metric';
+    const target = { expr: DEFAULT_QUERY_EXPRESSION };
+
+    afterEach(() => {
+      ctx.templateSrvMock.getAdhocFilters = DEFAULT_TEMPLATE_SRV_MOCK.getAdhocFilters;
+    });
+
+    it('should not modify expression with no filters', () => {
+      const result = ctx.ds.createQuery(target, { interval: '15s' });
+      expect(result).toMatchObject({ expr: DEFAULT_QUERY_EXPRESSION });
+    });
+
+    it('should add filters to expression', () => {
+      ctx.templateSrvMock.getAdhocFilters = () => [
+        {
+          key: 'k1',
+          operator: '=',
+          value: 'v1',
+        },
+        {
+          key: 'k2',
+          operator: '!=',
+          value: 'v2',
+        },
+      ];
+      const result = ctx.ds.createQuery(target, { interval: '15s' });
+      expect(result).toMatchObject({ expr: 'metric{job="foo",k1="v1",k2!="v2"} - metric{k1="v1",k2!="v2"}' });
+    });
+  });
+
   describe('When performing performSuggestQuery', () => {
     it('should cache response', async () => {
       ctx.backendSrvMock.datasourceRequest.mockReturnValue(
@@ -358,26 +392,6 @@ describe('PrometheusDatasource', () => {
       expect(intervalMs).toEqual({ text: 15000, value: 15000 });
     });
   });
-
-  describe('addLabelToQuery()', () => {
-    expect(() => {
-      addLabelToQuery('foo', '', '');
-    }).toThrow();
-    expect(addLabelToQuery('foo + foo', 'bar', 'baz')).toBe('foo{bar="baz"} + foo{bar="baz"}');
-    expect(addLabelToQuery('foo{}', 'bar', 'baz')).toBe('foo{bar="baz"}');
-    expect(addLabelToQuery('foo{x="yy"}', 'bar', 'baz')).toBe('foo{bar="baz",x="yy"}');
-    expect(addLabelToQuery('foo{x="yy"} + metric', 'bar', 'baz')).toBe('foo{bar="baz",x="yy"} + metric{bar="baz"}');
-    expect(addLabelToQuery('avg(foo) + sum(xx_yy)', 'bar', 'baz')).toBe('avg(foo{bar="baz"}) + sum(xx_yy{bar="baz"})');
-    expect(addLabelToQuery('foo{x="yy"} * metric{y="zz",a="bb"} * metric2', 'bar', 'baz')).toBe(
-      'foo{bar="baz",x="yy"} * metric{a="bb",bar="baz",y="zz"} * metric2{bar="baz"}'
-    );
-    expect(addLabelToQuery('sum by (xx) (foo)', 'bar', 'baz')).toBe('sum by (xx) (foo{bar="baz"})');
-    expect(addLabelToQuery('foo{instance="my-host.com:9100"}', 'bar', 'baz')).toBe(
-      'foo{bar="baz",instance="my-host.com:9100"}'
-    );
-    expect(addLabelToQuery('rate(metric[1m])', 'foo', 'bar')).toBe('rate(metric{foo="bar"}[1m])');
-    expect(addLabelToQuery('metric > 0.001', 'foo', 'bar')).toBe('metric{foo="bar"} > 0.001');
-  });
 });
 
 const SECOND = 1000;
@@ -399,6 +413,7 @@ const backendSrv = {
 } as any;
 
 const templateSrv = {
+  getAdhocFilters: () => [],
   replace: jest.fn(str => str),
 };