Explore: POC for datasource query importers

Explore is about keeping context between datasources if possible. When
changing from metrics to logging, some of the filtering can be kept to
narrow down logging streams relevant to the metrics.

- adds `importQueries` function in language providers
- query import dependent on origin datasource
- implemented  prometheus-to-logging import: keeping label selectors
  that are common to both datasources
- added types

public/app/features/explore/Explore.tsx

@@ -3,8 +3,9 @@ import { hot } from 'react-hot-loader';
 import Select from 'react-select';
 import _ from 'lodash';
 
+import { DataSource } from 'app/types/datasources';
 import { ExploreState, ExploreUrlState, HistoryItem, Query, QueryTransaction, ResultType } from 'app/types/explore';
-import { RawTimeRange } from 'app/types/series';
+import { RawTimeRange, DataQuery } from 'app/types/series';
 import kbn from 'app/core/utils/kbn';
 import colors from 'app/core/utils/colors';
 import store from 'app/core/store';
@@ -16,6 +17,7 @@ import PickerOption from 'app/core/components/Picker/PickerOption';
 import IndicatorsContainer from 'app/core/components/Picker/IndicatorsContainer';
 import NoOptionsMessage from 'app/core/components/Picker/NoOptionsMessage';
 import TableModel, { mergeTablesIntoModel } from 'app/core/table_model';
+import { DatasourceSrv } from 'app/features/plugins/datasource_srv';
 
 import QueryRows from './QueryRows';
 import Graph from './Graph';
@@ -24,7 +26,6 @@ import Table from './Table';
 import ErrorBoundary from './ErrorBoundary';
 import TimePicker from './TimePicker';
 import { ensureQueries, generateQueryKey, hasQuery } from './utils/query';
-import { DataSource } from 'app/types/datasources';
 
 const MAX_HISTORY_ITEMS = 100;
 
@@ -77,7 +78,7 @@ function updateHistory(history: HistoryItem[], datasourceId: string, queries: st
 }
 
 interface ExploreProps {
-  datasourceSrv: any;
+  datasourceSrv: DatasourceSrv;
   onChangeSplit: (split: boolean, state?: ExploreState) => void;
   onSaveState: (key: string, state: ExploreState) => void;
   position: string;
@@ -92,6 +93,7 @@ export class Explore extends React.PureComponent<ExploreProps, ExploreState> {
   /**
    * Current query expressions of the rows including their modifications, used for running queries.
    * Not kept in component state to prevent edit-render roundtrips.
+   * TODO: make this generic (other datasources might not have string representations of current query state)
    */
   queryExpressions: string[];
 
@@ -160,7 +162,7 @@ export class Explore extends React.PureComponent<ExploreProps, ExploreState> {
     }
   }
 
-  async setDatasource(datasource: DataSource) {
+  async setDatasource(datasource: any, origin?: DataSource) {
     const supportsGraph = datasource.meta.metrics;
     const supportsLogs = datasource.meta.logs;
     const supportsTable = datasource.meta.metrics;
@@ -181,12 +183,33 @@ export class Explore extends React.PureComponent<ExploreProps, ExploreState> {
       datasource.init();
     }
 
-    // Keep queries but reset edit state
+    // Check if queries can be imported from previously selected datasource
+    let queryExpressions = this.queryExpressions;
+    if (origin) {
+      if (origin.meta.id === datasource.meta.id) {
+        // Keep same queries if same type of datasource
+        queryExpressions = [...this.queryExpressions];
+      } else if (datasource.importQueries) {
+        // Datasource-specific importers, wrapping to satisfy interface
+        const wrappedQueries: DataQuery[] = this.queryExpressions.map((query, index) => ({
+          refId: String(index),
+          expr: query,
+        }));
+        const modifiedQueries: DataQuery[] = await datasource.importQueries(wrappedQueries, origin.meta);
+        queryExpressions = modifiedQueries.map(({ expr }) => expr);
+      } else {
+        // Default is blank queries
+        queryExpressions = this.queryExpressions.map(() => '');
+      }
+    }
+
+    // Reset edit state with new queries
     const nextQueries = this.state.queries.map((q, i) => ({
       ...q,
       key: generateQueryKey(i),
-      query: this.queryExpressions[i],
+      query: queryExpressions[i],
     }));
+    this.queryExpressions = queryExpressions;
 
     // Custom components
     const StartPage = datasource.pluginExports.ExploreStartPage;
@@ -246,6 +269,7 @@ export class Explore extends React.PureComponent<ExploreProps, ExploreState> {
   };
 
   onChangeDatasource = async option => {
+    const origin = this.state.datasource;
     this.setState({
       datasource: null,
       datasourceError: null,
@@ -254,7 +278,7 @@ export class Explore extends React.PureComponent<ExploreProps, ExploreState> {
     });
     const datasourceName = option.value;
     const datasource = await this.props.datasourceSrv.get(datasourceName);
-    this.setDatasource(datasource);
+    this.setDatasource(datasource as any, origin);
   };
 
   onChangeQuery = (value: string, index: number, override?: boolean) => {

public/app/features/plugins/datasource_srv.ts

@@ -22,7 +22,7 @@ export class DatasourceSrv {
     this.datasources = {};
   }
 
-  get(name?): Promise<DataSourceApi> {
+  get(name?: string): Promise<DataSourceApi> {
     if (!name) {
       return this.get(config.defaultDatasource);
     }
@@ -40,7 +40,7 @@ export class DatasourceSrv {
     return this.loadDatasource(name);
   }
 
-  loadDatasource(name) {
+  loadDatasource(name: string): Promise<DataSourceApi> {
     const dsConfig = config.datasources[name];
     if (!dsConfig) {
       return this.$q.reject({ message: 'Datasource named ' + name + ' was not found' });

public/app/plugins/datasource/logging/datasource.ts

@@ -1,10 +1,11 @@
 import _ from 'lodash';
 
 import * as dateMath from 'app/core/utils/datemath';
+import { LogsStream, LogsModel, makeSeriesForLogs } from 'app/core/logs_model';
+import { PluginMeta, DataQuery } from 'app/types';
 
 import LanguageProvider from './language_provider';
 import { mergeStreamsToLogs } from './result_transformer';
-import { LogsStream, LogsModel, makeSeriesForLogs } from 'app/core/logs_model';
 
 export const DEFAULT_LIMIT = 1000;
 
@@ -111,6 +112,10 @@ export default class LoggingDatasource {
     });
   }
 
+  async importQueries(queries: DataQuery[], originMeta: PluginMeta): Promise<DataQuery[]> {
+    return this.languageProvider.importQueries(queries, originMeta.id);
+  }
+
   metadataRequest(url) {
     // HACK to get label values for {job=|}, will be replaced when implementing LoggingQueryField
     const apiUrl = url.replace('v1', 'prom');

public/app/plugins/datasource/logging/language_provider.test.ts

@@ -0,0 +1,74 @@
+import Plain from 'slate-plain-serializer';
+
+import LanguageProvider from './language_provider';
+
+describe('Language completion provider', () => {
+  const datasource = {
+    metadataRequest: () => ({ data: { data: [] } }),
+  };
+
+  it('returns default suggestions on emtpty context', () => {
+    const instance = new LanguageProvider(datasource);
+    const result = instance.provideCompletionItems({ text: '', prefix: '', wrapperClasses: [] });
+    expect(result.context).toBeUndefined();
+    expect(result.refresher).toBeUndefined();
+    expect(result.suggestions.length).toEqual(0);
+  });
+
+  describe('label suggestions', () => {
+    it('returns default label suggestions on label context', () => {
+      const instance = new LanguageProvider(datasource);
+      const value = Plain.deserialize('{}');
+      const range = value.selection.merge({
+        anchorOffset: 1,
+      });
+      const valueWithSelection = value.change().select(range).value;
+      const result = instance.provideCompletionItems({
+        text: '',
+        prefix: '',
+        wrapperClasses: ['context-labels'],
+        value: valueWithSelection,
+      });
+      expect(result.context).toBe('context-labels');
+      expect(result.suggestions).toEqual([{ items: [{ label: 'job' }, { label: 'namespace' }], label: 'Labels' }]);
+    });
+  });
+});
+
+describe('Query imports', () => {
+  const datasource = {
+    metadataRequest: () => ({ data: { data: [] } }),
+  };
+
+  it('returns empty queries for unknown origin datasource', async () => {
+    const instance = new LanguageProvider(datasource);
+    const result = await instance.importQueries([{ refId: 'bar', expr: 'foo' }], 'unknown');
+    expect(result).toEqual([{ refId: 'bar', expr: '' }]);
+  });
+
+  describe('prometheus query imports', () => {
+    it('returns empty query from metric-only query', async () => {
+      const instance = new LanguageProvider(datasource);
+      const result = await instance.importPrometheusQuery('foo');
+      expect(result).toEqual('');
+    });
+
+    it('returns empty query from selector query if label is not available', async () => {
+      const datasourceWithLabels = {
+        metadataRequest: url => (url === '/api/prom/label' ? { data: { data: ['other'] } } : { data: { data: [] } }),
+      };
+      const instance = new LanguageProvider(datasourceWithLabels);
+      const result = await instance.importPrometheusQuery('{foo="bar"}');
+      expect(result).toEqual('{}');
+    });
+
+    it('returns selector query from selector query with common labels', async () => {
+      const datasourceWithLabels = {
+        metadataRequest: url => (url === '/api/prom/label' ? { data: { data: ['foo'] } } : { data: { data: [] } }),
+      };
+      const instance = new LanguageProvider(datasourceWithLabels);
+      const result = await instance.importPrometheusQuery('metric{foo="bar",baz="42"}');
+      expect(result).toEqual('{foo="bar"}');
+    });
+  });
+});

public/app/plugins/datasource/logging/language_provider.ts

@@ -8,9 +8,9 @@ import {
   TypeaheadInput,
   TypeaheadOutput,
 } from 'app/types/explore';
-
-import { parseSelector } from 'app/plugins/datasource/prometheus/language_utils';
+import { parseSelector, labelRegexp, selectorRegexp } from 'app/plugins/datasource/prometheus/language_utils';
 import PromqlSyntax from 'app/plugins/datasource/prometheus/promql';
+import { DataQuery } from 'app/types';
 
 const DEFAULT_KEYS = ['job', 'namespace'];
 const EMPTY_SELECTOR = '{}';
@@ -158,6 +158,56 @@ export default class LoggingLanguageProvider extends LanguageProvider {
     return { context, refresher, suggestions };
   }
 
+  async importQueries(queries: DataQuery[], datasourceType: string): Promise<DataQuery[]> {
+    if (datasourceType === 'prometheus') {
+      return Promise.all(
+        queries.map(async query => {
+          const expr = await this.importPrometheusQuery(query.expr);
+          return {
+            ...query,
+            expr,
+          };
+        })
+      );
+    }
+    return queries.map(query => ({
+      ...query,
+      expr: '',
+    }));
+  }
+
+  async importPrometheusQuery(query: string): Promise<string> {
+    // Consider only first selector in query
+    const selectorMatch = query.match(selectorRegexp);
+    if (selectorMatch) {
+      const selector = selectorMatch[0];
+      const labels = {};
+      selector.replace(labelRegexp, (_, key, operator, value) => {
+        labels[key] = { value, operator };
+        return '';
+      });
+
+      // Keep only labels that exist on origin and target datasource
+      await this.start(); // fetches all existing label keys
+      const commonLabels = {};
+      for (const key in labels) {
+        const existingKeys = this.labelKeys[EMPTY_SELECTOR];
+        if (existingKeys.indexOf(key) > -1) {
+          // Should we check for label value equality here?
+          commonLabels[key] = labels[key];
+        }
+      }
+      const labelKeys = Object.keys(commonLabels).sort();
+      const cleanSelector = labelKeys
+        .map(key => `${key}${commonLabels[key].operator}${commonLabels[key].value}`)
+        .join(',');
+
+      return ['{', cleanSelector, '}'].join('');
+    }
+
+    return '';
+  }
+
   async fetchLogLabels() {
     const url = '/api/prom/label';
     try {

public/app/plugins/datasource/prometheus/language_utils.ts

@@ -24,8 +24,8 @@ export function processLabels(labels, withName = false) {
 }
 
 // const cleanSelectorRegexp = /\{(\w+="[^"\n]*?")(,\w+="[^"\n]*?")*\}/;
-const selectorRegexp = /\{[^}]*?\}/;
-const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")/g;
+export const selectorRegexp = /\{[^}]*?\}/;
+export const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")/g;
 export function parseSelector(query: string, cursorOffset = 1): { labelKeys: any[]; selector: string } {
   if (!query.match(selectorRegexp)) {
     // Special matcher for metrics

public/app/types/datasources.ts

@@ -18,8 +18,6 @@ export interface DataSource {
   readOnly: boolean;
   meta?: PluginMeta;
   pluginExports?: PluginExports;
-  init?: () => void;
-  testDatasource?: () => Promise<any>;
 }
 
 export interface DataSourcesState {

public/app/types/series.ts

@@ -1,4 +1,5 @@
 import { Moment } from 'moment';
+import { PluginMeta } from './plugins';
 
 export enum LoadingState {
   NotStarted = 'NotStarted',
@@ -70,6 +71,7 @@ export interface DataQueryResponse {
 
 export interface DataQuery {
   refId: string;
+  [key: string]: any;
 }
 
 export interface DataQueryOptions {
@@ -87,5 +89,14 @@ export interface DataQueryOptions {
 }
 
 export interface DataSourceApi {
+  /**
+   * Imports queries from a different datasource
+   */
+  importQueries?(queries: DataQuery[], originMeta: PluginMeta): Promise<DataQuery[]>;
+  /**
+   * Initializes a datasource after instantiation
+   */
+  init?: () => void;
   query(options: DataQueryOptions): Promise<DataQueryResponse>;
+  testDatasource?: () => Promise<any>;
 }