Merge pull request #15012 from grafana/loki-query-editor

WIP: Loki query editor for dashboard panels

packages/grafana-ui/src/types/datasource.ts

@@ -3,7 +3,7 @@ import { PluginMeta } from './plugin';
 import { TableData, TimeSeries } from './data';
 
 export interface DataQueryResponse {
-  data: TimeSeries[] | [TableData];
+  data: TimeSeries[] | [TableData] | any;
 }
 
 export interface DataQuery {

packages/grafana-ui/src/types/plugin.ts

@@ -44,8 +44,8 @@ export interface DataSourceApi<TQuery extends DataQuery = DataQuery> {
 export interface QueryEditorProps<DSType extends DataSourceApi, TQuery extends DataQuery> {
   datasource: DSType;
   query: TQuery;
-  onExecuteQuery?: () => void;
-  onQueryChange?: (value: TQuery) => void;
+  onRunQuery: () => void;
+  onChange: (value: TQuery) => void;
 }
 
 export interface PluginExports {

public/app/core/components/Select/MetricSelect.tsx

@@ -1,8 +1,7 @@
 import React from 'react';
 import _ from 'lodash';
 
-import { Select } from '@grafana/ui';
-import { SelectOptionItem } from '@grafana/ui';
+import { Select, SelectOptionItem } from '@grafana/ui';
 import { Variable } from 'app/types/templates';
 
 export interface Props {

public/app/core/logs_model.ts

@@ -1,7 +1,6 @@
 import _ from 'lodash';
-import { colors } from '@grafana/ui';
 
-import { TimeSeries } from 'app/core/core';
+import { colors, TimeSeries } from '@grafana/ui';
 import { getThemeColor } from 'app/core/utils/colors';
 
 /**
@@ -341,6 +340,6 @@ export function makeSeriesForLogs(rows: LogRowModel[], intervalMs: number): Time
       return a[1] - b[1];
     });
 
-    return new TimeSeries(series);
+    return { datapoints: series.datapoints, target: series.alias, color: series.color };
   });
 }

public/app/features/dashboard/panel_editor/QueriesTab.tsx

@@ -165,6 +165,11 @@ export class QueriesTab extends PureComponent<Props, State> {
     this.setState({ isAddingMixed: false });
   };
 
+  onQueryChange = (query: DataQuery, index) => {
+    this.props.panel.changeQuery(query, index);
+    this.forceUpdate();
+  };
+
   setScrollTop = (event: React.MouseEvent<HTMLElement>) => {
     const target = event.target as HTMLElement;
     this.setState({ scrollTop: target.scrollTop });
@@ -201,6 +206,7 @@ export class QueriesTab extends PureComponent<Props, State> {
                 key={query.refId}
                 panel={panel}
                 query={query}
+                onChange={query => this.onQueryChange(query, index)}
                 onRemoveQuery={this.onRemoveQuery}
                 onAddQuery={this.onAddQuery}
                 onMoveQuery={this.onMoveQuery}

public/app/features/dashboard/panel_editor/QueryEditorRow.tsx

@@ -18,6 +18,7 @@ interface Props {
   onAddQuery: (query?: DataQuery) => void;
   onRemoveQuery: (query: DataQuery) => void;
   onMoveQuery: (query: DataQuery, direction: number) => void;
+  onChange: (query: DataQuery) => void;
   dataSourceValue: string | null;
   inMixedMode: boolean;
 }
@@ -105,17 +106,12 @@ export class QueryEditorRow extends PureComponent<Props, State> {
     this.setState({ isCollapsed: !this.state.isCollapsed });
   };
 
-  onQueryChange = (query: DataQuery) => {
-    Object.assign(this.props.query, query);
-    this.onExecuteQuery();
-  };
-
-  onExecuteQuery = () => {
+  onRunQuery = () => {
     this.props.panel.refresh();
   };
 
   renderPluginEditor() {
-    const { query } = this.props;
+    const { query, onChange } = this.props;
     const { datasource } = this.state;
 
     if (datasource.pluginExports.QueryCtrl) {
@@ -128,8 +124,8 @@ export class QueryEditorRow extends PureComponent<Props, State> {
         <QueryEditor
           query={query}
           datasource={datasource}
-          onQueryChange={this.onQueryChange}
-          onExecuteQuery={this.onExecuteQuery}
+          onChange={onChange}
+          onRunQuery={this.onRunQuery}
         />
       );
     }
@@ -166,7 +162,7 @@ export class QueryEditorRow extends PureComponent<Props, State> {
 
   onDisableQuery = () => {
     this.props.query.hide = !this.props.query.hide;
-    this.onExecuteQuery();
+    this.onRunQuery();
     this.forceUpdate();
   };
 

public/app/features/dashboard/state/PanelModel.ts

@@ -269,6 +269,19 @@ export class PanelModel {
     });
   }
 
+  changeQuery(query: DataQuery, index: number) {
+    // ensure refId is maintained
+    query.refId = this.targets[index].refId;
+
+    // update query in array
+    this.targets = this.targets.map((item, itemIndex) => {
+      if (itemIndex === index) {
+        return query;
+      }
+      return item;
+    });
+  }
+
   destroy() {
     this.events.emit('panel-teardown');
     this.events.removeAllListeners();

public/app/features/explore/Explore.tsx

@@ -216,7 +216,7 @@ export class Explore extends React.PureComponent<ExploreProps> {
                       {showingStartPage && <StartPage onClickExample={this.onClickExample} />}
                       {!showingStartPage && (
                         <>
-                          {supportsGraph && <GraphContainer exploreId={exploreId} />}
+                          {supportsGraph && !supportsLogs && <GraphContainer exploreId={exploreId} />}
                           {supportsTable && <TableContainer exploreId={exploreId} onClickCell={this.onClickLabel} />}
                           {supportsLogs && (
                             <LogsContainer

public/app/features/explore/Logs.tsx

@@ -3,6 +3,8 @@ import React, { PureComponent } from 'react';
 
 import * as rangeUtil from 'app/core/utils/rangeutil';
 import { RawTimeRange, Switch } from '@grafana/ui';
+import TimeSeries from 'app/core/time_series2';
+
 import {
   LogsDedupDescription,
   LogsDedupStrategy,
@@ -205,12 +207,13 @@ export default class Logs extends PureComponent<Props, State> {
 
     // React profiler becomes unusable if we pass all rows to all rows and their labels, using getter instead
     const getRows = () => processedRows;
+    const timeSeries = data.series.map(series => new TimeSeries(series));
 
     return (
       <div className="logs-panel">
         <div className="logs-panel-graph">
           <Graph
-            data={data.series}
+            data={timeSeries}
             height="100px"
             range={range}
             id={`explore-logs-graph-${exploreId}`}

public/app/features/explore/QueryField.tsx

@@ -104,11 +104,20 @@ export class QueryField extends React.PureComponent<QueryFieldProps, QueryFieldS
   }
 
   componentDidUpdate(prevProps: QueryFieldProps, prevState: QueryFieldState) {
+    const { initialQuery, syntax } = this.props;
+    const { value, suggestions } = this.state;
+
+    // if query changed from the outside
+    if (initialQuery !== prevProps.initialQuery) {
+      // and we have a version that differs
+      if (initialQuery !== Plain.serialize(value)) {
+        this.placeholdersBuffer = new PlaceholdersBuffer(initialQuery || '');
+        this.setState({ value: makeValue(this.placeholdersBuffer.toString(), syntax) });
+      }
+    }
+
     // Only update menu location when suggestion existence or text/selection changed
-    if (
-      this.state.value !== prevState.value ||
-      hasSuggestions(this.state.suggestions) !== hasSuggestions(prevState.suggestions)
-    ) {
+    if (value !== prevState.value || hasSuggestions(suggestions) !== hasSuggestions(prevState.suggestions)) {
       this.updateMenu();
     }
   }

public/app/features/explore/QueryRow.tsx

@@ -65,6 +65,10 @@ export class QueryRow extends PureComponent<QueryRowProps> {
     }
   };
 
+  componentWillUnmount() {
+    console.log('QueryRow will unmount');
+  }
+
   onClickAddButton = () => {
     const { exploreId, index } = this.props;
     this.props.addQueryRow(exploreId, index);

public/app/plugins/datasource/loki/components/LokiQueryEditor.tsx

@@ -0,0 +1,80 @@
+// Libraries
+import React, { PureComponent } from 'react';
+
+// Components
+import { Select, SelectOptionItem } from '@grafana/ui';
+
+// Types
+import { QueryEditorProps } from '@grafana/ui/src/types';
+import { LokiDatasource } from '../datasource';
+import { LokiQuery } from '../types';
+import { LokiQueryField } from './LokiQueryField';
+
+type Props = QueryEditorProps<LokiDatasource, LokiQuery>;
+
+interface State {
+  query: LokiQuery;
+}
+
+export class LokiQueryEditor extends PureComponent<Props> {
+  state: State = {
+    query: this.props.query,
+  };
+
+  onRunQuery = () => {
+    const { query } = this.state;
+
+    this.props.onChange(query);
+    this.props.onRunQuery();
+  };
+
+  onFieldChange = (query: LokiQuery, override?) => {
+    this.setState({
+      query: {
+        ...this.state.query,
+        expr: query.expr,
+      }
+    });
+  };
+
+  onFormatChanged = (option: SelectOptionItem) => {
+    this.props.onChange({
+      ...this.state.query,
+      resultFormat: option.value,
+    });
+  };
+
+  render() {
+    const { query } = this.state;
+    const { datasource } = this.props;
+    const formatOptions: SelectOptionItem[] = [
+      { label: 'Time Series', value: 'time_series' },
+      { label: 'Table', value: 'table' },
+    ];
+
+    query.resultFormat = query.resultFormat || 'time_series';
+    const currentFormat = formatOptions.find(item => item.value === query.resultFormat);
+
+    return (
+      <div>
+        <LokiQueryField
+          datasource={datasource}
+          initialQuery={query}
+          onQueryChange={this.onFieldChange}
+          onPressEnter={this.onRunQuery}
+        />
+        <div className="gf-form-inline">
+          <div className="gf-form">
+            <div className="gf-form-label">Format as</div>
+            <Select isSearchable={false} options={formatOptions} onChange={this.onFormatChanged} value={currentFormat} />
+          </div>
+          <div className="gf-form gf-form--grow">
+            <div className="gf-form-label gf-form-label--grow" />
+          </div>
+        </div>
+      </div>
+    );
+  }
+}
+
+export default LokiQueryEditor;

public/app/plugins/datasource/loki/components/LokiQueryField.tsx

@@ -12,6 +12,7 @@ import QueryField, { TypeaheadInput, QueryFieldState } from 'app/features/explor
 import { getNextCharacter, getPreviousCousin } from 'app/features/explore/utils/dom';
 import BracesPlugin from 'app/features/explore/slate-plugins/braces';
 import RunnerPlugin from 'app/features/explore/slate-plugins/runner';
+import LokiDatasource from '../datasource';
 
 // Types
 import { LokiQuery } from '../types';
@@ -65,7 +66,7 @@ interface CascaderOption {
 }
 
 interface LokiQueryFieldProps {
-  datasource: any;
+  datasource: LokiDatasource;
   error?: string | JSX.Element;
   hint?: any;
   history?: any[];
@@ -80,7 +81,7 @@ interface LokiQueryFieldState {
   syntaxLoaded: boolean;
 }
 
-class LokiQueryField extends React.PureComponent<LokiQueryFieldProps, LokiQueryFieldState> {
+export class LokiQueryField extends React.PureComponent<LokiQueryFieldProps, LokiQueryFieldState> {
   plugins: any[];
   pluginsSearch: any[];
   languageProvider: any;

public/app/plugins/datasource/loki/datasource.test.ts

@@ -7,6 +7,17 @@ describe('LokiDatasource', () => {
     url: 'myloggingurl',
   };
 
+  const testResp = {
+    data: {
+      streams: [
+        {
+          entries: [{ ts: '2019-02-01T10:27:37.498180581Z', line: 'hello' }],
+          labels: '{}',
+        },
+      ],
+    },
+  };
+
   describe('when querying', () => {
     const backendSrvMock = { datasourceRequest: jest.fn() };
 
@@ -17,7 +28,7 @@ describe('LokiDatasource', () => {
 
     test('should use default max lines when no limit given', () => {
       const ds = new LokiDatasource(instanceSettings, backendSrvMock, templateSrvMock);
-      backendSrvMock.datasourceRequest = jest.fn();
+      backendSrvMock.datasourceRequest = jest.fn(() => Promise.resolve(testResp));
       const options = getQueryOptions<LokiQuery>({ targets: [{ expr: 'foo', refId: 'B' }] });
 
       ds.query(options);
@@ -30,7 +41,7 @@ describe('LokiDatasource', () => {
       const customData = { ...(instanceSettings.jsonData || {}), maxLines: 20 };
       const customSettings = { ...instanceSettings, jsonData: customData };
       const ds = new LokiDatasource(customSettings, backendSrvMock, templateSrvMock);
-      backendSrvMock.datasourceRequest = jest.fn();
+      backendSrvMock.datasourceRequest = jest.fn(() => Promise.resolve(testResp));
 
       const options = getQueryOptions<LokiQuery>({ targets: [{ expr: 'foo', refId: 'B' }] });
       ds.query(options);
@@ -38,6 +49,34 @@ describe('LokiDatasource', () => {
       expect(backendSrvMock.datasourceRequest.mock.calls.length).toBe(1);
       expect(backendSrvMock.datasourceRequest.mock.calls[0][0].url).toContain('limit=20');
     });
+
+    test('should return log streams when resultFormat is undefined', async done => {
+      const ds = new LokiDatasource(instanceSettings, backendSrvMock, templateSrvMock);
+      backendSrvMock.datasourceRequest = jest.fn(() => Promise.resolve(testResp));
+
+      const options = getQueryOptions<LokiQuery>({
+        targets: [{ expr: 'foo', refId: 'B' }],
+      });
+
+      const res = await ds.query(options);
+
+      expect(res.data[0].entries[0].line).toBe('hello');
+      done();
+    });
+
+    test('should return time series when resultFormat is time_series', async done => {
+      const ds = new LokiDatasource(instanceSettings, backendSrvMock, templateSrvMock);
+      backendSrvMock.datasourceRequest = jest.fn(() => Promise.resolve(testResp));
+
+      const options = getQueryOptions<LokiQuery>({
+        targets: [{ expr: 'foo', refId: 'B', resultFormat: 'time_series' }],
+      });
+
+      const res = await ds.query(options);
+
+      expect(res.data[0].datapoints).toBeDefined();
+      done();
+    });
   });
 
   describe('when performing testDataSource', () => {

public/app/plugins/datasource/loki/datasource.ts

@@ -32,7 +32,7 @@ function serializeParams(data: any) {
     .join('&');
 }
 
-export default class LokiDatasource {
+export class LokiDatasource {
   languageProvider: LanguageProvider;
   maxLines: number;
 
@@ -73,10 +73,11 @@ export default class LokiDatasource {
     };
   }
 
-  query(options: DataQueryOptions<LokiQuery>): Promise<{ data: LogsStream[] }> {
+  async query(options: DataQueryOptions<LokiQuery>) {
     const queryTargets = options.targets
-      .filter(target => target.expr)
+      .filter(target => target.expr && !target.hide)
       .map(target => this.prepareQueryTarget(target, options));
+
     if (queryTargets.length === 0) {
       return Promise.resolve({ data: [] });
     }
@@ -84,20 +85,29 @@ export default class LokiDatasource {
     const queries = queryTargets.map(target => this._request('/api/prom/query', target));
 
     return Promise.all(queries).then((results: any[]) => {
-      // Flatten streams from multiple queries
-      const allStreams: LogsStream[] = results.reduce((acc, response, i) => {
-        if (!response) {
-          return acc;
+      const allStreams: LogsStream[] = [];
+
+      for (let i = 0; i < results.length; i++) {
+        const result = results[i];
+        const query = queryTargets[i];
+
+        // add search term to stream & add to array
+        if (result.data)  {
+          for (const stream of (result.data.streams || [])) {
+            stream.search = query.regexp;
+            allStreams.push(stream);
+          }
         }
-        const streams: LogsStream[] = response.data.streams || [];
-        // Inject search for match highlighting
-        const search: string = queryTargets[i].regexp;
-        streams.forEach(s => {
-          s.search = search;
-        });
-        return [...acc, ...streams];
-      }, []);
-      return { data: allStreams };
+      }
+
+      // check resultType
+      if (options.targets[0].resultFormat === 'time_series') {
+        const logs = mergeStreamsToLogs(allStreams, this.maxLines);
+        logs.series = makeSeriesForLogs(logs.rows, options.intervalMs);
+        return { data: logs.series };
+      } else {
+        return { data: allStreams };
+      }
     });
   }
 
@@ -142,34 +152,36 @@ export default class LokiDatasource {
 
   testDatasource() {
     return this._request('/api/prom/label')
-      .then(res => {
-        if (res && res.data && res.data.values && res.data.values.length > 0) {
-          return { status: 'success', message: 'Data source connected and labels found.' };
-        }
-        return {
-          status: 'error',
-          message:
-            'Data source connected, but no labels received. Verify that Loki and Promtail is configured properly.',
-        };
-      })
-      .catch(err => {
-        let message = 'Loki: ';
-        if (err.statusText) {
-          message += err.statusText;
-        } else {
-          message += 'Cannot connect to Loki';
-        }
+    .then(res => {
+      if (res && res.data && res.data.values && res.data.values.length > 0) {
+        return { status: 'success', message: 'Data source connected and labels found.' };
+      }
+      return {
+        status: 'error',
+        message:
+          'Data source connected, but no labels received. Verify that Loki and Promtail is configured properly.',
+      };
+    })
+    .catch(err => {
+      let message = 'Loki: ';
+      if (err.statusText) {
+        message += err.statusText;
+      } else {
+        message += 'Cannot connect to Loki';
+      }
 
-        if (err.status) {
-          message += `. ${err.status}`;
-        }
+      if (err.status) {
+        message += `. ${err.status}`;
+      }
 
-        if (err.data && err.data.message) {
-          message += `. ${err.data.message}`;
-        } else if (err.data) {
-          message += `. ${err.data}`;
-        }
-        return { status: 'error', message: message };
-      });
+      if (err.data && err.data.message) {
+        message += `. ${err.data.message}`;
+      } else if (err.data) {
+        message += `. ${err.data}`;
+      }
+      return { status: 'error', message: message };
+    });
   }
 }
+
+export default LokiDatasource;

public/app/plugins/datasource/loki/module.ts

@@ -2,6 +2,7 @@ import Datasource from './datasource';
 
 import LokiStartPage from './components/LokiStartPage';
 import LokiQueryField from './components/LokiQueryField';
+import LokiQueryEditor from './components/LokiQueryEditor';
 
 export class LokiConfigCtrl {
   static templateUrl = 'partials/config.html';
@@ -9,6 +10,7 @@ export class LokiConfigCtrl {
 
 export {
   Datasource,
+  LokiQueryEditor as QueryEditor,
   LokiConfigCtrl as ConfigCtrl,
   LokiQueryField as ExploreQueryField,
   LokiStartPage as ExploreStartPage,

public/app/plugins/datasource/loki/plugin.json

@@ -2,12 +2,14 @@
   "type": "datasource",
   "name": "Loki",
   "id": "loki",
-  "metrics": false,
+
+  "metrics": true,
   "alerting": false,
   "annotations": false,
   "logs": true,
   "explore": true,
   "tables": false,
+
   "info": {
     "description": "Loki Logging Data Source for Grafana",
     "author": {

public/app/plugins/datasource/loki/types.ts

@@ -2,5 +2,8 @@ import { DataQuery } from '@grafana/ui/src/types';
 
 export interface LokiQuery extends DataQuery {
   expr: string;
+  resultFormat?: LokiQueryResultFormats;
 }
 
+export type LokiQueryResultFormats = 'time_series' | 'logs';
+

public/app/plugins/datasource/testdata/QueryEditor.tsx

@@ -41,9 +41,9 @@ export class QueryEditor extends PureComponent<Props> {
   }
 
   onScenarioChange = (item: SelectOptionItem) => {
-    this.props.onQueryChange({
+    this.props.onChange({
+      ...this.props.query,
       scenarioId: item.value,
-      ...this.props.query
     });
   }
 

public/sass/components/_slate_editor.scss

@@ -19,6 +19,7 @@
   border: $panel-border;
   border-radius: $border-radius;
   transition: all 0.3s;
+  line-height: $input-line-height;
 }
 
 .slate-query-field__wrapper--disabled {