Explore: Add multiline syntax highlighting to query field

- the non-nested query field schema did not allow for multi-line
  highlighting
- added explicit code schema and a `makeValue` function that enforces
  the nested structure
- replaced vendored prism-slate adapter with official slate-prism
  package
- renamed language to syntax

package.json

@@ -173,6 +173,7 @@
     "rxjs": "^5.4.3",
     "slate": "^0.33.4",
     "slate-plain-serializer": "^0.5.10",
+    "slate-prism": "^0.5.0",
     "slate-react": "^0.12.4",
     "tether": "^1.4.0",
     "tether-drop": "https://github.com/torkelo/drop/tarball/master",

public/app/containers/Explore/PromQueryField.tsx

@@ -3,10 +3,11 @@ import moment from 'moment';
 import React from 'react';
 import { Value } from 'slate';
 import Cascader from 'rc-cascader';
+import PluginPrism from 'slate-prism';
+import Prism from 'prismjs';
 
 // dom also includes Element polyfills
 import { getNextCharacter, getPreviousCousin } from './utils/dom';
-import PluginPrism, { setPrismTokens } from './slate-plugins/prism/index';
 import PrismPromql, { FUNCTIONS } from './slate-plugins/prism/promql';
 import BracesPlugin from './slate-plugins/braces';
 import RunnerPlugin from './slate-plugins/runner';
@@ -27,7 +28,7 @@ const HISTOGRAM_SELECTOR = '{le!=""}'; // Returns all timeseries for histograms
 const HISTORY_ITEM_COUNT = 5;
 const HISTORY_COUNT_CUTOFF = 1000 * 60 * 60 * 24; // 24h
 const METRIC_MARK = 'metric';
-const PRISM_LANGUAGE = 'promql';
+const PRISM_SYNTAX = 'promql';
 export const RECORDING_RULES_GROUP = '__recording_rules__';
 
 export const wrapLabel = (label: string) => ({ label });
@@ -36,6 +37,15 @@ export const setFunctionMove = (suggestion: Suggestion): Suggestion => {
   return suggestion;
 };
 
+// Syntax highlighting
+Prism.languages[PRISM_SYNTAX] = PrismPromql;
+function setPrismTokens(language, field, values, alias = 'variable') {
+  Prism.languages[language][field] = {
+    alias,
+    pattern: new RegExp(`(?:^|\\s)(${values.join('|')})(?:$|\\s)`),
+  };
+}
+
 export function addHistoryMetadata(item: Suggestion, history: any[]): Suggestion {
   const cutoffTs = Date.now() - HISTORY_COUNT_CUTOFF;
   const historyForItem = history.filter(h => h.ts > cutoffTs && h.query === item.label);
@@ -164,7 +174,10 @@ class PromQueryField extends React.Component<PromQueryFieldProps, PromQueryField
     this.plugins = [
       BracesPlugin(),
       RunnerPlugin({ handler: props.onPressEnter }),
-      PluginPrism({ definition: PrismPromql, language: PRISM_LANGUAGE }),
+      PluginPrism({
+        onlyIn: node => node.type === 'code_block',
+        getSyntax: node => 'promql',
+      }),
     ];
 
     this.state = {
@@ -221,7 +234,7 @@ class PromQueryField extends React.Component<PromQueryFieldProps, PromQueryField
     if (!this.state.metrics) {
       return;
     }
-    setPrismTokens(PRISM_LANGUAGE, METRIC_MARK, this.state.metrics);
+    setPrismTokens(PRISM_SYNTAX, METRIC_MARK, this.state.metrics);
   };
 
   onTypeahead = (typeahead: TypeaheadInput): TypeaheadOutput => {

public/app/containers/Explore/QueryField.tsx

@@ -1,7 +1,7 @@
 import _ from 'lodash';
 import React from 'react';
 import ReactDOM from 'react-dom';
-import { Block, Change, Document, Text, Value } from 'slate';
+import { Change, Value } from 'slate';
 import { Editor } from 'slate-react';
 import Plain from 'slate-plain-serializer';
 
@@ -9,6 +9,7 @@ import ClearPlugin from './slate-plugins/clear';
 import NewlinePlugin from './slate-plugins/newline';
 
 import Typeahead from './Typeahead';
+import { makeFragment, makeValue } from './Value';
 
 export const TYPEAHEAD_DEBOUNCE = 300;
 
@@ -16,22 +17,6 @@ function flattenSuggestions(s: any[]): any[] {
   return s ? s.reduce((acc, g) => acc.concat(g.items), []) : [];
 }
 
-export const makeFragment = (text: string): Document => {
-  const lines = text.split('\n').map(line =>
-    Block.create({
-      type: 'paragraph',
-      nodes: [Text.create(line)],
-    })
-  );
-
-  const fragment = Document.create({
-    nodes: lines,
-  });
-  return fragment;
-};
-
-export const getInitialValue = (value: string): Value => Value.create({ document: makeFragment(value) });
-
 export interface Suggestion {
   /**
    * The label of this completion item. By default
@@ -113,6 +98,7 @@ interface TypeaheadFieldProps {
   onWillApplySuggestion?: (suggestion: string, state: TypeaheadFieldState) => string;
   placeholder?: string;
   portalPrefix?: string;
+  syntax?: string;
 }
 
 export interface TypeaheadFieldState {
@@ -156,7 +142,7 @@ class QueryField extends React.Component<TypeaheadFieldProps, TypeaheadFieldStat
       typeaheadIndex: 0,
       typeaheadPrefix: '',
       typeaheadText: '',
-      value: getInitialValue(props.initialValue || ''),
+      value: makeValue(props.initialValue || '', props.syntax),
     };
   }
 
@@ -175,7 +161,7 @@ class QueryField extends React.Component<TypeaheadFieldProps, TypeaheadFieldStat
   componentWillReceiveProps(nextProps) {
     // initialValue is null in case the user typed
     if (nextProps.initialValue !== null && nextProps.initialValue !== this.props.initialValue) {
-      this.setState({ value: getInitialValue(nextProps.initialValue) });
+      this.setState({ value: makeValue(nextProps.initialValue, nextProps.syntax) });
     }
   }
 
@@ -272,7 +258,7 @@ class QueryField extends React.Component<TypeaheadFieldProps, TypeaheadFieldStat
   }, TYPEAHEAD_DEBOUNCE);
 
   applyTypeahead(change: Change, suggestion: Suggestion): Change {
-    const { cleanText, onWillApplySuggestion } = this.props;
+    const { cleanText, onWillApplySuggestion, syntax } = this.props;
     const { typeaheadPrefix, typeaheadText } = this.state;
     let suggestionText = suggestion.insertText || suggestion.label;
     const move = suggestion.move || 0;
@@ -293,7 +279,7 @@ class QueryField extends React.Component<TypeaheadFieldProps, TypeaheadFieldStat
 
     // If new-lines, apply suggestion as block
     if (suggestionText.match(/\n/)) {
-      const fragment = makeFragment(suggestionText);
+      const fragment = makeFragment(suggestionText, syntax);
       return change
         .deleteBackward(backward)
         .deleteForward(forward)

public/app/containers/Explore/Value.ts

@@ -0,0 +1,41 @@
+import { Block, Document, Text, Value } from 'slate';
+
+const SCHEMA = {
+  blocks: {
+    paragraph: 'paragraph',
+    codeblock: 'code_block',
+    codeline: 'code_line',
+  },
+  inlines: {},
+  marks: {},
+};
+
+export const makeFragment = (text: string, syntax?: string) => {
+  const lines = text.split('\n').map(line =>
+    Block.create({
+      type: 'code_line',
+      nodes: [Text.create(line)],
+    })
+  );
+
+  const block = Block.create({
+    data: {
+      syntax,
+    },
+    type: 'code_block',
+    nodes: lines,
+  });
+
+  return Document.create({
+    nodes: [block],
+  });
+};
+
+export const makeValue = (text: string, syntax?: string) => {
+  const fragment = makeFragment(text, syntax);
+
+  return Value.create({
+    document: fragment,
+    SCHEMA,
+  });
+};

public/app/containers/Explore/slate-plugins/prism/index.tsx

@@ -1,123 +0,0 @@
-import React from 'react';
-import Prism from 'prismjs';
-
-const TOKEN_MARK = 'prism-token';
-
-export function setPrismTokens(language, field, values, alias = 'variable') {
-  Prism.languages[language][field] = {
-    alias,
-    pattern: new RegExp(`(?:^|\\s)(${values.join('|')})(?:$|\\s)`),
-  };
-}
-
-/**
- * Code-highlighting plugin based on Prism and
- * https://github.com/ianstormtaylor/slate/blob/master/examples/code-highlighting/index.js
- *
- * (Adapted to handle nested grammar definitions.)
- */
-
-export default function PrismPlugin({ definition, language }) {
-  if (definition) {
-    // Don't override exising modified definitions
-    Prism.languages[language] = Prism.languages[language] || definition;
-  }
-
-  return {
-    /**
-     * Render a Slate mark with appropiate CSS class names
-     *
-     * @param {Object} props
-     * @return {Element}
-     */
-
-    renderMark(props) {
-      const { children, mark } = props;
-      // Only apply spans to marks identified by this plugin
-      if (mark.type !== TOKEN_MARK) {
-        return undefined;
-      }
-      const className = `token ${mark.data.get('types')}`;
-      return <span className={className}>{children}</span>;
-    },
-
-    /**
-     * Decorate code blocks with Prism.js highlighting.
-     *
-     * @param {Node} node
-     * @return {Array}
-     */
-
-    decorateNode(node) {
-      if (node.type !== 'paragraph') {
-        return [];
-      }
-
-      const texts = node.getTexts().toArray();
-      const tstring = texts.map(t => t.text).join('\n');
-      const grammar = Prism.languages[language];
-      const tokens = Prism.tokenize(tstring, grammar);
-      const decorations = [];
-      let startText = texts.shift();
-      let endText = startText;
-      let startOffset = 0;
-      let endOffset = 0;
-      let start = 0;
-
-      function processToken(token, acc?) {
-        // Accumulate token types down the tree
-        const types = `${acc || ''} ${token.type || ''} ${token.alias || ''}`;
-
-        // Add mark for token node
-        if (typeof token === 'string' || typeof token.content === 'string') {
-          startText = endText;
-          startOffset = endOffset;
-
-          const content = typeof token === 'string' ? token : token.content;
-          const newlines = content.split('\n').length - 1;
-          const length = content.length - newlines;
-          const end = start + length;
-
-          let available = startText.text.length - startOffset;
-          let remaining = length;
-
-          endOffset = startOffset + remaining;
-
-          while (available < remaining) {
-            endText = texts.shift();
-            remaining = length - available;
-            available = endText.text.length;
-            endOffset = remaining;
-          }
-
-          // Inject marks from up the tree (acc) as well
-          if (typeof token !== 'string' || acc) {
-            const range = {
-              anchorKey: startText.key,
-              anchorOffset: startOffset,
-              focusKey: endText.key,
-              focusOffset: endOffset,
-              marks: [{ type: TOKEN_MARK, data: { types } }],
-            };
-
-            decorations.push(range);
-          }
-
-          start = end;
-        } else if (token.content && token.content.length) {
-          // Tokens can be nested
-          for (const subToken of token.content) {
-            processToken(subToken, types);
-          }
-        }
-      }
-
-      // Process top-level tokens
-      for (const token of tokens) {
-        processToken(token);
-      }
-
-      return decorations;
-    },
-  };
-}

yarn.lock

@@ -9322,7 +9322,7 @@ pretty-format@^23.6.0:
     ansi-regex "^3.0.0"
     ansi-styles "^3.2.0"
 
-prismjs@^1.6.0:
+prismjs@^1.13.0, prismjs@^1.6.0:
   version "1.15.0"
   resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.15.0.tgz#8801d332e472091ba8def94976c8877ad60398d9"
   optionalDependencies:
@@ -10736,6 +10736,12 @@ slate-plain-serializer@^0.5.10, slate-plain-serializer@^0.5.17:
   dependencies:
     slate-dev-logger "^0.1.43"
 
+slate-prism@^0.5.0:
+  version "0.5.0"
+  resolved "http://registry.npmjs.org/slate-prism/-/slate-prism-0.5.0.tgz#009eb74fea38ad76c64db67def7ea0884917adec"
+  dependencies:
+    prismjs "^1.13.0"
+
 slate-prop-types@^0.4.34:
   version "0.4.61"
   resolved "https://registry.yarnpkg.com/slate-prop-types/-/slate-prop-types-0.4.61.tgz#141c109bed81b130dd03ab86dd7541b28d6d962a"