import _ from 'lodash'; import { TimeSeries } from 'app/core/core'; import colors, { getThemeColor } from 'app/core/utils/colors'; export enum LogLevel { crit = 'critical', critical = 'critical', warn = 'warning', warning = 'warning', err = 'error', error = 'error', info = 'info', debug = 'debug', trace = 'trace', unkown = 'unkown', } export const LogLevelColor = { [LogLevel.critical]: colors[7], [LogLevel.warning]: colors[1], [LogLevel.error]: colors[4], [LogLevel.info]: colors[0], [LogLevel.debug]: colors[5], [LogLevel.trace]: colors[2], [LogLevel.unkown]: getThemeColor('#8e8e8e', '#dde4ed'), }; export interface LogSearchMatch { start: number; length: number; text: string; } export interface LogRow { duplicates?: number; entry: string; key: string; // timestamp + labels labels: LogsStreamLabels; logLevel: LogLevel; searchWords?: string[]; timestamp: string; // ISO with nanosec precision timeFromNow: string; timeEpochMs: number; timeLocal: string; uniqueLabels?: LogsStreamLabels; } export interface LogsLabelStat { active?: boolean; count: number; proportion: number; value: string; } export enum LogsMetaKind { Number, String, LabelsMap, } export interface LogsMetaItem { label: string; value: string | number | LogsStreamLabels; kind: LogsMetaKind; } export interface LogsModel { id: string; // Identify one logs result from another meta?: LogsMetaItem[]; rows: LogRow[]; series?: TimeSeries[]; } export interface LogsStream { labels: string; entries: LogsStreamEntry[]; search?: string; parsedLabels?: LogsStreamLabels; uniqueLabels?: LogsStreamLabels; } export interface LogsStreamEntry { line: string; timestamp: string; } export interface LogsStreamLabels { [key: string]: string; } export enum LogsDedupStrategy { none = 'none', exact = 'exact', numbers = 'numbers', signature = 'signature', } export interface LogsParser { /** * Value-agnostic matcher for a field label. * Used to filter rows, and first capture group contains the value. */ buildMatcher: (label: string) => RegExp; /** * Regex to find a field in the log line. * First capture group contains the label value, second capture group the value. */ fieldRegex: RegExp; /** * Function to verify if this is a valid parser for the given line. * The parser accepts the line unless it returns undefined. */ test: (line: string) => any; } export const LogsParsers: { [name: string]: LogsParser } = { JSON: { buildMatcher: label => new RegExp(`(?:{|,)\\s*"${label}"\\s*:\\s*"([^"]*)"`), fieldRegex: /"(\w+)"\s*:\s*"([^"]*)"/, test: line => { try { return JSON.parse(line); } catch (error) {} }, }, logfmt: { buildMatcher: label => new RegExp(`(?:^|\\s)${label}=("[^"]*"|\\S+)`), fieldRegex: /(?:^|\s)(\w+)=("[^"]*"|\S+)/, test: line => LogsParsers.logfmt.fieldRegex.test(line), }, }; export function calculateFieldStats(rows: LogRow[], extractor: RegExp): LogsLabelStat[] { // Consider only rows that satisfy the matcher const rowsWithField = rows.filter(row => extractor.test(row.entry)); const rowCount = rowsWithField.length; // Get field value counts for eligible rows const countsByValue = _.countBy(rowsWithField, row => (row as LogRow).entry.match(extractor)[1]); const sortedCounts = _.chain(countsByValue) .map((count, value) => ({ count, value, proportion: count / rowCount })) .sortBy('count') .reverse() .value(); return sortedCounts; } export function calculateLogsLabelStats(rows: LogRow[], label: string): LogsLabelStat[] { // Consider only rows that have the given label const rowsWithLabel = rows.filter(row => row.labels[label] !== undefined); const rowCount = rowsWithLabel.length; // Get label value counts for eligible rows const countsByValue = _.countBy(rowsWithLabel, row => (row as LogRow).labels[label]); const sortedCounts = _.chain(countsByValue) .map((count, value) => ({ count, value, proportion: count / rowCount })) .sortBy('count') .reverse() .value(); return sortedCounts; } const isoDateRegexp = /\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-6]\d[,\.]\d+([+-][0-2]\d:[0-5]\d|Z)/g; function isDuplicateRow(row: LogRow, other: LogRow, strategy: LogsDedupStrategy): boolean { switch (strategy) { case LogsDedupStrategy.exact: // Exact still strips dates return row.entry.replace(isoDateRegexp, '') === other.entry.replace(isoDateRegexp, ''); case LogsDedupStrategy.numbers: return row.entry.replace(/\d/g, '') === other.entry.replace(/\d/g, ''); case LogsDedupStrategy.signature: return row.entry.replace(/\w/g, '') === other.entry.replace(/\w/g, ''); default: return false; } } export function dedupLogRows(logs: LogsModel, strategy: LogsDedupStrategy): LogsModel { if (strategy === LogsDedupStrategy.none) { return logs; } const dedupedRows = logs.rows.reduce((result: LogRow[], row: LogRow, index, list) => { const previous = result[result.length - 1]; if (index > 0 && isDuplicateRow(row, previous, strategy)) { previous.duplicates++; } else { row.duplicates = 0; result.push(row); } return result; }, []); return { ...logs, rows: dedupedRows, }; } export function getParser(line: string): LogsParser { let parser; try { if (LogsParsers.JSON.test(line)) { parser = LogsParsers.JSON; } } catch (error) {} if (!parser && LogsParsers.logfmt.test(line)) { parser = LogsParsers.logfmt; } return parser; } export function filterLogLevels(logs: LogsModel, hiddenLogLevels: Set): LogsModel { if (hiddenLogLevels.size === 0) { return logs; } const filteredRows = logs.rows.reduce((result: LogRow[], row: LogRow, index, list) => { if (!hiddenLogLevels.has(row.logLevel)) { result.push(row); } return result; }, []); return { ...logs, rows: filteredRows, }; } export function makeSeriesForLogs(rows: LogRow[], intervalMs: number): TimeSeries[] { // currently interval is rangeMs / resolution, which is too low for showing series as bars. // need at least 10px per bucket, so we multiply interval by 10. Should be solved higher up the chain // when executing queries & interval calculated and not here but this is a temporary fix. // intervalMs = intervalMs * 10; // Graph time series by log level const seriesByLevel = {}; const bucketSize = intervalMs * 10; for (const row of rows) { if (!seriesByLevel[row.logLevel]) { seriesByLevel[row.logLevel] = { lastTs: null, datapoints: [], alias: row.logLevel }; } const levelSeries = seriesByLevel[row.logLevel]; // Bucket to nearest minute const time = Math.round(row.timeEpochMs / bucketSize) * bucketSize; // Entry for time if (time === levelSeries.lastTs) { levelSeries.datapoints[levelSeries.datapoints.length - 1][0]++; } else { levelSeries.datapoints.push([1, time]); levelSeries.lastTs = time; } } return Object.keys(seriesByLevel).reduce((acc, level) => { if (seriesByLevel[level]) { const gs = new TimeSeries(seriesByLevel[level]); gs.setColor(LogLevelColor[level]); acc.push(gs); } return acc; }, []); }