Add details about stream in the query editor

dist/plugin.json

@@ -5,7 +5,7 @@
   "id": "parseable-datasource",
   "logs": true,
   "info": {
-    "description": "Parseable is a log observability platform that helps you collect, parse, and analyze your logs. This plugin allows you to query your logs in Grafana.",
+    "description": "Parseable is a logging platform that helps you collect, parse, and analyze your logs. This plugin allows you to visualize your Parseable logs in Grafana.",
     "author": {
       "name": "Parseable Team <[email protected]>"
     },
@@ -24,7 +24,7 @@
       {"name": "Log Dashboard", "path": "img/log-visualisation.png"}
     ],
     "version": "1.0.0",
-    "updated": "2023-01-10"
+    "updated": "2023-01-11"
   },
   "routes": [
     {

package.json

@@ -1,7 +1,7 @@
 {
   "name": "parseable",
   "version": "1.0.0",
-  "description": "Y",
+  "description": "Parseable is a logging platform that helps you collect, parse, and analyze your logs. This plugin allows you to visualize your Parseable logs in Grafana.",
   "scripts": {
     "build": "webpack -c ./.config/webpack/webpack.config.ts --env production",
     "dev": "webpack -w -c ./.config/webpack/webpack.config.ts --env development",
@@ -18,7 +18,7 @@
     "build-storybook": "build-storybook"
   },
   "author": "Parseable",
-  "license": "Affero General Public License v3.0",
+  "license": "AGPL-3.0-only",
   "devDependencies": {
     "@babel/core": "^7.16.7",
     "@grafana/e2e": "9.2.5",

releases/parseable-datasource-1.0.0.zip.sha1

@@ -1 +1 @@
-a106522fa8af91dc1277c63ee19a0a0b8bd3b3bf
+dd08ae43a151ee69e30f74592a34e9726373fc4d

src/components/QueryEditor.tsx

@@ -2,7 +2,7 @@ import React, { ComponentType, ChangeEvent, useState } from 'react';
 import { LegacyForms, AsyncSelect, Label, InlineField, InlineFieldRow } from '@grafana/ui';
 import { QueryEditorProps, SelectableValue } from '@grafana/data';
 import { DataSource } from '../datasource';
-import { Fields, MyDataSourceOptions, MyQuery } from '../types';
+import { SchemaFields, MyDataSourceOptions, MyQuery } from '../types';
 
 const { FormField } = LegacyForms;
 
@@ -30,14 +30,18 @@ export const QueryEditor: ComponentType<Props> = ({ datasource, onChange, onRunQ
 
   const [value, setValue] = useState<SelectableValue<string>>();
   const [schema = '', setSchema] = React.useState<string | number>();
+  const [count = '', setEventCount] = React.useState<string | number>();
+  const [jsonsize = '', setJsonSize] = React.useState<string | number>();
+  const [parquetsize = '', setParquetSize] = React.useState<string | number>();
+  const [streamname = '', setStreamName] = React.useState<string | number>();
   //const [fielder, setFielder] = React.useState<string | number>();
 
-  const loadSchemaOptions = React.useCallback((value) => {
+  const loadStreamSchema = React.useCallback((value) => {
     if (value) {
-      return datasource.listSchema(value).then(
+      return datasource.streamSchema(value).then(
         (result) => {
           if (result.fields) {
-            const schema = result.fields.map((data: Fields) => (data.name));
+            const schema = result.fields.map((data: SchemaFields) => (data.name));
             const schemaToText = schema.join(", ")
             setSchema(schemaToText);
             return schema;
@@ -52,6 +56,31 @@ export const QueryEditor: ComponentType<Props> = ({ datasource, onChange, onRunQ
     return '';
   }, [datasource, schema]);
 
+  const loadStreamStats = React.useCallback((value) => {
+    if (value) {
+      return datasource.streamStats(value).then(
+        (result) => {
+          if (result.ingestion) {
+            const count = result.ingestion.count;
+            const jsonsize = result.ingestion.size;
+            const parquetsize = result.storage?.size;
+            const streamname = result.stream;
+            setJsonSize(jsonsize);
+            setParquetSize(parquetsize);
+            setStreamName(streamname);
+            setEventCount(count);
+            return count;
+          }
+          return count;
+        },
+        (response) => {
+          throw new Error(response.statusText);
+        }
+      );
+    }
+    return '';
+  }, [datasource, count]);
+
   const onQueryTextChange = (event: ChangeEvent<HTMLInputElement>) => {
     onChange({ ...query, queryText: event.target.value });
   };
@@ -65,15 +94,21 @@ export const QueryEditor: ComponentType<Props> = ({ datasource, onChange, onRunQ
   }, [onRunQuery, queryText])
 
   React.useEffect(() => {
-    loadSchemaOptions(value)
-  }, [loadSchemaOptions, value]);
+    loadStreamSchema(value)
+  }, [loadStreamSchema, value]);
+
+  React.useEffect(() => {
+    loadStreamStats(value)
+  }, [loadStreamStats, value]);
 
   return (
     <>
       <div className="gf-form">
         <InlineField>
           <Label >
-            StreamName:
+            <div style={{ width: 'fit-content', color: 'blue' }}>
+            Select a log stream:
+            </div>
             <div style={{ width: 200 + 'px', marginRight: '20px', marginLeft: '20px' }}>
               <AsyncSelect
                 loadOptions={loadAsyncOptions}
@@ -89,8 +124,42 @@ export const QueryEditor: ComponentType<Props> = ({ datasource, onChange, onRunQ
 
         <InlineFieldRow>
           <div>
+          <Label>
+              <div style={{ width: 'fit-content', textAlign: 'center', color: 'blue'}}>
+                Stream {streamname} details
+              </div>
+            </Label>
+            <Label>
+              <div style={{ width: 'fit-content', color: 'blue' }}>
+                Columns: 
+              </div>
+              <div style={{ width: 'fit-content' }}>
+                { schema}
+              </div>
+            </Label>
+            <Label>
+              <div style={{ width: 'fit-content', color: 'blue' }}>
+                Total events ingested:             
+              </div>
+              <div style={{ width: 'fit-content' }}>
+                { count}
+              </div>
+            </Label>
+            <Label>
+              <div style={{ width: 'fit-content', color: 'blue' }}>
+                Total ingested data size:             
+              </div>
+              <div style={{ width: 'fit-content' }}>
+                { jsonsize}
+              </div>
+            </Label>
             <Label>
-              Schema: {schema}
+              <div style={{ width: 'fit-content', color: 'blue' }}>
+                Total compressed data stored:             
+              </div>
+              <div style={{ width: 'fit-content' }}>
+                { parquetsize}
+              </div>
             </Label>
           </div>
         </InlineFieldRow>
@@ -102,7 +171,7 @@ export const QueryEditor: ComponentType<Props> = ({ datasource, onChange, onRunQ
         value={queryText || ''}
         onChange={onQueryTextChange}
         label="SQL Query"
-        tooltip="Enter the search SQL query here."
+        tooltip="Enter the search SQL query here (use column names as displayed above)"
       />
     </>
   );

src/datasource.ts

@@ -20,7 +20,8 @@ import {
   StreamName,
   StreamList,
   //Fields,
-  ListSchemaResponse
+  StreamSchemaResponse,
+  StreamStatsResponse
 } from './types';
 export class DataSource extends DataSourceApi<MyQuery, MyDataSourceOptions> {
   url: string;
@@ -149,7 +150,32 @@ export class DataSource extends DataSourceApi<MyQuery, MyDataSourceOptions> {
     );
   }
 
-  async listSchema(streamname: StreamName): Promise<ListSchemaResponse> {
+  async streamStats(streamname: StreamName): Promise<StreamStatsResponse> {
+    if (streamname) {
+      return lastValueFrom(
+        this.doFetch({
+          url: this.url + '/api/v1/logstream/' + streamname.value + '/stats',
+          method: 'GET',
+        }).pipe(
+          map((response) =>
+            (typeof response.data === 'object' && !isNull(response.data))
+              ? response.data
+              : {}
+          ),
+          catchError((err) => {
+            console.error(err);
+            return of({
+              status: 'error',
+              message: err.statusText
+            })
+
+          }))
+      )
+    }
+    return {}
+  }
+
+  async streamSchema(streamname: StreamName): Promise<StreamSchemaResponse> {
     if (streamname) {
       return lastValueFrom(
         this.doFetch({

src/plugin.json

@@ -5,7 +5,7 @@
   "id": "parseable-datasource",
   "logs": true,
   "info": {
-    "description": "Parseable is a log observability platform that helps you collect, parse, and analyze your logs. This plugin allows you to query your logs in Grafana.",
+    "description": "Parseable is a logging platform that helps you collect, parse, and analyze your logs. This plugin allows you to visualize your Parseable logs in Grafana.",
     "author": {
       "name": "Parseable Team <[email protected]>"
     },

src/types.ts

@@ -53,20 +53,40 @@ export interface StreamName {
   value: String | Number;
 }
 
-export interface ListSchemaResponse {
+export interface StreamSchemaResponse {
   fields?: [] | undefined;
   status?: string;
   message?: string;
 }
 
-export interface Fields {
+export interface StreamStatsResponse {
+  ingestion?: Ingestion;
+  storage?: Storage;
+  time?: string;
+  stream?: string;
+  status?: string;
+  message?: string;
+}
+
+export interface SchemaFields {
   name?: string;
   data_type?: "Utf8";
   nullable?: boolean;
   dict_id?: number;
   dict_is_ordered?: boolean;
 }
 
+export interface Ingestion {
+  count?: number;
+  format?: string;
+  size?: string;
+}
+
+export interface Storage {
+  format?: string;
+  size?: string;
+}
+
 export interface Schema {
   schema?: string[];
 }