feat(webui): Add compression job submission for local files only. (#1294)

Author: junhaoliao

components/clp-package-utils/clp_package_utils/controller.py

@@ -488,6 +488,7 @@ def _set_up_env_for_webui(self, container_clp_config: ClpConfig) -> EnvVarsDict:
         client_settings_json_updates = {
             "ClpStorageEngine": self._clp_config.package.storage_engine,
             "ClpQueryEngine": self._clp_config.package.query_engine,
+            "LogsInputType": self._clp_config.logs_input.type,
             "MongoDbSearchResultsMetadataCollectionName": (
                 self._clp_config.webui.results_metadata_collection_name
             ),

components/webui/client/public/settings.json

@@ -1,6 +1,7 @@
 {
     "ClpStorageEngine": "clp-s",
     "ClpQueryEngine": "clp-s",
+    "LogsInputType": "fs",
     "MongoDbSearchResultsMetadataCollectionName": "results-metadata",
     "SqlDbClpArchivesTableName": "clp_archives",
     "SqlDbClpDatasetsTableName": "clp_datasets",

components/webui/client/src/api/compress/index.ts

@@ -0,0 +1,22 @@
+import {
+    CompressionJob,
+    CompressionJobCreation,
+} from "@webui/common/schemas/compression";
+import axios from "axios";
+
+
+/**
+ * Submits a compression job.
+ *
+ * @param payload
+ * @return
+ */
+const submitCompressionJob = async (payload: CompressionJobCreation): Promise<CompressionJob> => {
+    console.log("Submitting compression job:", JSON.stringify(payload));
+    const {data} = await axios.post<CompressionJob>("/api/compress", payload);
+
+    return data;
+};
+
+
+export {submitCompressionJob};

components/webui/client/src/config/index.ts

@@ -1,16 +1,13 @@
-import {CLP_QUERY_ENGINES} from "@webui/common/config";
+import {
+    CLP_QUERY_ENGINES,
+    CLP_STORAGE_ENGINES,
+    STORAGE_TYPE,
+} from "@webui/common/config";
 
 import {settings} from "../settings";
 
 
-/**
- * CLP variants.
- */
-enum CLP_STORAGE_ENGINES {
-    CLP = "clp",
-    CLP_S = "clp-s",
-}
-
+const SETTINGS_LOGS_INPUT_TYPE = settings.LogsInputType as STORAGE_TYPE;
 const SETTINGS_STORAGE_ENGINE = settings.ClpStorageEngine as CLP_STORAGE_ENGINES;
 const SETTINGS_QUERY_ENGINE = settings.ClpQueryEngine as CLP_QUERY_ENGINES;
 
@@ -22,8 +19,7 @@ const STREAM_TYPE = CLP_STORAGE_ENGINES.CLP === SETTINGS_STORAGE_ENGINE ?
     "json";
 
 export {
-    CLP_QUERY_ENGINES,
-    CLP_STORAGE_ENGINES,
+    SETTINGS_LOGS_INPUT_TYPE,
     SETTINGS_QUERY_ENGINE,
     SETTINGS_STORAGE_ENGINE,
     STREAM_TYPE,

components/webui/client/src/pages/IngestPage/Compress/ClpSFormItems.tsx

@@ -0,0 +1,40 @@
+import {CLP_DEFAULT_DATASET_NAME} from "@webui/common/config";
+import {
+    Form,
+    Input,
+} from "antd";
+
+
+const DATASET_HELPER_TEXT = `If left empty, dataset "${CLP_DEFAULT_DATASET_NAME}" will be used.`;
+const DATASET_PLACEHOLDER_TEXT = "The dataset for new archives";
+const TIMESTAMP_KEY_HELPER_TEXT = "If not provided, events will not have assigned" +
+    " timestamps and can only be searched from the command line without a timestamp filter.";
+const TIMESTAMP_KEY_PLACEHOLDER_TEXT =
+    "The path (e.g. x.y) for the field containing the log event's timestamp";
+
+/**
+ * Renders additional compression job submission form items for CLP-S storage engine.
+ *
+ * @return
+ */
+const ClpSFormItems = () => (
+    <>
+        <Form.Item
+            label={"Dataset"}
+            name={"dataset"}
+            tooltip={DATASET_HELPER_TEXT}
+        >
+            <Input placeholder={DATASET_PLACEHOLDER_TEXT}/>
+        </Form.Item>
+        <Form.Item
+            label={"Timestamp Key"}
+            name={"timestampKey"}
+            tooltip={TIMESTAMP_KEY_HELPER_TEXT}
+        >
+            <Input placeholder={TIMESTAMP_KEY_PLACEHOLDER_TEXT}/>
+        </Form.Item>
+    </>
+);
+
+
+export default ClpSFormItems;

components/webui/client/src/pages/IngestPage/Compress/PathsInputFormItem.tsx

@@ -0,0 +1,25 @@
+import {
+    Form,
+    Input,
+} from "antd";
+
+
+/**
+ * Renders a form item for inputting file paths for compression job submission.
+ *
+ * @return
+ */
+const PathsInputFormItem = () => (
+    <Form.Item
+        label={"Paths"}
+        name={"paths"}
+        rules={[{required: true, message: "Please enter at least one path"}]}
+    >
+        <Input.TextArea
+            autoSize={{minRows: 4, maxRows: 10}}
+            placeholder={"Enter paths to compress, one per line"}/>
+    </Form.Item>
+);
+
+
+export default PathsInputFormItem;

components/webui/client/src/pages/IngestPage/Compress/SubmitFormItem.tsx

@@ -0,0 +1,31 @@
+import {
+    Button,
+    Form,
+} from "antd";
+
+
+interface SubmitFormItemProps {
+    isSubmitting: boolean;
+}
+
+/**
+ * Render a submit button for a form.
+ *
+ * @param props
+ * @param props.isSubmitting
+ * @return
+ */
+const SubmitFormItem = ({isSubmitting}: SubmitFormItemProps) => (
+    <Form.Item>
+        <Button
+            htmlType={"submit"}
+            loading={isSubmitting}
+            type={"primary"}
+        >
+            Submit
+        </Button>
+    </Form.Item>
+);
+
+
+export default SubmitFormItem;

components/webui/client/src/pages/IngestPage/Compress/index.tsx

@@ -0,0 +1,113 @@
+import {
+    useMutation,
+    useQueryClient,
+} from "@tanstack/react-query";
+import {
+    CLP_DEFAULT_DATASET_NAME,
+    CLP_STORAGE_ENGINES,
+} from "@webui/common/config";
+import {CompressionJobCreation} from "@webui/common/schemas/compression";
+import {
+    Form,
+    Typography,
+} from "antd";
+
+import {submitCompressionJob} from "../../../api/compress";
+import {DashboardCard} from "../../../components/DashboardCard";
+import {SETTINGS_STORAGE_ENGINE} from "../../../config";
+import ClpSFormItems from "./ClpSFormItems";
+import PathsInputFormItem from "./PathsInputFormItem";
+import SubmitFormItem from "./SubmitFormItem";
+
+
+type FormValues = {
+    paths: string;
+    dataset?: string;
+    timestampKey?: string;
+};
+
+
+/**
+ * Renders a compression job submission form.
+ *
+ * @return
+ */
+const Compress = () => {
+    const [form] = Form.useForm<FormValues>();
+
+    const queryClient = useQueryClient();
+    const {
+        mutate,
+        isPending: isSubmitting,
+        isSuccess,
+        isError,
+        data,
+        error,
+    } = useMutation({
+        mutationFn: submitCompressionJob,
+        onSettled: async () => {
+            // Invalidate queries that are affected by a new compression job.
+            await queryClient.invalidateQueries({queryKey: ["jobs"]});
+        },
+        onSuccess: () => {
+            form.resetFields();
+        },
+    });
+
+    const handleSubmit = (values: FormValues) => {
+        // eslint-disable-next-line no-warning-comments
+        // TODO: replace the UI with a file selector and remove below string manipulation.
+        // Convert multiline input to array of paths.
+        const paths = values.paths
+            .split("\n")
+            .map((path) => path.trim())
+            .filter((path) => 0 < path.length);
+
+        const payload: CompressionJobCreation = {paths};
+
+        if (CLP_STORAGE_ENGINES.CLP_S === SETTINGS_STORAGE_ENGINE) {
+            payload.dataset = ("undefined" === typeof values.dataset) ?
+                CLP_DEFAULT_DATASET_NAME :
+                values.dataset;
+            if ("undefined" !== typeof values.timestampKey) {
+                payload.timestampKey = values.timestampKey;
+            }
+        }
+
+        mutate(payload);
+    };
+
+    return (
+        <DashboardCard title={"Submit Compression Job"}>
+            <Form
+                form={form}
+                layout={"vertical"}
+                onFinish={handleSubmit}
+            >
+                <PathsInputFormItem/>
+                {CLP_STORAGE_ENGINES.CLP_S === SETTINGS_STORAGE_ENGINE && <ClpSFormItems/>}
+                <SubmitFormItem isSubmitting={isSubmitting}/>
+
+                {isSuccess && (
+                    <Typography.Text type={"success"}>
+                        Compression job submitted successfully with ID:
+                        {" "}
+                        {data.jobId}
+                    </Typography.Text>
+                )}
+                {isError && (
+                    <Typography.Text type={"danger"}>
+                        Failed to submit compression job:
+                        {" "}
+                        {error instanceof Error ?
+                            error.message :
+                            "Unknown error"}
+                    </Typography.Text>
+                )}
+            </Form>
+        </DashboardCard>
+    );
+};
+
+
+export default Compress;

components/webui/client/src/pages/IngestPage/Details/index.tsx

@@ -1,10 +1,8 @@
 import {useQuery} from "@tanstack/react-query";
+import {CLP_STORAGE_ENGINES} from "@webui/common/config";
 import dayjs from "dayjs";
 
-import {
-    CLP_STORAGE_ENGINES,
-    SETTINGS_STORAGE_ENGINE,
-} from "../../../config";
+import {SETTINGS_STORAGE_ENGINE} from "../../../config";
 import {fetchDatasetNames} from "../../SearchPage/SearchControls/Dataset/sql";
 import Files from "./Files";
 import styles from "./index.module.css";

components/webui/client/src/pages/IngestPage/SpaceSavings/index.tsx

@@ -1,12 +1,10 @@
 import {useQuery} from "@tanstack/react-query";
+import {CLP_STORAGE_ENGINES} from "@webui/common/config";
 import {theme} from "antd";
 
 import {DashboardCard} from "../../../components/DashboardCard";
 import Stat from "../../../components/Stat";
-import {
-    CLP_STORAGE_ENGINES,
-    SETTINGS_STORAGE_ENGINE,
-} from "../../../config";
+import {SETTINGS_STORAGE_ENGINE} from "../../../config";
 import {fetchDatasetNames} from "../../SearchPage/SearchControls/Dataset/sql";
 import CompressedSize from "./CompressedSize";
 import styles from "./index.module.css";