feat: Implement Time-To-Live (TTL) enforcement for specific datasets (#264)

* Implement Time-To-Live (TTL) enforcement for local datasets
This change introduces a strict 55-day retention policy for local datasets to align with data privacy guidelines.
- Datasets are assigned a retention date upon import (Oldest Log + 55 days).
- Fresh imports get a minimum 1-hour grace period.
- Expired datasets are automatically deleted on startup or when switching datasets.
- Warnings are displayed 10 days before expiration.
- Adds 'HAS_EXTRA_DATA_SOURCE' constant to simplify environment checks.

* fix: clean up DatasetLoading and use HAS_EXTRA_DATA_SOURCE constant

Author: regeter

src/App.js

@@ -24,6 +24,7 @@ import { log } from "./Utils";
 import { toast, ToastContainer } from "react-toastify";
 import "react-toastify/dist/ReactToastify.css";
 import { ALL_TOGGLES, getVisibleToggles } from "./MapToggles";
+import { HAS_EXTRA_DATA_SOURCE } from "./constants";
 
 const MARKER_COLORS = [
   "#EA4335", // Red
@@ -414,6 +415,22 @@ class App extends React.Component {
         const data = await getUploadedData(index);
         log(`Dataset ${index}:`, data);
         if (data && data.rawLogs && Array.isArray(data.rawLogs) && data.rawLogs.length > 0) {
+          if (HAS_EXTRA_DATA_SOURCE && data.retentionDate) {
+            const retentionDate = new Date(data.retentionDate);
+            if (retentionDate <= new Date()) {
+              log(`Startup: Dataset ${index} expired. Deleting...`);
+              await deleteUploadedData(index);
+              log(`Dataset ${index + 1} has expired and was deleted.`, "error");
+              return { status: null, index };
+            } else {
+              const now = new Date();
+              const timeLeftMs = retentionDate - now;
+              const daysLeft = timeLeftMs / (1000 * 60 * 60 * 24);
+              if (daysLeft <= 10) {
+                log(`Dataset ${index + 1} will expire in ${Math.ceil(daysLeft)} days.`, "warn");
+              }
+            }
+          }
           return { status: "Uploaded", index };
         }
         return { status: null, index };
@@ -786,8 +803,48 @@ class App extends React.Component {
     }
   };
 
+  checkAndEnforceTTL = async (index) => {
+    if (!HAS_EXTRA_DATA_SOURCE) return "Valid";
+
+    try {
+      const data = await getUploadedData(index);
+      if (!data || !data.retentionDate) return "Valid";
+
+      const retentionDate = new Date(data.retentionDate);
+      const now = new Date();
+      const timeLeftMs = retentionDate - now;
+      const daysLeft = timeLeftMs / (1000 * 60 * 60 * 24);
+
+      if (timeLeftMs <= 0) {
+        log(`Dataset ${index} expired. Deleting...`);
+        await deleteUploadedData(index);
+        this.setState((prevState) => {
+          const newUploadedDatasets = [...prevState.uploadedDatasets];
+          newUploadedDatasets[index] = null;
+          return {
+            uploadedDatasets: newUploadedDatasets,
+            activeDatasetIndex: prevState.activeDatasetIndex === index ? null : prevState.activeDatasetIndex,
+          };
+        });
+        log(`Dataset ${index + 1} has expired and was deleted (Retention limit reached).`, "error");
+        return "Expired";
+      } else if (daysLeft <= 10) {
+        log(`Dataset ${index + 1} will expire in ${Math.ceil(daysLeft)} days.`, "warn");
+        return "Warning";
+      }
+      return "Valid";
+    } catch (e) {
+      console.error("Error verifying TTL:", e);
+      return "Error";
+    }
+  };
+
   switchDataset = async (index) => {
     log(`Attempting to switch to dataset ${index}`);
+
+    const ttlStatus = await this.checkAndEnforceTTL(index);
+    if (ttlStatus === "Expired") return;
+
     if (this.state.uploadedDatasets[index] !== "Uploaded") {
       console.error(`Attempted to switch to dataset ${index}, but it's not uploaded or is empty`);
       return;

src/DatasetLoading.js

@@ -6,11 +6,7 @@ import { log } from "./Utils";
 import { toast, ToastContainer } from "react-toastify";
 import "react-toastify/dist/ReactToastify.css";
 import { isTokenValid, fetchLogsWithToken, useCloudLoggingLogin, buildQueryFilter } from "./CloudLogging";
-
-const DATA_SOURCES = {
-  CLOUD_LOGGING: "cloudLogging",
-  EXTRA: "extra",
-};
+import { HAS_EXTRA_DATA_SOURCE } from "./constants";
 
 const CloudLoggingFormComponent = ({ onLogsReceived, onFileUpload }) => {
   const getStoredValue = (key, defaultValue = "") => localStorage.getItem(`datasetLoading_${key}`) || defaultValue;
@@ -181,38 +177,37 @@ const CloudLoggingFormComponent = ({ onLogsReceived, onFileUpload }) => {
 
 export default function DatasetLoading(props) {
   const [activeDataSource, setActiveDataSource] = useState(
-    localStorage.getItem("datasetLoading_dataSource") ||
-      (ExtraDataSource.isAvailable() ? DATA_SOURCES.EXTRA : DATA_SOURCES.CLOUD_LOGGING)
+    localStorage.getItem("lastUsedDataSource") || (HAS_EXTRA_DATA_SOURCE ? "extra" : "cloudLogging")
   );
 
   useEffect(() => {
-    localStorage.setItem("datasetLoading_dataSource", activeDataSource);
+    localStorage.setItem("lastUsedDataSource", activeDataSource);
   }, [activeDataSource]);
 
-  const isExtra = activeDataSource === DATA_SOURCES.EXTRA;
+  const isExtra = activeDataSource === "extra";
   const ExtraFormComponent = isExtra ? ExtraDataSource.getFormComponent(props) : null;
-  const isExtraAvailable = ExtraDataSource.isAvailable();
+
+  const renderSourceSelection = () => {
+    if (!HAS_EXTRA_DATA_SOURCE) {
+      return <button className="active static">Cloud Logging</button>;
+    }
+
+    return (
+      <>
+        <button onClick={() => setActiveDataSource("cloudLogging")} className={!isExtra ? "active" : ""}>
+          Cloud Logging
+        </button>
+        <button onClick={() => setActiveDataSource("extra")} className={isExtra ? "active" : ""}>
+          {ExtraDataSource.getDisplayName()}
+        </button>
+      </>
+    );
+  };
 
   return (
     <>
       <ToastContainer position="top-right" autoClose={5000} />
-      <div className="data-source-toggle">
-        {isExtraAvailable ? (
-          <>
-            <button
-              onClick={() => setActiveDataSource(DATA_SOURCES.CLOUD_LOGGING)}
-              className={!isExtra ? "active" : ""}
-            >
-              Cloud Logging
-            </button>
-            <button onClick={() => setActiveDataSource(DATA_SOURCES.EXTRA)} className={isExtra ? "active" : ""}>
-              {ExtraDataSource.getDisplayName()}
-            </button>
-          </>
-        ) : (
-          <button className="active static">Cloud Logging</button>
-        )}
-      </div>
+      <div className="data-source-toggle">{renderSourceSelection()}</div>
 
       {isExtra ? (
         ExtraFormComponent

src/constants.js

@@ -1,4 +1,6 @@
 // constants.js
+import ExtraDataSource from "./ExtraDataSource";
 
 export const DEFAULT_API_KEY = "";
 export const DEFAULT_MAP_ID = "e6ead35a6ace8599";
+export const HAS_EXTRA_DATA_SOURCE = ExtraDataSource.isAvailable();

src/localStorage.js

@@ -337,6 +337,27 @@ export function ensureCorrectFormat(data) {
 
   if (!hasPoints) log("Bounds Calculation Failed: Could not find vehicle location data in any row.");
 
+  // Calculate retention date
+  let oldestTimestamp = Infinity;
+  logsArray.forEach((row) => {
+    const ts = new Date(row.timestamp || row.insertTimestamp || row.jsonPayload?.timestamp).getTime();
+    if (!isNaN(ts) && ts < oldestTimestamp) {
+      oldestTimestamp = ts;
+    }
+  });
+
+  let retentionDateIdentifier = null;
+  if (oldestTimestamp !== Infinity) {
+    const fiftyFiveDaysMs = 55 * 24 * 60 * 60 * 1000;
+    const oneHourMs = 60 * 60 * 1000;
+    const expirationBasedOnLogs = oldestTimestamp + fiftyFiveDaysMs;
+    const minimumRetention = Date.now() + oneHourMs;
+    retentionDateIdentifier = new Date(Math.max(expirationBasedOnLogs, minimumRetention)).toISOString();
+  } else {
+    // If no valid timestamps found, default to 1 hour from now for safety
+    retentionDateIdentifier = new Date(Date.now() + 60 * 60 * 1000).toISOString();
+  }
+
   return {
     APIKEY: DEFAULT_API_KEY,
     vehicle: "",
@@ -345,6 +366,7 @@ export function ensureCorrectFormat(data) {
     solutionType: solutionType,
     rawLogs: fullyNormalizedLogs,
     bounds: hasPoints ? bounds : null,
+    retentionDate: retentionDateIdentifier,
   };
 }
 

src/localStorage.test.js

@@ -208,3 +208,53 @@ describe("ensureCorrectFormat realistic merge scenarios", () => {
     expect(finalPayload.request.vehicle.name).toBe("test-vehicle");
   });
 });
+
+describe("ensureCorrectFormat TTL Logic", () => {
+  const ONE_DAY_MS = 24 * 60 * 60 * 1000;
+  const FIFTY_FIVE_DAYS_MS = 55 * ONE_DAY_MS;
+  const ONE_HOUR_MS = 60 * 60 * 1000;
+
+  it("should set retentionDate to Now + 1h for very old logs (Grace Period)", () => {
+    const oldTimestamp = new Date(Date.now() - 100 * ONE_DAY_MS).toISOString();
+    const mockLogs = [{ timestamp: oldTimestamp, jsonPayload: { test: 1 } }];
+    const result = ensureCorrectFormat(mockLogs);
+
+    const retention = new Date(result.retentionDate).getTime();
+    const expectedMin = Date.now() + ONE_HOUR_MS;
+
+    // Allow small delta for execution time
+    expect(retention).toBeGreaterThanOrEqual(expectedMin - 1000);
+    expect(retention).toBeLessThanOrEqual(expectedMin + 5000);
+  });
+
+  it("should set retentionDate to Log + 55d for recent logs", () => {
+    const tenDaysAgo = new Date(Date.now() - 10 * ONE_DAY_MS).getTime();
+    const mockLogs = [{ timestamp: new Date(tenDaysAgo).toISOString(), jsonPayload: { test: 1 } }];
+    const result = ensureCorrectFormat(mockLogs);
+
+    const retention = new Date(result.retentionDate).getTime();
+    const expected = tenDaysAgo + FIFTY_FIVE_DAYS_MS;
+
+    expect(retention).toBe(expected);
+  });
+
+  it("should default to Now + 1h if no timestamps found", () => {
+    const mockLogs = [{ jsonPayload: { no_timestamp: true } }];
+    const result = ensureCorrectFormat(mockLogs);
+
+    const retention = new Date(result.retentionDate).getTime();
+    const expectedMin = Date.now() + ONE_HOUR_MS;
+
+    expect(retention).toBeGreaterThanOrEqual(expectedMin - 1000);
+    expect(retention).toBeLessThanOrEqual(expectedMin + 5000);
+  });
+
+  it("should store retentionDate as a valid ISO string", () => {
+    const mockLogs = [{ timestamp: new Date().toISOString(), jsonPayload: { test: 1 } }];
+    const result = ensureCorrectFormat(mockLogs);
+
+    expect(typeof result.retentionDate).toBe("string");
+    // Simple regex to check ISO format YYYY-MM-DDTHH:mm:ss.sssZ
+    expect(result.retentionDate).toMatch(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/);
+  });
+});