From 2d1aa22f3c4c6c7dc8afc5df860f11341e67a760 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 7 Apr 2025 03:38:04 -0400
Subject: [PATCH 01/42] generic metadata for ingestor and indexer

---
 src/handlers/http/modal/mod.rs | 362 ++++++++++++++-------------------
 src/parseable/mod.rs           |  23 ++-
 src/storage/object_storage.rs  |   6 +-
 3 files changed, 175 insertions(+), 216 deletions(-)

diff --git a/src/handlers/http/modal/mod.rs b/src/handlers/http/modal/mod.rs
index a0774f36b..11c06b6f7 100644
--- a/src/handlers/http/modal/mod.rs
+++ b/src/handlers/http/modal/mod.rs
@@ -58,7 +58,7 @@ pub mod utils;
 pub type OpenIdClient = Arc<openid::Client<Discovered, Claims>>;
 
 // to be decided on what the Default version should be
-pub const DEFAULT_VERSION: &str = "v3";
+pub const DEFAULT_VERSION: &str = "v4";
 
 include!(concat!(env!("OUT_DIR"), "/generated.rs"));
 
@@ -199,201 +199,46 @@ pub async fn load_on_init() -> anyhow::Result<()> {
     Ok(())
 }
 
-#[derive(Debug, Serialize, Deserialize, Default, Clone, Eq, PartialEq)]
-pub struct IngestorMetadata {
-    pub version: String,
-    pub port: String,
-    pub domain_name: String,
-    pub bucket_name: String,
-    pub token: String,
-    pub ingestor_id: String,
-    pub flight_port: String,
+#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Default)]
+pub enum NodeType {
+    #[default]
+    Ingestor,
+    Indexer,
 }
 
-impl IngestorMetadata {
-    pub fn new(
-        port: String,
-        domain_name: String,
-        bucket_name: String,
-        username: &str,
-        password: &str,
-        ingestor_id: String,
-        flight_port: String,
-    ) -> Self {
-        let token = base64::prelude::BASE64_STANDARD.encode(format!("{username}:{password}"));
-
-        Self {
-            port,
-            domain_name,
-            version: DEFAULT_VERSION.to_string(),
-            bucket_name,
-            token: format!("Basic {token}"),
-            ingestor_id,
-            flight_port,
+impl NodeType {
+    fn as_str(&self) -> &'static str {
+        match self {
+            NodeType::Ingestor => "ingestor",
+            NodeType::Indexer => "indexer",
         }
     }
-
-    /// Capture metadata information by either loading it from staging or starting fresh
-    pub fn load(options: &Options, storage: &dyn ObjectStorageProvider) -> Arc<Self> {
-        // all the files should be in the staging directory root
-        let entries = options
-            .staging_dir()
-            .read_dir()
-            .expect("Couldn't read from file");
-        let url = options.get_url(Mode::Ingest);
-        let port = url.port().unwrap_or(80).to_string();
-        let url = url.to_string();
-        let Options {
-            username, password, ..
-        } = options;
-        let staging_path = options.staging_dir();
-        let flight_port = options.flight_port.to_string();
-
-        for entry in entries {
-            // cause the staging directory will have only one file with ingestor in the name
-            // so the JSON Parse should not error unless the file is corrupted
-            let path = entry.expect("Should be a directory entry").path();
-            if !path
-                .file_name()
-                .and_then(|s| s.to_str())
-                .is_some_and(|s| s.contains("ingestor"))
-            {
-                continue;
-            }
-
-            // get the ingestor metadata from staging
-            let bytes = std::fs::read(path).expect("File should be present");
-            let mut meta =
-                Self::from_bytes(&bytes, options.flight_port).expect("Extracted ingestor metadata");
-
-            // compare url endpoint and port, update
-            if meta.domain_name != url {
-                info!(
-                    "Domain Name was Updated. Old: {} New: {}",
-                    meta.domain_name, url
-                );
-                meta.domain_name = url;
-            }
-
-            if meta.port != port {
-                info!("Port was Updated. Old: {} New: {}", meta.port, port);
-                meta.port = port;
-            }
-
-            let token = format!(
-                "Basic {}",
-                BASE64_STANDARD.encode(format!("{username}:{password}"))
-            );
-            if meta.token != token {
-                // TODO: Update the message to be more informative with username and password
-                warn!(
-                    "Credentials were Updated. Tokens updated; Old: {} New: {}",
-                    meta.token, token
-                );
-                meta.token = token;
-            }
-            meta.put_on_disk(staging_path)
-                .expect("Couldn't write to disk");
-
-            return Arc::new(meta);
-        }
-
-        let storage = storage.get_object_store();
-        let meta = Self::new(
-            port,
-            url,
-            storage.get_bucket_name(),
-            username,
-            password,
-            get_ingestor_id(),
-            flight_port,
-        );
-
-        meta.put_on_disk(staging_path)
-            .expect("Should Be valid Json");
-        Arc::new(meta)
-    }
-
-    pub fn get_ingestor_id(&self) -> String {
-        self.ingestor_id.clone()
-    }
-
-    #[inline(always)]
-    pub fn file_path(&self) -> RelativePathBuf {
-        RelativePathBuf::from_iter([
-            PARSEABLE_ROOT_DIRECTORY,
-            &format!("ingestor.{}.json", self.get_ingestor_id()),
-        ])
-    }
-
-    /// Updates json with `flight_port` field if not already present
-    fn from_bytes(bytes: &[u8], flight_port: u16) -> anyhow::Result<Self> {
-        let mut json: Map<String, Value> = serde_json::from_slice(bytes)?;
-        json.entry("flight_port")
-            .or_insert_with(|| Value::String(flight_port.to_string()));
-
-        Ok(serde_json::from_value(Value::Object(json))?)
-    }
-
-    pub async fn migrate(&self) -> anyhow::Result<Option<IngestorMetadata>> {
-        let imp = self.file_path();
-        let bytes = match PARSEABLE.storage.get_object_store().get_object(&imp).await {
-            Ok(bytes) => bytes,
-            Err(_) => {
-                return Ok(None);
-            }
-        };
-
-        let resource = Self::from_bytes(&bytes, PARSEABLE.options.flight_port)?;
-        let bytes = Bytes::from(serde_json::to_vec(&resource)?);
-
-        resource.put_on_disk(PARSEABLE.options.staging_dir())?;
-
-        PARSEABLE
-            .storage
-            .get_object_store()
-            .put_object(&imp, bytes)
-            .await?;
-
-        Ok(Some(resource))
-    }
-
-    /// Puts the ingestor info into the staging.
-    ///
-    /// This function takes the ingestor info as a parameter and stores it in staging.
-    /// # Parameters
-    ///
-    /// * `staging_path`: Staging root directory.
-    pub fn put_on_disk(&self, staging_path: &Path) -> anyhow::Result<()> {
-        let file_name = format!("ingestor.{}.json", self.ingestor_id);
-        let file_path = staging_path.join(file_name);
-
-        std::fs::write(file_path, serde_json::to_vec(&self)?)?;
-
-        Ok(())
-    }
 }
 
 #[derive(Debug, Serialize, Deserialize, Default, Clone, Eq, PartialEq)]
-pub struct IndexerMetadata {
+pub struct NodeMetadata {
     pub version: String,
     pub port: String,
     pub domain_name: String,
     pub bucket_name: String,
     pub token: String,
-    pub indexer_id: String,
+    pub node_id: String,
     pub flight_port: String,
+    #[serde(skip)]
+    pub node_type: NodeType,
 }
 
-impl IndexerMetadata {
+impl NodeMetadata {
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         port: String,
         domain_name: String,
         bucket_name: String,
         username: &str,
         password: &str,
-        indexer_id: String,
+        node_id: String,
         flight_port: String,
+        node_type: NodeType,
     ) -> Self {
         let token = base64::prelude::BASE64_STANDARD.encode(format!("{username}:{password}"));
 
@@ -403,19 +248,30 @@ impl IndexerMetadata {
             version: DEFAULT_VERSION.to_string(),
             bucket_name,
             token: format!("Basic {token}"),
-            indexer_id,
+            node_id,
             flight_port,
+            node_type,
         }
     }
 
     /// Capture metadata information by either loading it from staging or starting fresh
-    pub fn load(options: &Options, storage: &dyn ObjectStorageProvider) -> Arc<Self> {
+    pub fn load(
+        options: &Options,
+        storage: &dyn ObjectStorageProvider,
+        node_type: NodeType,
+    ) -> Arc<Self> {
         // all the files should be in the staging directory root
         let entries = options
             .staging_dir()
             .read_dir()
             .expect("Couldn't read from file");
-        let url = options.get_url(Mode::Index);
+
+        let mode = match node_type {
+            NodeType::Ingestor => Mode::Ingest,
+            NodeType::Indexer => Mode::Index,
+        };
+
+        let url = options.get_url(mode);
         let port = url.port().unwrap_or(80).to_string();
         let url = url.to_string();
         let Options {
@@ -423,23 +279,24 @@ impl IndexerMetadata {
         } = options;
         let staging_path = options.staging_dir();
         let flight_port = options.flight_port.to_string();
+        let type_str = node_type.as_str();
 
         for entry in entries {
-            // cause the staging directory will have only one file with indexer in the name
+            // the staging directory will have only one file with the node type in the name
             // so the JSON Parse should not error unless the file is corrupted
             let path = entry.expect("Should be a directory entry").path();
             if !path
                 .file_name()
                 .and_then(|s| s.to_str())
-                .is_some_and(|s| s.contains("indexer"))
+                .is_some_and(|s| s.contains(type_str))
             {
                 continue;
             }
 
-            // get the indexer metadata from staging
+            // get the metadata from staging
             let bytes = std::fs::read(path).expect("File should be present");
-            let mut meta =
-                Self::from_bytes(&bytes, options.flight_port).expect("Extracted indexer metadata");
+            let mut meta = Self::from_bytes(&bytes, options.flight_port)
+                .unwrap_or_else(|_| panic!("Extracted {} metadata", type_str));
 
             // compare url endpoint and port, update
             if meta.domain_name != url {
@@ -467,6 +324,8 @@ impl IndexerMetadata {
                 );
                 meta.token = token;
             }
+
+            meta.node_type = node_type.clone();
             meta.put_on_disk(staging_path)
                 .expect("Couldn't write to disk");
 
@@ -474,14 +333,20 @@ impl IndexerMetadata {
         }
 
         let storage = storage.get_object_store();
+        let node_id = match node_type {
+            NodeType::Ingestor => get_ingestor_id(),
+            NodeType::Indexer => get_indexer_id(),
+        };
+
         let meta = Self::new(
             port,
             url,
             storage.get_bucket_name(),
             username,
             password,
-            get_indexer_id(),
+            node_id,
             flight_port,
+            node_type,
         );
 
         meta.put_on_disk(staging_path)
@@ -489,28 +354,59 @@ impl IndexerMetadata {
         Arc::new(meta)
     }
 
-    pub fn get_indexer_id(&self) -> String {
-        self.indexer_id.clone()
+    pub fn get_node_id(&self) -> String {
+        self.node_id.clone()
     }
 
     #[inline(always)]
     pub fn file_path(&self) -> RelativePathBuf {
         RelativePathBuf::from_iter([
             PARSEABLE_ROOT_DIRECTORY,
-            &format!("indexer.{}.json", self.get_indexer_id()),
+            &format!("{}.{}.json", self.node_type.as_str(), self.get_node_id()),
         ])
     }
 
     /// Updates json with `flight_port` field if not already present
     fn from_bytes(bytes: &[u8], flight_port: u16) -> anyhow::Result<Self> {
         let mut json: Map<String, Value> = serde_json::from_slice(bytes)?;
+
+        // Check version
+        let version = json.get("version").and_then(|version| version.as_str());
+
+        if version == Some("v3") {
+            if json.contains_key("ingestor_id") {
+                // Migration: get ingestor_id value, remove it, and add as node_id
+                if let Some(id) = json.remove("ingestor_id") {
+                    json.insert("node_id".to_string(), id);
+                    json.insert(
+                        "version".to_string(),
+                        Value::String(DEFAULT_VERSION.to_string()),
+                    );
+                }
+            } else if json.contains_key("indexer_id") {
+                // Migration: get indexer_id value, remove it, and add as node_id
+                if let Some(id) = json.remove("indexer_id") {
+                    json.insert("node_id".to_string(), id);
+                    json.insert(
+                        "version".to_string(),
+                        Value::String(DEFAULT_VERSION.to_string()),
+                    );
+                }
+            }
+        }
+        // Determine node type and perform migration if needed
+
+        // Add flight_port if missing
         json.entry("flight_port")
             .or_insert_with(|| Value::String(flight_port.to_string()));
 
-        Ok(serde_json::from_value(Value::Object(json))?)
+        // Parse the JSON to our struct
+        let metadata: Self = serde_json::from_value(Value::Object(json))?;
+
+        Ok(metadata)
     }
 
-    pub async fn migrate(&self) -> anyhow::Result<Option<IndexerMetadata>> {
+    pub async fn migrate(&self) -> anyhow::Result<Option<Self>> {
         let imp = self.file_path();
         let bytes = match PARSEABLE.storage.get_object_store().get_object(&imp).await {
             Ok(bytes) => bytes,
@@ -519,7 +415,8 @@ impl IndexerMetadata {
             }
         };
 
-        let resource = Self::from_bytes(&bytes, PARSEABLE.options.flight_port)?;
+        let mut resource = Self::from_bytes(&bytes, PARSEABLE.options.flight_port)?;
+        resource.node_type = self.node_type.clone();
         let bytes = Bytes::from(serde_json::to_vec(&resource)?);
 
         resource.put_on_disk(PARSEABLE.options.staging_dir())?;
@@ -533,14 +430,14 @@ impl IndexerMetadata {
         Ok(Some(resource))
     }
 
-    /// Puts the indexer info into the staging.
+    /// Puts the node info into the staging.
     ///
-    /// This function takes the indexer info as a parameter and stores it in staging.
+    /// This function takes the node info as a parameter and stores it in staging.
     /// # Parameters
     ///
     /// * `staging_path`: Staging root directory.
     pub fn put_on_disk(&self, staging_path: &Path) -> anyhow::Result<()> {
-        let file_name = format!("indexer.{}.json", self.indexer_id);
+        let file_name = format!("{}.{}.json", self.node_type.as_str(), self.node_id);
         let file_path = staging_path.join(file_name);
 
         std::fs::write(file_path, serde_json::to_vec(&self)?)?;
@@ -556,7 +453,7 @@ pub trait Metadata {
     fn file_path(&self) -> RelativePathBuf;
 }
 
-impl Metadata for IngestorMetadata {
+impl Metadata for NodeMetadata {
     fn domain_name(&self) -> &str {
         &self.domain_name
     }
@@ -564,28 +461,75 @@ impl Metadata for IngestorMetadata {
     fn token(&self) -> &str {
         &self.token
     }
+
     fn node_type(&self) -> &str {
-        "ingestor"
+        self.node_type.as_str()
     }
+
     fn file_path(&self) -> RelativePathBuf {
         self.file_path()
     }
 }
 
-impl Metadata for IndexerMetadata {
-    fn domain_name(&self) -> &str {
-        &self.domain_name
-    }
+// Type aliases for backward compatibility
+pub type IngestorMetadata = NodeMetadata;
+pub type IndexerMetadata = NodeMetadata;
+
+// Helper functions for creating specific node types
+pub fn create_ingestor_metadata(
+    port: String,
+    domain_name: String,
+    bucket_name: String,
+    username: &str,
+    password: &str,
+    ingestor_id: String,
+    flight_port: String,
+) -> NodeMetadata {
+    NodeMetadata::new(
+        port,
+        domain_name,
+        bucket_name,
+        username,
+        password,
+        ingestor_id,
+        flight_port,
+        NodeType::Ingestor,
+    )
+}
 
-    fn token(&self) -> &str {
-        &self.token
-    }
-    fn node_type(&self) -> &str {
-        "indexer"
-    }
-    fn file_path(&self) -> RelativePathBuf {
-        self.file_path()
-    }
+pub fn load_ingestor_metadata(
+    options: &Options,
+    storage: &dyn ObjectStorageProvider,
+) -> Arc<NodeMetadata> {
+    NodeMetadata::load(options, storage, NodeType::Ingestor)
+}
+
+pub fn create_indexer_metadata(
+    port: String,
+    domain_name: String,
+    bucket_name: String,
+    username: &str,
+    password: &str,
+    indexer_id: String,
+    flight_port: String,
+) -> NodeMetadata {
+    NodeMetadata::new(
+        port,
+        domain_name,
+        bucket_name,
+        username,
+        password,
+        indexer_id,
+        flight_port,
+        NodeType::Indexer,
+    )
+}
+
+pub fn load_indexer_metadata(
+    options: &Options,
+    storage: &dyn ObjectStorageProvider,
+) -> Arc<NodeMetadata> {
+    NodeMetadata::load(options, storage, NodeType::Indexer)
 }
 #[cfg(test)]
 mod test {
@@ -593,6 +537,8 @@ mod test {
     use bytes::Bytes;
     use rstest::rstest;
 
+    use crate::handlers::http::modal::NodeType;
+
     use super::IngestorMetadata;
 
     #[rstest]
@@ -605,6 +551,7 @@ mod test {
             "admin",
             "ingestor_id".to_owned(),
             "8002".to_string(),
+            NodeType::Ingestor,
         );
 
         let rhs = serde_json::from_slice::<IngestorMetadata>(br#"{"version":"v3","port":"8000","domain_name":"https://localhost:8000","bucket_name":"somebucket","token":"Basic YWRtaW46YWRtaW4=", "ingestor_id": "ingestor_id","flight_port": "8002"}"#).unwrap();
@@ -636,6 +583,7 @@ mod test {
             "admin",
             "ingestor_id".to_owned(),
             "8002".to_string(),
+            NodeType::Ingestor,
         );
 
         let lhs = Bytes::from(serde_json::to_vec(&im).unwrap());
diff --git a/src/parseable/mod.rs b/src/parseable/mod.rs
index ed6354eb8..a3220945a 100644
--- a/src/parseable/mod.rs
+++ b/src/parseable/mod.rs
@@ -47,7 +47,10 @@ use crate::{
             cluster::{sync_streams_with_ingestors, INTERNAL_STREAM_NAME},
             ingest::PostError,
             logstream::error::{CreateStreamError, StreamError},
-            modal::{utils::logstream_utils::PutStreamHeaders, IndexerMetadata, IngestorMetadata},
+            modal::{
+                utils::logstream_utils::PutStreamHeaders, IndexerMetadata, IngestorMetadata,
+                NodeType,
+            },
         },
         STREAM_TYPE_KEY,
     },
@@ -143,11 +146,19 @@ impl Parseable {
         storage: Arc<dyn ObjectStorageProvider>,
     ) -> Self {
         let ingestor_metadata = match &options.mode {
-            Mode::Ingest => Some(IngestorMetadata::load(&options, storage.as_ref())),
+            Mode::Ingest => Some(IngestorMetadata::load(
+                &options,
+                storage.as_ref(),
+                NodeType::Ingestor,
+            )),
             _ => None,
         };
         let indexer_metadata = match &options.mode {
-            Mode::Index => Some(IndexerMetadata::load(&options, storage.as_ref())),
+            Mode::Index => Some(IndexerMetadata::load(
+                &options,
+                storage.as_ref(),
+                NodeType::Indexer,
+            )),
             _ => None,
         };
         Parseable {
@@ -184,7 +195,7 @@ impl Parseable {
             LogStreamMetadata::default(),
             self.ingestor_metadata
                 .as_ref()
-                .map(|meta| meta.get_ingestor_id()),
+                .map(|meta| meta.get_node_id()),
         )
     }
 
@@ -388,7 +399,7 @@ impl Parseable {
             metadata,
             self.ingestor_metadata
                 .as_ref()
-                .map(|meta| meta.get_ingestor_id()),
+                .map(|meta| meta.get_node_id()),
         );
 
         Ok(true)
@@ -693,7 +704,7 @@ impl Parseable {
                     metadata,
                     self.ingestor_metadata
                         .as_ref()
-                        .map(|meta| meta.get_ingestor_id()),
+                        .map(|meta| meta.get_node_id()),
                 );
             }
             Err(err) => {
diff --git a/src/storage/object_storage.rs b/src/storage/object_storage.rs
index a40072f62..e20a9556c 100644
--- a/src/storage/object_storage.rs
+++ b/src/storage/object_storage.rs
@@ -901,7 +901,7 @@ pub fn schema_path(stream_name: &str) -> RelativePathBuf {
                 .ingestor_metadata
                 .as_ref()
                 .expect(INGESTOR_EXPECT)
-                .get_ingestor_id();
+                .get_node_id();
             let file_name = format!(".ingestor.{id}{SCHEMA_FILE_NAME}");
 
             RelativePathBuf::from_iter([stream_name, STREAM_ROOT_DIRECTORY, &file_name])
@@ -920,7 +920,7 @@ pub fn stream_json_path(stream_name: &str) -> RelativePathBuf {
                 .ingestor_metadata
                 .as_ref()
                 .expect(INGESTOR_EXPECT)
-                .get_ingestor_id();
+                .get_node_id();
             let file_name = format!(".ingestor.{id}{STREAM_METADATA_FILE_NAME}",);
             RelativePathBuf::from_iter([stream_name, STREAM_ROOT_DIRECTORY, &file_name])
         }
@@ -970,7 +970,7 @@ pub fn manifest_path(prefix: &str) -> RelativePathBuf {
                 .ingestor_metadata
                 .as_ref()
                 .expect(INGESTOR_EXPECT)
-                .get_ingestor_id();
+                .get_node_id();
             let manifest_file_name = format!("ingestor.{id}.{MANIFEST_FILE}");
             RelativePathBuf::from_iter([prefix, &manifest_file_name])
         }

From b8243a247589e1ab149e0fa105c544f7dcacc046 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 7 Apr 2025 07:42:22 -0400
Subject: [PATCH 02/42] add querier metadata

---
 src/analytics.rs                              |   3 +-
 src/catalog/mod.rs                            |   9 +-
 src/cli.rs                                    |  22 ++++
 src/handlers/airplane.rs                      |   5 +-
 src/handlers/http/cluster/mod.rs              | 111 ++++++++++--------
 src/handlers/http/cluster/utils.rs            |  11 +-
 src/handlers/http/mod.rs                      |   6 +-
 src/handlers/http/modal/mod.rs                |  72 ++++++++++--
 .../http/modal/query/querier_logstream.rs     |  10 +-
 src/handlers/http/modal/query_server.rs       |   4 +
 src/parseable/mod.rs                          | 101 +++++++---------
 src/utils/mod.rs                              |  11 +-
 12 files changed, 219 insertions(+), 146 deletions(-)

diff --git a/src/analytics.rs b/src/analytics.rs
index 42af72d36..0e6d43618 100644
--- a/src/analytics.rs
+++ b/src/analytics.rs
@@ -36,6 +36,7 @@ use crate::{
         http::{
             base_path_without_preceding_slash,
             cluster::{self, utils::check_liveness},
+            modal::NodeMetadata,
         },
         STREAM_NAME_HEADER_KEY,
     },
@@ -228,7 +229,7 @@ async fn fetch_ingestors_metrics(
         // send analytics for ingest servers
 
         // ingestor infos should be valid here, if not some thing is wrong
-        let ingestor_infos = cluster::get_ingestor_info().await.unwrap();
+        let ingestor_infos: Vec<NodeMetadata> = cluster::get_node_info("ingestor").await.unwrap();
 
         for im in ingestor_infos {
             if !check_liveness(&im.domain_name).await {
diff --git a/src/catalog/mod.rs b/src/catalog/mod.rs
index d44ac877b..1f69a071a 100644
--- a/src/catalog/mod.rs
+++ b/src/catalog/mod.rs
@@ -28,7 +28,10 @@ use tracing::{error, info};
 
 use crate::{
     event::DEFAULT_TIMESTAMP_KEY,
-    handlers::{self, http::base_path_without_preceding_slash},
+    handlers::{
+        self,
+        http::{base_path_without_preceding_slash, modal::NodeMetadata},
+    },
     metrics::{EVENTS_INGESTED_DATE, EVENTS_INGESTED_SIZE_DATE, EVENTS_STORAGE_SIZE_DATE},
     option::Mode,
     parseable::PARSEABLE,
@@ -406,8 +409,8 @@ pub async fn get_first_event(
             }
         }
         Mode::Query => {
-            let ingestor_metadata =
-                handlers::http::cluster::get_ingestor_info()
+            let ingestor_metadata: Vec<NodeMetadata> =
+                handlers::http::cluster::get_node_info("ingestor")
                     .await
                     .map_err(|err| {
                         error!("Fatal: failed to get ingestor info: {:?}", err);
diff --git a/src/cli.rs b/src/cli.rs
index ad2db244d..b302f3f4b 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -328,6 +328,14 @@ pub struct Options {
     )]
     pub indexer_endpoint: String,
 
+    #[arg(
+        long,
+        env = "P_QUERIER_ENDPOINT",
+        default_value = "",
+        help = "URL to connect to this specific querier. Default is the address of the server"
+    )]
+    pub querier_endpoint: String,
+
     #[command(flatten)]
     pub oidc: Option<OidcConfig>,
 
@@ -470,6 +478,20 @@ impl Options {
                 }
                 (&self.indexer_endpoint, "P_INDEXER_ENDPOINT")
             }
+            Mode::Query => {
+                if self.querier_endpoint.is_empty() {
+                    return format!(
+                        "{}://{}",
+                        self.get_scheme(),
+                        self.address
+                    )
+                    .parse::<Url>() // if the value was improperly set, this will panic before hand
+                    .unwrap_or_else(|err| {
+                        panic!("{err}, failed to parse `{}` as Url. Please set the environment variable `P_ADDR` to `<ip address>:<port>` without the scheme (e.g., 192.168.1.1:8000). Please refer to the documentation: https://logg.ing/env for more details.", self.address)
+                    });
+                }
+                (&self.querier_endpoint, "P_QUERIER_ENDPOINT")
+            }
             _ => panic!("Invalid mode"),
         };
 
diff --git a/src/handlers/airplane.rs b/src/handlers/airplane.rs
index 0b03dc9e2..cea0f9778 100644
--- a/src/handlers/airplane.rs
+++ b/src/handlers/airplane.rs
@@ -33,7 +33,8 @@ use futures_util::{Future, TryFutureExt};
 use tonic::transport::{Identity, Server, ServerTlsConfig};
 use tonic_web::GrpcWebLayer;
 
-use crate::handlers::http::cluster::get_ingestor_info;
+use crate::handlers::http::cluster::get_node_info;
+use crate::handlers::http::modal::NodeMetadata;
 use crate::handlers::http::query::{into_query, update_schema_when_distributed};
 use crate::handlers::livetail::cross_origin_config;
 use crate::metrics::QUERY_EXECUTE_TIME;
@@ -179,7 +180,7 @@ impl FlightService for AirServiceImpl {
             })
             .to_string();
 
-            let ingester_metadatas = get_ingestor_info()
+            let ingester_metadatas: Vec<NodeMetadata> = get_node_info("ingestor")
                 .await
                 .map_err(|err| Status::failed_precondition(err.to_string()))?;
             let mut minute_result: Vec<RecordBatch> = vec![];
diff --git a/src/handlers/http/cluster/mod.rs b/src/handlers/http/cluster/mod.rs
index a452dedca..456cf5aa2 100644
--- a/src/handlers/http/cluster/mod.rs
+++ b/src/handlers/http/cluster/mod.rs
@@ -54,14 +54,10 @@ use crate::HTTP_CLIENT;
 use super::base_path_without_preceding_slash;
 use super::ingest::PostError;
 use super::logstream::error::StreamError;
-use super::modal::{IndexerMetadata, IngestorMetadata, Metadata};
+use super::modal::{IndexerMetadata, IngestorMetadata, Metadata, NodeMetadata, QuerierMetadata};
 use super::rbac::RBACError;
 use super::role::RoleError;
 
-type IngestorMetadataArr = Vec<IngestorMetadata>;
-
-type IndexerMetadataArr = Vec<IndexerMetadata>;
-
 pub const INTERNAL_STREAM_NAME: &str = "pmeta";
 
 const CLUSTER_METRICS_INTERVAL_SECONDS: Interval = clokwerk::Interval::Minutes(1);
@@ -77,7 +73,7 @@ pub async fn sync_streams_with_ingestors(
     for (key, value) in headers.iter() {
         reqwest_headers.insert(key.clone(), value.clone());
     }
-    let ingestor_infos = get_ingestor_info().await.map_err(|err| {
+    let ingestor_infos: Vec<NodeMetadata> = get_node_info("ingestor").await.map_err(|err| {
         error!("Fatal: failed to get ingestor info: {:?}", err);
         StreamError::Anyhow(err)
     })?;
@@ -125,7 +121,7 @@ pub async fn sync_users_with_roles_with_ingestors(
     username: &String,
     role: &HashSet<String>,
 ) -> Result<(), RBACError> {
-    let ingestor_infos = get_ingestor_info().await.map_err(|err| {
+    let ingestor_infos: Vec<NodeMetadata> = get_node_info("ingestor").await.map_err(|err| {
         error!("Fatal: failed to get ingestor info: {:?}", err);
         RBACError::Anyhow(err)
     })?;
@@ -175,7 +171,7 @@ pub async fn sync_users_with_roles_with_ingestors(
 
 // forward the delete user request to all ingestors to keep them in sync
 pub async fn sync_user_deletion_with_ingestors(username: &String) -> Result<(), RBACError> {
-    let ingestor_infos = get_ingestor_info().await.map_err(|err| {
+    let ingestor_infos: Vec<NodeMetadata> = get_node_info("ingestor").await.map_err(|err| {
         error!("Fatal: failed to get ingestor info: {:?}", err);
         RBACError::Anyhow(err)
     })?;
@@ -222,7 +218,7 @@ pub async fn sync_user_creation_with_ingestors(
     user: User,
     role: &Option<HashSet<String>>,
 ) -> Result<(), RBACError> {
-    let ingestor_infos = get_ingestor_info().await.map_err(|err| {
+    let ingestor_infos: Vec<NodeMetadata> = get_node_info("ingestor").await.map_err(|err| {
         error!("Fatal: failed to get ingestor info: {:?}", err);
         RBACError::Anyhow(err)
     })?;
@@ -280,7 +276,7 @@ pub async fn sync_user_creation_with_ingestors(
 
 // forward the password reset request to all ingestors to keep them in sync
 pub async fn sync_password_reset_with_ingestors(username: &String) -> Result<(), RBACError> {
-    let ingestor_infos = get_ingestor_info().await.map_err(|err| {
+    let ingestor_infos: Vec<NodeMetadata> = get_node_info("ingestor").await.map_err(|err| {
         error!("Fatal: failed to get ingestor info: {:?}", err);
         RBACError::Anyhow(err)
     })?;
@@ -328,7 +324,7 @@ pub async fn sync_role_update_with_ingestors(
     name: String,
     privileges: Vec<DefaultPrivilege>,
 ) -> Result<(), RoleError> {
-    let ingestor_infos = get_ingestor_info().await.map_err(|err| {
+    let ingestor_infos: Vec<NodeMetadata> = get_node_info("ingestor").await.map_err(|err| {
         error!("Fatal: failed to get ingestor info: {:?}", err);
         RoleError::Anyhow(err)
     })?;
@@ -545,11 +541,23 @@ pub async fn send_retention_cleanup_request(
 
 pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
     // Get ingestor and indexer metadata concurrently
-    let (ingestor_result, indexer_result) =
-        future::join(get_ingestor_info(), get_indexer_info()).await;
+    let (querier_result, ingestor_result, indexer_result) = future::join3(
+        get_node_info("querier"),
+        get_node_info("ingestor"),
+        get_node_info("indexer"),
+    )
+    .await;
+
+    // Handle querier metadata result
+    let querier_metadata: Vec<NodeMetadata> = querier_result
+        .map_err(|err| {
+            error!("Fatal: failed to get querier info: {:?}", err);
+            PostError::Invalid(err)
+        })
+        .map_err(|err| StreamError::Anyhow(err.into()))?;
 
     // Handle ingestor metadata result
-    let ingestor_metadata = ingestor_result
+    let ingestor_metadata: Vec<NodeMetadata> = ingestor_result
         .map_err(|err| {
             error!("Fatal: failed to get ingestor info: {:?}", err);
             PostError::Invalid(err)
@@ -557,7 +565,7 @@ pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
         .map_err(|err| StreamError::Anyhow(err.into()))?;
 
     // Handle indexer metadata result
-    let indexer_metadata = indexer_result
+    let indexer_metadata: Vec<NodeMetadata> = indexer_result
         .map_err(|err| {
             error!("Fatal: failed to get indexer info: {:?}", err);
             PostError::Invalid(err)
@@ -565,7 +573,8 @@ pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
         .map_err(|err| StreamError::Anyhow(err.into()))?;
 
     // Fetch info for both node types concurrently
-    let (ingestor_infos, indexer_infos) = future::join(
+    let (querier_infos, ingestor_infos, indexer_infos) = future::join3(
+        fetch_nodes_info(querier_metadata),
         fetch_nodes_info(ingestor_metadata),
         fetch_nodes_info(indexer_metadata),
     )
@@ -573,6 +582,7 @@ pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
 
     // Combine results from both node types
     let mut infos = Vec::new();
+    infos.extend(querier_infos?);
     infos.extend(ingestor_infos?);
     infos.extend(indexer_infos?);
 
@@ -671,40 +681,22 @@ pub async fn get_cluster_metrics() -> Result<impl Responder, PostError> {
     Ok(actix_web::HttpResponse::Ok().json(dresses))
 }
 
-pub async fn get_ingestor_info() -> anyhow::Result<IngestorMetadataArr> {
+pub async fn get_node_info<T: Metadata + DeserializeOwned>(prefix: &str) -> anyhow::Result<Vec<T>> {
     let store = PARSEABLE.storage.get_object_store();
-
     let root_path = RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY);
-    let arr = store
-        .get_objects(
-            Some(&root_path),
-            Box::new(|file_name| file_name.starts_with("ingestor")),
-        )
-        .await?
-        .iter()
-        // this unwrap will most definateley shoot me in the foot later
-        .map(|x| serde_json::from_slice::<IngestorMetadata>(x).unwrap_or_default())
-        .collect_vec();
-
-    Ok(arr)
-}
+    let prefix_owned = prefix.to_string(); // Create an owned copy of the prefix
 
-pub async fn get_indexer_info() -> anyhow::Result<IndexerMetadataArr> {
-    let store = PARSEABLE.storage.get_object_store();
-
-    let root_path = RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY);
-    let arr = store
+    let metadata = store
         .get_objects(
             Some(&root_path),
-            Box::new(|file_name| file_name.starts_with("indexer")),
+            Box::new(move |file_name| file_name.starts_with(&prefix_owned)), // Use the owned copy
         )
         .await?
         .iter()
-        // this unwrap will most definateley shoot me in the foot later
-        .map(|x| serde_json::from_slice::<IndexerMetadata>(x).unwrap_or_default())
-        .collect_vec();
+        .filter_map(|x| serde_json::from_slice::<T>(x).ok())
+        .collect();
 
-    Ok(arr)
+    Ok(metadata)
 }
 
 pub async fn remove_node(node_url: Path<String>) -> Result<impl Responder, PostError> {
@@ -725,7 +717,11 @@ pub async fn remove_node(node_url: Path<String>) -> Result<impl Responder, PostE
     let removed_indexer =
         remove_node_metadata::<IndexerMetadata>(&object_store, &domain_name).await?;
 
-    let msg = if removed_ingestor || removed_indexer {
+    // Delete querier metadata
+    let removed_querier =
+        remove_node_metadata::<QuerierMetadata>(&object_store, &domain_name).await?;
+
+    let msg = if removed_ingestor || removed_indexer || removed_querier {
         format!("node {} removed successfully", domain_name)
     } else {
         format!("node {} is not found", domain_name)
@@ -836,6 +832,9 @@ where
     T: Metadata + Send + Sync + 'static,
 {
     let nodes_len = nodes.len();
+    if nodes_len == 0 {
+        return Ok(vec![]);
+    }
     let results = stream::iter(nodes)
         .map(|node| async move { fetch_node_metrics(&node).await })
         .buffer_unordered(nodes_len) // No concurrency limit
@@ -858,23 +857,31 @@ where
 /// Main function to fetch all cluster metrics, parallelized and refactored
 async fn fetch_cluster_metrics() -> Result<Vec<Metrics>, PostError> {
     // Get ingestor and indexer metadata concurrently
-    let (ingestor_result, indexer_result) =
-        future::join(get_ingestor_info(), get_indexer_info()).await;
+    let (querier_result, ingestor_result, indexer_result) = future::join3(
+        get_node_info("querier"),
+        get_node_info("ingestor"),
+        get_node_info("indexer"),
+    )
+    .await;
 
+    // Handle querier metadata result
+    let querier_metadata: Vec<NodeMetadata> = querier_result.map_err(|err| {
+        error!("Fatal: failed to get querier info: {:?}", err);
+        PostError::Invalid(err)
+    })?;
     // Handle ingestor metadata result
-    let ingestor_metadata = ingestor_result.map_err(|err| {
+    let ingestor_metadata: Vec<NodeMetadata> = ingestor_result.map_err(|err| {
         error!("Fatal: failed to get ingestor info: {:?}", err);
         PostError::Invalid(err)
     })?;
-
     // Handle indexer metadata result
-    let indexer_metadata = indexer_result.map_err(|err| {
+    let indexer_metadata: Vec<NodeMetadata> = indexer_result.map_err(|err| {
         error!("Fatal: failed to get indexer info: {:?}", err);
         PostError::Invalid(err)
     })?;
-
     // Fetch metrics from ingestors and indexers concurrently
-    let (ingestor_metrics, indexer_metrics) = future::join(
+    let (querier_metrics, ingestor_metrics, indexer_metrics) = future::join3(
+        fetch_nodes_metrics(querier_metadata),
         fetch_nodes_metrics(ingestor_metadata),
         fetch_nodes_metrics(indexer_metadata),
     )
@@ -883,6 +890,12 @@ async fn fetch_cluster_metrics() -> Result<Vec<Metrics>, PostError> {
     // Combine all metrics
     let mut all_metrics = Vec::new();
 
+    // Add querier metrics
+    match querier_metrics {
+        Ok(metrics) => all_metrics.extend(metrics),
+        Err(err) => return Err(err),
+    }
+
     // Add ingestor metrics
     match ingestor_metrics {
         Ok(metrics) => all_metrics.extend(metrics),
diff --git a/src/handlers/http/cluster/utils.rs b/src/handlers/http/cluster/utils.rs
index 54d451cb7..a4038aa5c 100644
--- a/src/handlers/http/cluster/utils.rs
+++ b/src/handlers/http/cluster/utils.rs
@@ -16,7 +16,10 @@
  *
  */
 
-use crate::{handlers::http::base_path_without_preceding_slash, HTTP_CLIENT};
+use crate::{
+    handlers::http::{base_path_without_preceding_slash, modal::NodeType},
+    HTTP_CLIENT,
+};
 use actix_web::http::header;
 use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
@@ -55,7 +58,7 @@ pub struct ClusterInfo {
     storage_path: String,
     error: Option<String>,  // error message if the ingestor is not reachable
     status: Option<String>, // status message if the ingestor is reachable
-    node_type: String,
+    node_type: NodeType,
 }
 
 impl ClusterInfo {
@@ -66,7 +69,7 @@ impl ClusterInfo {
         storage_path: String,
         error: Option<String>,
         status: Option<String>,
-        node_type: &str,
+        node_type: &NodeType,
     ) -> Self {
         Self {
             domain_name: domain_name.to_string(),
@@ -75,7 +78,7 @@ impl ClusterInfo {
             storage_path,
             error,
             status,
-            node_type: node_type.to_string(),
+            node_type: node_type.clone(),
         }
     }
 }
diff --git a/src/handlers/http/mod.rs b/src/handlers/http/mod.rs
index aa2e0a02d..ea19ed3be 100644
--- a/src/handlers/http/mod.rs
+++ b/src/handlers/http/mod.rs
@@ -19,13 +19,15 @@
 use actix_cors::Cors;
 use actix_web::Responder;
 use arrow_schema::Schema;
+use cluster::get_node_info;
 use http::StatusCode;
 use itertools::Itertools;
+use modal::NodeMetadata;
 use serde_json::Value;
 
 use crate::{parseable::PARSEABLE, storage::STREAM_ROOT_DIRECTORY, HTTP_CLIENT};
 
-use self::{cluster::get_ingestor_info, query::Query};
+use self::query::Query;
 
 pub mod about;
 pub mod alerts;
@@ -108,7 +110,7 @@ pub async fn fetch_schema(stream_name: &str) -> anyhow::Result<arrow_schema::Sch
 pub async fn send_query_request_to_ingestor(query: &Query) -> anyhow::Result<Vec<Value>> {
     // send the query request to the ingestor
     let mut res = vec![];
-    let ima = get_ingestor_info().await?;
+    let ima: Vec<NodeMetadata> = get_node_info("ingestor").await?;
 
     for im in ima.iter() {
         let uri = format!(
diff --git a/src/handlers/http/modal/mod.rs b/src/handlers/http/modal/mod.rs
index 11c06b6f7..8580c95b2 100644
--- a/src/handlers/http/modal/mod.rs
+++ b/src/handlers/http/modal/mod.rs
@@ -16,7 +16,7 @@
  *
  */
 
-use std::{path::Path, sync::Arc};
+use std::{fmt, path::Path, sync::Arc};
 
 use actix_web::{middleware::from_fn, web::ServiceConfig, App, HttpServer};
 use actix_web_prometheus::PrometheusMetrics;
@@ -42,7 +42,7 @@ use crate::{
     parseable::PARSEABLE,
     storage::{ObjectStorageProvider, PARSEABLE_ROOT_DIRECTORY},
     users::{dashboards::DASHBOARDS, filters::FILTERS},
-    utils::{get_indexer_id, get_ingestor_id},
+    utils::get_node_id,
 };
 
 use super::{audit, cross_origin_config, health_check, API_BASE_PATH, API_VERSION};
@@ -202,8 +202,12 @@ pub async fn load_on_init() -> anyhow::Result<()> {
 #[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Default)]
 pub enum NodeType {
     #[default]
+    #[serde(rename = "ingestor")]
     Ingestor,
+    #[serde(rename = "indexer")]
     Indexer,
+    #[serde(rename = "querier")]
+    Querier,
 }
 
 impl NodeType {
@@ -211,6 +215,17 @@ impl NodeType {
         match self {
             NodeType::Ingestor => "ingestor",
             NodeType::Indexer => "indexer",
+            NodeType::Querier => "querier",
+        }
+    }
+}
+
+impl fmt::Display for NodeType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            NodeType::Ingestor => write!(f, "ingestor"),
+            NodeType::Indexer => write!(f, "indexer"),
+            NodeType::Querier => write!(f, "querier"),
         }
     }
 }
@@ -224,7 +239,6 @@ pub struct NodeMetadata {
     pub token: String,
     pub node_id: String,
     pub flight_port: String,
-    #[serde(skip)]
     pub node_type: NodeType,
 }
 
@@ -269,6 +283,7 @@ impl NodeMetadata {
         let mode = match node_type {
             NodeType::Ingestor => Mode::Ingest,
             NodeType::Indexer => Mode::Index,
+            NodeType::Querier => Mode::Query,
         };
 
         let url = options.get_url(mode);
@@ -333,10 +348,7 @@ impl NodeMetadata {
         }
 
         let storage = storage.get_object_store();
-        let node_id = match node_type {
-            NodeType::Ingestor => get_ingestor_id(),
-            NodeType::Indexer => get_indexer_id(),
-        };
+        let node_id = get_node_id();
 
         let meta = Self::new(
             port,
@@ -383,6 +395,10 @@ impl NodeMetadata {
                         Value::String(DEFAULT_VERSION.to_string()),
                     );
                 }
+                json.insert(
+                    "node_type".to_string(),
+                    Value::String("ingestor".to_string()),
+                );
             } else if json.contains_key("indexer_id") {
                 // Migration: get indexer_id value, remove it, and add as node_id
                 if let Some(id) = json.remove("indexer_id") {
@@ -392,6 +408,10 @@ impl NodeMetadata {
                         Value::String(DEFAULT_VERSION.to_string()),
                     );
                 }
+                json.insert(
+                    "node_type".to_string(),
+                    Value::String("indexer".to_string()),
+                );
             }
         }
         // Determine node type and perform migration if needed
@@ -449,7 +469,7 @@ impl NodeMetadata {
 pub trait Metadata {
     fn domain_name(&self) -> &str;
     fn token(&self) -> &str;
-    fn node_type(&self) -> &str;
+    fn node_type(&self) -> &NodeType;
     fn file_path(&self) -> RelativePathBuf;
 }
 
@@ -462,8 +482,8 @@ impl Metadata for NodeMetadata {
         &self.token
     }
 
-    fn node_type(&self) -> &str {
-        self.node_type.as_str()
+    fn node_type(&self) -> &NodeType {
+        &self.node_type
     }
 
     fn file_path(&self) -> RelativePathBuf {
@@ -474,6 +494,7 @@ impl Metadata for NodeMetadata {
 // Type aliases for backward compatibility
 pub type IngestorMetadata = NodeMetadata;
 pub type IndexerMetadata = NodeMetadata;
+pub type QuerierMetadata = NodeMetadata;
 
 // Helper functions for creating specific node types
 pub fn create_ingestor_metadata(
@@ -531,6 +552,33 @@ pub fn load_indexer_metadata(
 ) -> Arc<NodeMetadata> {
     NodeMetadata::load(options, storage, NodeType::Indexer)
 }
+
+pub fn create_querier_metadata(
+    port: String,
+    domain_name: String,
+    bucket_name: String,
+    username: &str,
+    password: &str,
+    querier_id: String,
+    flight_port: String,
+) -> NodeMetadata {
+    NodeMetadata::new(
+        port,
+        domain_name,
+        bucket_name,
+        username,
+        password,
+        querier_id,
+        flight_port,
+        NodeType::Querier,
+    )
+}
+pub fn load_querier_metadata(
+    options: &Options,
+    storage: &dyn ObjectStorageProvider,
+) -> Arc<NodeMetadata> {
+    NodeMetadata::load(options, storage, NodeType::Querier)
+}
 #[cfg(test)]
 mod test {
     use actix_web::body::MessageBody;
@@ -554,7 +602,7 @@ mod test {
             NodeType::Ingestor,
         );
 
-        let rhs = serde_json::from_slice::<IngestorMetadata>(br#"{"version":"v3","port":"8000","domain_name":"https://localhost:8000","bucket_name":"somebucket","token":"Basic YWRtaW46YWRtaW4=", "ingestor_id": "ingestor_id","flight_port": "8002"}"#).unwrap();
+        let rhs = serde_json::from_slice::<IngestorMetadata>(br#"{"version":"v4","port":"8000","domain_name":"https://localhost:8000","bucket_name":"somebucket","token":"Basic YWRtaW46YWRtaW4=","node_id": "ingestor_id","flight_port": "8002","node_type":"ingestor"}"#).unwrap();
 
         assert_eq!(rhs, lhs);
     }
@@ -587,7 +635,7 @@ mod test {
         );
 
         let lhs = Bytes::from(serde_json::to_vec(&im).unwrap());
-        let rhs = br#"{"version":"v3","port":"8000","domain_name":"https://localhost:8000","bucket_name":"somebucket","token":"Basic YWRtaW46YWRtaW4=","ingestor_id":"ingestor_id","flight_port":"8002"}"#
+        let rhs = br#"{"version":"v4","port":"8000","domain_name":"https://localhost:8000","bucket_name":"somebucket","token":"Basic YWRtaW46YWRtaW4=","node_id":"ingestor_id","flight_port":"8002","node_type":"ingestor"}"#
                 .try_into_bytes()
                 .unwrap();
 
diff --git a/src/handlers/http/modal/query/querier_logstream.rs b/src/handlers/http/modal/query/querier_logstream.rs
index 76d282f76..27837d91f 100644
--- a/src/handlers/http/modal/query/querier_logstream.rs
+++ b/src/handlers/http/modal/query/querier_logstream.rs
@@ -41,6 +41,7 @@ use crate::{
             utils::{merge_quried_stats, IngestionStats, QueriedStats, StorageStats},
         },
         logstream::{error::StreamError, get_stats_date},
+        modal::NodeMetadata,
     },
     hottier::HotTierManager,
     parseable::{StreamNotFound, PARSEABLE},
@@ -81,10 +82,11 @@ pub async fn delete(stream_name: Path<String>) -> Result<impl Responder, StreamE
         }
     }
 
-    let ingestor_metadata = cluster::get_ingestor_info().await.map_err(|err| {
-        error!("Fatal: failed to get ingestor info: {:?}", err);
-        err
-    })?;
+    let ingestor_metadata: Vec<NodeMetadata> =
+        cluster::get_node_info("ingestor").await.map_err(|err| {
+            error!("Fatal: failed to get ingestor info: {:?}", err);
+            err
+        })?;
 
     for ingestor in ingestor_metadata {
         let url = format!(
diff --git a/src/handlers/http/modal/query_server.rs b/src/handlers/http/modal/query_server.rs
index d4ce26f72..44a208234 100644
--- a/src/handlers/http/modal/query_server.rs
+++ b/src/handlers/http/modal/query_server.rs
@@ -25,6 +25,7 @@ use crate::handlers::http::{base_path, prism_base_path};
 use crate::handlers::http::{logstream, MAX_EVENT_PAYLOAD_SIZE};
 use crate::handlers::http::{rbac, role};
 use crate::hottier::HotTierManager;
+use crate::option::Mode;
 use crate::rbac::role::Action;
 use crate::{analytics, migration, storage, sync};
 use actix_web::web::{resource, ServiceConfig};
@@ -129,6 +130,9 @@ impl ParseableServer for QueryServer {
 
         tokio::spawn(airplane::server());
 
+        // write the querier metadata to storage
+        PARSEABLE.store_metadata(Mode::Query).await?;
+
         let result = self
             .start(shutdown_rx, prometheus.clone(), PARSEABLE.options.openid())
             .await?;
diff --git a/src/parseable/mod.rs b/src/parseable/mod.rs
index a3220945a..b1c61d9ad 100644
--- a/src/parseable/mod.rs
+++ b/src/parseable/mod.rs
@@ -49,7 +49,7 @@ use crate::{
             logstream::error::{CreateStreamError, StreamError},
             modal::{
                 utils::logstream_utils::PutStreamHeaders, IndexerMetadata, IngestorMetadata,
-                NodeType,
+                NodeType, QuerierMetadata,
             },
         },
         STREAM_TYPE_KEY,
@@ -130,6 +130,8 @@ pub struct Parseable {
     /// Metadata and staging realting to each logstreams
     /// A globally shared mapping of `Streams` that parseable is aware of.
     pub streams: Streams,
+    /// Metadata associated only with a querier
+    pub querier_metadata: Option<Arc<QuerierMetadata>>,
     /// Metadata associated only with an ingestor
     pub ingestor_metadata: Option<Arc<IngestorMetadata>>,
     /// Metadata associated only with an indexer
@@ -161,12 +163,21 @@ impl Parseable {
             )),
             _ => None,
         };
+        let querier_metadata = match &options.mode {
+            Mode::Query => Some(QuerierMetadata::load(
+                &options,
+                storage.as_ref(),
+                NodeType::Querier,
+            )),
+            _ => None,
+        };
         Parseable {
             options: Arc::new(options),
             storage,
             streams: Streams::default(),
             ingestor_metadata,
             indexer_metadata,
+            querier_metadata,
             #[cfg(feature = "kafka")]
             kafka_config,
         }
@@ -275,69 +286,41 @@ impl Parseable {
         }
     }
 
-    // create the ingestor metadata and put the .ingestor.json file in the object store
     pub async fn store_metadata(&self, mode: Mode) -> anyhow::Result<()> {
-        match mode {
-            Mode::Ingest => {
-                let Some(meta) = self.ingestor_metadata.as_ref() else {
-                    return Ok(());
-                };
-                let storage_ingestor_metadata = meta.migrate().await?;
-                let store = self.storage.get_object_store();
-
-                // use the id that was generated/found in the staging and
-                // generate the path for the object store
-                let path = meta.file_path();
-
-                // we are considering that we can always get from object store
-                if let Some(mut store_data) = storage_ingestor_metadata {
-                    if store_data.domain_name != meta.domain_name {
-                        store_data.domain_name.clone_from(&meta.domain_name);
-                        store_data.port.clone_from(&meta.port);
-
-                        let resource = Bytes::from(serde_json::to_vec(&store_data)?);
-
-                        // if pushing to object store fails propagate the error
-                        store.put_object(&path, resource).await?;
-                    }
-                } else {
-                    let resource = serde_json::to_vec(&meta)?.into();
+        // Get the appropriate metadata based on mode
+        let meta_ref = match mode {
+            Mode::Ingest => self.ingestor_metadata.as_ref(),
+            Mode::Index => self.indexer_metadata.as_ref(),
+            Mode::Query => self.querier_metadata.as_ref(),
+            _ => return Err(anyhow::anyhow!("Invalid mode")),
+        };
 
-                    store.put_object(&path, resource).await?;
-                }
-                Ok(())
-            }
-            Mode::Index => {
-                let Some(meta) = self.indexer_metadata.as_ref() else {
-                    return Ok(());
-                };
-                let storage_indexer_metadata = meta.migrate().await?;
-                let store = self.storage.get_object_store();
-
-                // use the id that was generated/found in the staging and
-                // generate the path for the object store
-                let path = meta.file_path();
-
-                // we are considering that we can always get from object store
-                if let Some(mut store_data) = storage_indexer_metadata {
-                    if store_data.domain_name != meta.domain_name {
-                        store_data.domain_name.clone_from(&meta.domain_name);
-                        store_data.port.clone_from(&meta.port);
-
-                        let resource = Bytes::from(serde_json::to_vec(&store_data)?);
-
-                        // if pushing to object store fails propagate the error
-                        store.put_object(&path, resource).await?;
-                    }
-                } else {
-                    let resource = serde_json::to_vec(&meta)?.into();
+        let Some(meta) = meta_ref else {
+            return Ok(());
+        };
 
-                    store.put_object(&path, resource).await?;
-                }
-                Ok(())
+        // Get storage metadata
+        let storage_metadata = meta.migrate().await?;
+        let store = self.storage.get_object_store();
+        let path = meta.file_path();
+
+        // Process metadata and store it
+        if let Some(mut store_data) = storage_metadata {
+            // Update domain and port if needed
+            if store_data.domain_name != meta.domain_name {
+                store_data.domain_name.clone_from(&meta.domain_name);
+                store_data.port.clone_from(&meta.port);
+
+                let resource = Bytes::from(serde_json::to_vec(&store_data)?);
+                store.put_object(&path, resource).await?;
             }
-            _ => Err(anyhow::anyhow!("Invalid mode")),
+        } else {
+            // Store new metadata
+            let resource = serde_json::to_vec(&meta)?.into();
+            store.put_object(&path, resource).await?;
         }
+
+        Ok(())
     }
 
     /// list all streams from storage
diff --git a/src/utils/mod.rs b/src/utils/mod.rs
index d2ba24b54..f2e2685d6 100644
--- a/src/utils/mod.rs
+++ b/src/utils/mod.rs
@@ -36,19 +36,10 @@ use chrono::{NaiveDate, NaiveDateTime, NaiveTime, Utc};
 use datafusion::common::tree_node::TreeNode;
 use regex::Regex;
 use sha2::{Digest, Sha256};
-use tracing::debug;
 
-pub fn get_ingestor_id() -> String {
+pub fn get_node_id() -> String {
     let now = Utc::now().to_rfc3339();
     let id = get_hash(&now).to_string().split_at(15).0.to_string();
-    debug!("Ingestor ID: {id}");
-    id
-}
-
-pub fn get_indexer_id() -> String {
-    let now = Utc::now().to_rfc3339();
-    let id = get_hash(&now).to_string().split_at(15).0.to_string();
-    debug!("Indexer ID: {id}");
     id
 }
 

From 59b98679ce55fe13799cbc4e88ac545fa4cc8968 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 7 Apr 2025 08:10:51 -0400
Subject: [PATCH 03/42] remove writing query metadata to storage

---
 src/handlers/http/modal/query_server.rs | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/handlers/http/modal/query_server.rs b/src/handlers/http/modal/query_server.rs
index 44a208234..7fc4df57a 100644
--- a/src/handlers/http/modal/query_server.rs
+++ b/src/handlers/http/modal/query_server.rs
@@ -130,9 +130,6 @@ impl ParseableServer for QueryServer {
 
         tokio::spawn(airplane::server());
 
-        // write the querier metadata to storage
-        PARSEABLE.store_metadata(Mode::Query).await?;
-
         let result = self
             .start(shutdown_rx, prometheus.clone(), PARSEABLE.options.openid())
             .await?;

From dcd9b1b66485840a8b00683e416b18f054b71d8e Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 7 Apr 2025 08:14:51 -0400
Subject: [PATCH 04/42] remove unused

---
 src/handlers/http/modal/query_server.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/handlers/http/modal/query_server.rs b/src/handlers/http/modal/query_server.rs
index 7fc4df57a..d4ce26f72 100644
--- a/src/handlers/http/modal/query_server.rs
+++ b/src/handlers/http/modal/query_server.rs
@@ -25,7 +25,6 @@ use crate::handlers::http::{base_path, prism_base_path};
 use crate::handlers::http::{logstream, MAX_EVENT_PAYLOAD_SIZE};
 use crate::handlers::http::{rbac, role};
 use crate::hottier::HotTierManager;
-use crate::option::Mode;
 use crate::rbac::role::Action;
 use crate::{analytics, migration, storage, sync};
 use actix_web::web::{resource, ServiceConfig};

From a718691c0474d686be1f3eb9fafde851b332d43b Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 7 Apr 2025 08:31:19 -0400
Subject: [PATCH 05/42] clone from

---
 src/handlers/http/modal/mod.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/handlers/http/modal/mod.rs b/src/handlers/http/modal/mod.rs
index 8580c95b2..e43967dde 100644
--- a/src/handlers/http/modal/mod.rs
+++ b/src/handlers/http/modal/mod.rs
@@ -340,7 +340,7 @@ impl NodeMetadata {
                 meta.token = token;
             }
 
-            meta.node_type = node_type.clone();
+            meta.node_type.clone_from(&node_type);
             meta.put_on_disk(staging_path)
                 .expect("Couldn't write to disk");
 
@@ -436,7 +436,7 @@ impl NodeMetadata {
         };
 
         let mut resource = Self::from_bytes(&bytes, PARSEABLE.options.flight_port)?;
-        resource.node_type = self.node_type.clone();
+        resource.node_type.clone_from(&self.node_type);
         let bytes = Bytes::from(serde_json::to_vec(&resource)?);
 
         resource.put_on_disk(PARSEABLE.options.staging_dir())?;

From 358af90c77ebc29a9b7dc96c10f3d23c5d10b004 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 7 Apr 2025 08:50:01 -0400
Subject: [PATCH 06/42] split into smaller functions

---
 src/handlers/http/modal/mod.rs | 169 +++++++++++++++++++--------------
 1 file changed, 100 insertions(+), 69 deletions(-)

diff --git a/src/handlers/http/modal/mod.rs b/src/handlers/http/modal/mod.rs
index e43967dde..7d3097274 100644
--- a/src/handlers/http/modal/mod.rs
+++ b/src/handlers/http/modal/mod.rs
@@ -218,6 +218,14 @@ impl NodeType {
             NodeType::Querier => "querier",
         }
     }
+
+    fn to_mode(&self) -> Mode {
+        match self {
+            NodeType::Ingestor => Mode::Ingest,
+            NodeType::Indexer => Mode::Index,
+            NodeType::Querier => Mode::Query,
+        }
+    }
 }
 
 impl fmt::Display for NodeType {
@@ -274,96 +282,119 @@ impl NodeMetadata {
         storage: &dyn ObjectStorageProvider,
         node_type: NodeType,
     ) -> Arc<Self> {
-        // all the files should be in the staging directory root
-        let entries = options
-            .staging_dir()
-            .read_dir()
-            .expect("Couldn't read from file");
+        let staging_path = options.staging_dir();
+        let node_type_str = node_type.as_str();
 
-        let mode = match node_type {
-            NodeType::Ingestor => Mode::Ingest,
-            NodeType::Indexer => Mode::Index,
-            NodeType::Querier => Mode::Query,
-        };
+        // Attempt to load metadata from staging
+        if let Some(mut meta) = Self::load_from_staging(staging_path, node_type_str, options) {
+            Self::update_metadata(&mut meta, options, node_type);
+            meta.put_on_disk(staging_path)
+                .expect("Couldn't write updated metadata to disk");
+            return Arc::new(meta);
+        }
 
-        let url = options.get_url(mode);
-        let port = url.port().unwrap_or(80).to_string();
-        let url = url.to_string();
-        let Options {
-            username, password, ..
-        } = options;
-        let staging_path = options.staging_dir();
-        let flight_port = options.flight_port.to_string();
-        let type_str = node_type.as_str();
+        // If no metadata is found in staging, create a new one
+        let meta = Self::create_new_metadata(options, storage, node_type);
+        meta.put_on_disk(staging_path)
+            .expect("Couldn't write new metadata to disk");
+        Arc::new(meta)
+    }
+
+    /// Load metadata from the staging directory
+    fn load_from_staging(
+        staging_path: &Path,
+        node_type_str: &str,
+        options: &Options,
+    ) -> Option<Self> {
+        let entries = staging_path
+            .read_dir()
+            .expect("Couldn't read from staging directory");
 
         for entry in entries {
-            // the staging directory will have only one file with the node type in the name
-            // so the JSON Parse should not error unless the file is corrupted
             let path = entry.expect("Should be a directory entry").path();
-            if !path
-                .file_name()
-                .and_then(|s| s.to_str())
-                .is_some_and(|s| s.contains(type_str))
-            {
+            if !Self::is_valid_metadata_file(&path, node_type_str) {
                 continue;
             }
 
-            // get the metadata from staging
-            let bytes = std::fs::read(path).expect("File should be present");
-            let mut meta = Self::from_bytes(&bytes, options.flight_port)
-                .unwrap_or_else(|_| panic!("Extracted {} metadata", type_str));
-
-            // compare url endpoint and port, update
-            if meta.domain_name != url {
-                info!(
-                    "Domain Name was Updated. Old: {} New: {}",
-                    meta.domain_name, url
-                );
-                meta.domain_name = url;
+            let bytes = std::fs::read(&path).expect("File should be present");
+            match Self::from_bytes(&bytes, options.flight_port) {
+                Ok(meta) => return Some(meta),
+                Err(e) => {
+                    error!("Failed to extract {} metadata: {}", node_type_str, e);
+                    return None;
+                }
             }
+        }
 
-            if meta.port != port {
-                info!("Port was Updated. Old: {} New: {}", meta.port, port);
-                meta.port = port;
-            }
+        None
+    }
+
+    /// Check if a file is a valid metadata file for the given node type
+    fn is_valid_metadata_file(path: &Path, node_type_str: &str) -> bool {
+        path.file_name()
+            .and_then(|s| s.to_str())
+            .map_or(false, |s| s.contains(node_type_str))
+    }
+
+    /// Update metadata fields if they differ from the current configuration
+    fn update_metadata(meta: &mut Self, options: &Options, node_type: NodeType) {
+        let url = options.get_url(node_type.to_mode());
+        let port = url.port().unwrap_or(80).to_string();
+        let url = url.to_string();
 
-            let token = format!(
-                "Basic {}",
-                BASE64_STANDARD.encode(format!("{username}:{password}"))
+        if meta.domain_name != url {
+            info!(
+                "Domain Name was Updated. Old: {} New: {}",
+                meta.domain_name, url
             );
-            if meta.token != token {
-                // TODO: Update the message to be more informative with username and password
-                warn!(
-                    "Credentials were Updated. Tokens updated; Old: {} New: {}",
-                    meta.token, token
-                );
-                meta.token = token;
-            }
+            meta.domain_name = url;
+        }
 
-            meta.node_type.clone_from(&node_type);
-            meta.put_on_disk(staging_path)
-                .expect("Couldn't write to disk");
+        if meta.port != port {
+            info!("Port was Updated. Old: {} New: {}", meta.port, port);
+            meta.port = port;
+        }
 
-            return Arc::new(meta);
+        let token = Self::generate_token(&options.username, &options.password);
+        if meta.token != token {
+            warn!(
+                "Credentials were Updated. Tokens updated; Old: {} New: {}",
+                meta.token, token
+            );
+            meta.token = token;
         }
 
-        let storage = storage.get_object_store();
-        let node_id = get_node_id();
+        meta.node_type = node_type;
+    }
+
+    /// Create a new metadata instance
+    fn create_new_metadata(
+        options: &Options,
+        storage: &dyn ObjectStorageProvider,
+        node_type: NodeType,
+    ) -> Self {
+        let url = options.get_url(node_type.to_mode());
+        let port = url.port().unwrap_or(80).to_string();
+        let url = url.to_string();
 
-        let meta = Self::new(
+        Self::new(
             port,
             url,
-            storage.get_bucket_name(),
-            username,
-            password,
-            node_id,
-            flight_port,
+            storage.get_object_store().get_bucket_name(),
+            &options.username,
+            &options.password,
+            get_node_id(),
+            options.flight_port.to_string(),
             node_type,
-        );
+        )
+    }
 
-        meta.put_on_disk(staging_path)
-            .expect("Should Be valid Json");
-        Arc::new(meta)
+    /// Generate a token from the username and password
+    fn generate_token(username: &str, password: &str) -> String {
+        format!(
+            "Basic {}",
+            BASE64_STANDARD.encode(format!("{username}:{password}"))
+        )
     }
 
     pub fn get_node_id(&self) -> String {

From 1c1fb339ba3b31cbc3c3204eb0c31ec74f3b0ac3 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 7 Apr 2025 08:54:41 -0400
Subject: [PATCH 07/42] clippy suggestions

---
 src/handlers/http/modal/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/handlers/http/modal/mod.rs b/src/handlers/http/modal/mod.rs
index 7d3097274..41a548bff 100644
--- a/src/handlers/http/modal/mod.rs
+++ b/src/handlers/http/modal/mod.rs
@@ -333,7 +333,7 @@ impl NodeMetadata {
     fn is_valid_metadata_file(path: &Path, node_type_str: &str) -> bool {
         path.file_name()
             .and_then(|s| s.to_str())
-            .map_or(false, |s| s.contains(node_type_str))
+            .is_some_and(|s| s.contains(node_type_str))
     }
 
     /// Update metadata fields if they differ from the current configuration

From d39fb47bc24d07e534eaac1aec7ee4b46e801b4f Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 7 Apr 2025 09:24:04 -0400
Subject: [PATCH 08/42] helper function

---
 src/handlers/http/modal/mod.rs | 63 +++++++++++++++++++---------------
 1 file changed, 36 insertions(+), 27 deletions(-)

diff --git a/src/handlers/http/modal/mod.rs b/src/handlers/http/modal/mod.rs
index 41a548bff..c4cf63bef 100644
--- a/src/handlers/http/modal/mod.rs
+++ b/src/handlers/http/modal/mod.rs
@@ -306,12 +306,22 @@ impl NodeMetadata {
         node_type_str: &str,
         options: &Options,
     ) -> Option<Self> {
-        let entries = staging_path
-            .read_dir()
-            .expect("Couldn't read from staging directory");
+        let entries = match staging_path.read_dir() {
+            Ok(entries) => entries,
+            Err(e) => {
+                error!("Couldn't read from staging directory: {}", e);
+                return None;
+            }
+        };
 
         for entry in entries {
-            let path = entry.expect("Should be a directory entry").path();
+            let path = match entry {
+                Ok(entry) => entry.path(),
+                Err(e) => {
+                    error!("Error reading directory entry: {}", e);
+                    continue;
+                }
+            };
             if !Self::is_valid_metadata_file(&path, node_type_str) {
                 continue;
             }
@@ -417,32 +427,31 @@ impl NodeMetadata {
         let version = json.get("version").and_then(|version| version.as_str());
 
         if version == Some("v3") {
-            if json.contains_key("ingestor_id") {
-                // Migration: get ingestor_id value, remove it, and add as node_id
-                if let Some(id) = json.remove("ingestor_id") {
-                    json.insert("node_id".to_string(), id);
+            fn migrate_legacy_id(
+                json: &mut Map<String, Value>,
+                legacy_id_key: &str,
+                node_type_str: &str,
+            ) -> bool {
+                if json.contains_key(legacy_id_key) {
+                    if let Some(id) = json.remove(legacy_id_key) {
+                        json.insert("node_id".to_string(), id);
+                        json.insert(
+                            "version".to_string(),
+                            Value::String(DEFAULT_VERSION.to_string()),
+                        );
+                    }
                     json.insert(
-                        "version".to_string(),
-                        Value::String(DEFAULT_VERSION.to_string()),
+                        "node_type".to_string(),
+                        Value::String(node_type_str.to_string()),
                     );
+                    true
+                } else {
+                    false
                 }
-                json.insert(
-                    "node_type".to_string(),
-                    Value::String("ingestor".to_string()),
-                );
-            } else if json.contains_key("indexer_id") {
-                // Migration: get indexer_id value, remove it, and add as node_id
-                if let Some(id) = json.remove("indexer_id") {
-                    json.insert("node_id".to_string(), id);
-                    json.insert(
-                        "version".to_string(),
-                        Value::String(DEFAULT_VERSION.to_string()),
-                    );
-                }
-                json.insert(
-                    "node_type".to_string(),
-                    Value::String("indexer".to_string()),
-                );
+            }
+
+            if !migrate_legacy_id(&mut json, "ingestor_id", "ingestor") {
+                migrate_legacy_id(&mut json, "indexer_id", "indexer");
             }
         }
         // Determine node type and perform migration if needed

From bef79842f971976bc106ab9d6e84818dc4cc0f94 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 7 Apr 2025 14:28:57 -0400
Subject: [PATCH 09/42] add Prism mode

---
 src/catalog/mod.rs              |  6 ++---
 src/handlers/http/middleware.rs |  2 +-
 src/handlers/http/modal/mod.rs  |  6 +----
 src/main.rs                     |  4 +++
 src/option.rs                   |  1 +
 src/parseable/mod.rs            |  1 +
 src/storage/object_storage.rs   | 48 +++++++++++++++------------------
 src/storage/store_metadata.rs   |  2 +-
 8 files changed, 34 insertions(+), 36 deletions(-)

diff --git a/src/catalog/mod.rs b/src/catalog/mod.rs
index 1f69a071a..23f732053 100644
--- a/src/catalog/mod.rs
+++ b/src/catalog/mod.rs
@@ -343,10 +343,10 @@ pub async fn remove_manifest_from_snapshot(
             Ok(get_first_event(storage.clone(), stream_name, Vec::new()).await?)
         }
         Mode::Query => Ok(get_first_event(storage, stream_name, dates).await?),
-        Mode::Index => Err(ObjectStorageError::UnhandledError(Box::new(
+        Mode::Index | Mode::Prism => Err(ObjectStorageError::UnhandledError(Box::new(
             std::io::Error::new(
                 std::io::ErrorKind::Unsupported,
-                "Can't remove manifest from within Index server",
+                "Can't remove manifest from within Index or Prism server",
             ),
         ))),
     }
@@ -359,7 +359,7 @@ pub async fn get_first_event(
 ) -> Result<Option<String>, ObjectStorageError> {
     let mut first_event_at: String = String::default();
     match PARSEABLE.options.mode {
-        Mode::Index => unimplemented!(),
+        Mode::Index | Mode::Prism => unimplemented!(),
         Mode::All | Mode::Ingest => {
             // get current snapshot
             let stream_first_event = PARSEABLE.get_stream(stream_name)?.get_first_event();
diff --git a/src/handlers/http/middleware.rs b/src/handlers/http/middleware.rs
index d329174bd..4c1e92f39 100644
--- a/src/handlers/http/middleware.rs
+++ b/src/handlers/http/middleware.rs
@@ -358,7 +358,7 @@ where
                 })
             }
 
-            Mode::Index => {
+            Mode::Index | Mode::Prism => {
                 let fut = self.service.call(req);
 
                 Box::pin(async move {
diff --git a/src/handlers/http/modal/mod.rs b/src/handlers/http/modal/mod.rs
index c4cf63bef..2bafcb792 100644
--- a/src/handlers/http/modal/mod.rs
+++ b/src/handlers/http/modal/mod.rs
@@ -230,11 +230,7 @@ impl NodeType {
 
 impl fmt::Display for NodeType {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            NodeType::Ingestor => write!(f, "ingestor"),
-            NodeType::Indexer => write!(f, "indexer"),
-            NodeType::Querier => write!(f, "querier"),
-        }
+        write!(f, "{}", self.as_str())
     }
 }
 
diff --git a/src/main.rs b/src/main.rs
index 5894b304f..ca25702a1 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -43,6 +43,10 @@ async fn main() -> anyhow::Result<()> {
             println!("Indexing is an enterprise feature. Check out https://www.parseable.com/pricing to know more!");
             exit(0)
         }
+        Mode::Prism => {
+            println!("Prism is an enterprise feature. Check out https://www.parseable.com/pricing to know more!");
+            exit(0)
+        }
         Mode::All => Box::new(Server),
     };
 
diff --git a/src/option.rs b/src/option.rs
index 0e659905b..e9a17f18d 100644
--- a/src/option.rs
+++ b/src/option.rs
@@ -23,6 +23,7 @@ pub enum Mode {
     Query,
     Ingest,
     Index,
+    Prism,
     #[default]
     All,
 }
diff --git a/src/parseable/mod.rs b/src/parseable/mod.rs
index b1c61d9ad..05551818c 100644
--- a/src/parseable/mod.rs
+++ b/src/parseable/mod.rs
@@ -282,6 +282,7 @@ impl Parseable {
             Mode::Query => "Distributed (Query)",
             Mode::Ingest => "Distributed (Ingest)",
             Mode::Index => "Distributed (Index)",
+            Mode::Prism => "Distributed (Prism)",
             Mode::All => "Standalone",
         }
     }
diff --git a/src/storage/object_storage.rs b/src/storage/object_storage.rs
index e20a9556c..a41bb8447 100644
--- a/src/storage/object_storage.rs
+++ b/src/storage/object_storage.rs
@@ -895,40 +895,36 @@ pub fn to_bytes(any: &(impl ?Sized + serde::Serialize)) -> Bytes {
 }
 
 pub fn schema_path(stream_name: &str) -> RelativePathBuf {
-    match &PARSEABLE.options.mode {
-        Mode::Ingest => {
-            let id = PARSEABLE
-                .ingestor_metadata
-                .as_ref()
-                .expect(INGESTOR_EXPECT)
-                .get_node_id();
-            let file_name = format!(".ingestor.{id}{SCHEMA_FILE_NAME}");
-
-            RelativePathBuf::from_iter([stream_name, STREAM_ROOT_DIRECTORY, &file_name])
-        }
-        Mode::All | Mode::Query | Mode::Index => {
-            RelativePathBuf::from_iter([stream_name, STREAM_ROOT_DIRECTORY, SCHEMA_FILE_NAME])
-        }
+    if PARSEABLE.options.mode == Mode::Ingest {
+        let id = PARSEABLE
+            .ingestor_metadata
+            .as_ref()
+            .expect(INGESTOR_EXPECT)
+            .get_node_id();
+        let file_name = format!(".ingestor.{id}{SCHEMA_FILE_NAME}");
+
+        RelativePathBuf::from_iter([stream_name, STREAM_ROOT_DIRECTORY, &file_name])
+    } else {
+        RelativePathBuf::from_iter([stream_name, STREAM_ROOT_DIRECTORY, SCHEMA_FILE_NAME])
     }
 }
 
 #[inline(always)]
 pub fn stream_json_path(stream_name: &str) -> RelativePathBuf {
-    match &PARSEABLE.options.mode {
-        Mode::Ingest => {
-            let id = PARSEABLE
-                .ingestor_metadata
-                .as_ref()
-                .expect(INGESTOR_EXPECT)
-                .get_node_id();
-            let file_name = format!(".ingestor.{id}{STREAM_METADATA_FILE_NAME}",);
-            RelativePathBuf::from_iter([stream_name, STREAM_ROOT_DIRECTORY, &file_name])
-        }
-        Mode::Query | Mode::All | Mode::Index => RelativePathBuf::from_iter([
+    if PARSEABLE.options.mode == Mode::Ingest {
+        let id = PARSEABLE
+            .ingestor_metadata
+            .as_ref()
+            .expect(INGESTOR_EXPECT)
+            .get_node_id();
+        let file_name = format!(".ingestor.{id}{STREAM_METADATA_FILE_NAME}",);
+        RelativePathBuf::from_iter([stream_name, STREAM_ROOT_DIRECTORY, &file_name])
+    } else {
+        RelativePathBuf::from_iter([
             stream_name,
             STREAM_ROOT_DIRECTORY,
             STREAM_METADATA_FILE_NAME,
-        ]),
+        ])
     }
 }
 
diff --git a/src/storage/store_metadata.rs b/src/storage/store_metadata.rs
index 89f638161..72e0e8ac0 100644
--- a/src/storage/store_metadata.rs
+++ b/src/storage/store_metadata.rs
@@ -159,7 +159,7 @@ pub async fn resolve_parseable_metadata(
                         metadata.server_mode = PARSEABLE.options.mode;
                         metadata.staging = PARSEABLE.options.staging_dir().to_path_buf();
                       },
-                    Mode::Index => {
+                    Mode::Index | Mode::Prism => {
                         // if index server is started fetch the metadata from remote
                         // update the server mode for local metadata
                         metadata.server_mode = PARSEABLE.options.mode;

From 4145db24de9555e531795a393a4cdd55f8e6bee1 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Tue, 8 Apr 2025 00:42:21 -0400
Subject: [PATCH 10/42] prism mode to option

---
 src/option.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/option.rs b/src/option.rs
index e9a17f18d..aca27bb49 100644
--- a/src/option.rs
+++ b/src/option.rs
@@ -129,6 +129,7 @@ pub mod validation {
         match s {
             "query" => Ok(Mode::Query),
             "ingest" => Ok(Mode::Ingest),
+            "prism" => Ok(Mode::Prism),
             "all" => Ok(Mode::All),
             "index" => Ok(Mode::Index),
             _ => Err("Invalid MODE provided".to_string()),

From 8ba0f90b9a1d29969c04c7840b6d733a69c857e7 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Tue, 8 Apr 2025 01:03:39 -0400
Subject: [PATCH 11/42] add serde error to QueryError

---
 src/handlers/http/query.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/handlers/http/query.rs b/src/handlers/http/query.rs
index 3b00268b2..06f90ea96 100644
--- a/src/handlers/http/query.rs
+++ b/src/handlers/http/query.rs
@@ -325,6 +325,8 @@ Description: {0}"#
     Anyhow(#[from] anyhow::Error),
     #[error("Error: {0}")]
     StreamNotFound(#[from] StreamNotFound),
+    #[error("SerdeJsonError: {0}")]
+    SerdeJsonError(#[from] serde_json::Error),
 }
 
 impl actix_web::ResponseError for QueryError {

From f5c2088351156e7f96a4c0207cba49de28769d35 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Tue, 8 Apr 2025 01:24:01 -0400
Subject: [PATCH 12/42] add custom error

---
 src/handlers/http/query.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/handlers/http/query.rs b/src/handlers/http/query.rs
index 06f90ea96..c1d38ad3d 100644
--- a/src/handlers/http/query.rs
+++ b/src/handlers/http/query.rs
@@ -327,6 +327,8 @@ Description: {0}"#
     StreamNotFound(#[from] StreamNotFound),
     #[error("SerdeJsonError: {0}")]
     SerdeJsonError(#[from] serde_json::Error),
+    #[error("CustomError: {0}")]
+    CustomError(String),
 }
 
 impl actix_web::ResponseError for QueryError {

From 4e4c38f4fe3cfd518ff3367d415869cae8b361da Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Tue, 8 Apr 2025 03:32:32 -0400
Subject: [PATCH 13/42] overwrite remote for Prism mode

---
 src/storage/store_metadata.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/storage/store_metadata.rs b/src/storage/store_metadata.rs
index 72e0e8ac0..a33a8a5d0 100644
--- a/src/storage/store_metadata.rs
+++ b/src/storage/store_metadata.rs
@@ -148,7 +148,7 @@ pub async fn resolve_parseable_metadata(
                             })?;
                             overwrite_remote = true;
                     },
-                    Mode::Query => {
+                    Mode::Query | Mode::Prism => {
                         overwrite_remote = true;
                         metadata.server_mode = PARSEABLE.options.mode;
                         metadata.staging = PARSEABLE.options.staging_dir().to_path_buf();
@@ -159,7 +159,7 @@ pub async fn resolve_parseable_metadata(
                         metadata.server_mode = PARSEABLE.options.mode;
                         metadata.staging = PARSEABLE.options.staging_dir().to_path_buf();
                       },
-                    Mode::Index | Mode::Prism => {
+                    Mode::Index => {
                         // if index server is started fetch the metadata from remote
                         // update the server mode for local metadata
                         metadata.server_mode = PARSEABLE.options.mode;
@@ -175,7 +175,7 @@ pub async fn resolve_parseable_metadata(
             // new metadata needs to be set
             // if mode is query or all then both staging and remote
             match PARSEABLE.options.mode {
-                Mode::All | Mode::Query => overwrite_remote = true,
+                Mode::All | Mode::Query | Mode::Prism => overwrite_remote = true,
                 _ => (),
             }
             // else only staging

From 2599aa83aec5ea2ded5ffe8ed13758a2596bebec Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Tue, 8 Apr 2025 05:20:58 -0400
Subject: [PATCH 14/42] make public

---
 src/storage/store_metadata.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/storage/store_metadata.rs b/src/storage/store_metadata.rs
index a33a8a5d0..edcdb7e27 100644
--- a/src/storage/store_metadata.rs
+++ b/src/storage/store_metadata.rs
@@ -202,7 +202,7 @@ pub async fn resolve_parseable_metadata(
     Ok(metadata)
 }
 
-fn determine_environment(
+pub fn determine_environment(
     staging_metadata: Option<StorageMetadata>,
     remote_metadata: Option<StorageMetadata>,
 ) -> EnvChange {

From 166b2cb13f032b44381e20b10959d8252cc4a41b Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Tue, 8 Apr 2025 05:29:51 -0400
Subject: [PATCH 15/42] make mod store_metadata public

---
 src/storage/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index b6bc9bb25..63e3803bd 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -41,7 +41,7 @@ mod metrics_layer;
 pub mod object_storage;
 pub mod retention;
 mod s3;
-mod store_metadata;
+pub mod store_metadata;
 
 use self::retention::Retention;
 pub use azure_blob::AzureBlobConfig;

From 303b7c05dec0b61375a1cd48d44644ff7d72a5db Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Tue, 8 Apr 2025 06:44:51 -0400
Subject: [PATCH 16/42] active/inactive indexers and queriers in analytics

---
 src/analytics.rs | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/src/analytics.rs b/src/analytics.rs
index 0e6d43618..5c71ab371 100644
--- a/src/analytics.rs
+++ b/src/analytics.rs
@@ -75,6 +75,10 @@ pub struct Report {
     commit_hash: String,
     active_ingestors: u64,
     inactive_ingestors: u64,
+    active_indexers: u64,
+    inactive_indexers: u64,
+    active_queries: u64,
+    inactive_queries: u64,
     stream_count: usize,
     total_events_count: u64,
     total_json_bytes: u64,
@@ -107,7 +111,28 @@ impl Report {
             mem_total = info.total_memory();
         }
         let ingestor_metrics = fetch_ingestors_metrics().await?;
+        let mut active_indexers = 0;
+        let mut inactive_indexers = 0;
+        let mut active_queries = 0;
+        let mut inactive_queries = 0;
+
+        let indexer_infos: Vec<NodeMetadata> = cluster::get_node_info("indexer").await?;
+        for indexer in indexer_infos {
+            if check_liveness(&indexer.domain_name).await {
+                active_indexers += 1;
+            } else {
+                inactive_indexers += 1;
+            }
+        }
 
+        let query_infos: Vec<NodeMetadata> = cluster::get_node_info("query").await?;
+        for query in query_infos {
+            if check_liveness(&query.domain_name).await {
+                active_queries += 1;
+            } else {
+                inactive_queries += 1;
+            }
+        }
         Ok(Self {
             deployment_id: storage::StorageMetadata::global().deployment_id,
             uptime: upt,
@@ -123,6 +148,10 @@ impl Report {
             commit_hash: current().commit_hash,
             active_ingestors: ingestor_metrics.0,
             inactive_ingestors: ingestor_metrics.1,
+            active_indexers,
+            inactive_indexers,
+            active_queries,
+            inactive_queries,
             stream_count: ingestor_metrics.2,
             total_events_count: ingestor_metrics.3,
             total_json_bytes: ingestor_metrics.4,
@@ -225,11 +254,11 @@ async fn fetch_ingestors_metrics(
     let mut vec = vec![];
     let mut active_ingestors = 0u64;
     let mut offline_ingestors = 0u64;
-    if PARSEABLE.options.mode == Mode::Query {
+    if PARSEABLE.options.mode == Mode::Query || PARSEABLE.options.mode == Mode::Prism {
         // send analytics for ingest servers
 
         // ingestor infos should be valid here, if not some thing is wrong
-        let ingestor_infos: Vec<NodeMetadata> = cluster::get_node_info("ingestor").await.unwrap();
+        let ingestor_infos: Vec<NodeMetadata> = cluster::get_node_info("ingestor").await?;
 
         for im in ingestor_infos {
             if !check_liveness(&im.domain_name).await {

From 64ce9158346a5d55ab34eb5d9b8c8531bbaf2e46 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Tue, 8 Apr 2025 07:21:28 -0400
Subject: [PATCH 17/42] add error to query error

---
 src/handlers/http/query.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/handlers/http/query.rs b/src/handlers/http/query.rs
index c1d38ad3d..bab407f28 100644
--- a/src/handlers/http/query.rs
+++ b/src/handlers/http/query.rs
@@ -329,6 +329,8 @@ Description: {0}"#
     SerdeJsonError(#[from] serde_json::Error),
     #[error("CustomError: {0}")]
     CustomError(String),
+    #[error("No available queriers found")]
+    NoAvailableQuerier,
 }
 
 impl actix_web::ResponseError for QueryError {

From 074b44c3a661b39876d860377dea439d0bf16d47 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Tue, 8 Apr 2025 14:31:51 -0400
Subject: [PATCH 18/42] add prism check for various methods

---
 src/handlers/http/about.rs          | 3 ++-
 src/handlers/http/query.rs          | 6 +-----
 src/parseable/mod.rs                | 3 ++-
 src/query/mod.rs                    | 2 +-
 src/query/stream_schema_provider.rs | 2 +-
 src/utils/arrow/flight.rs           | 3 ++-
 6 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/src/handlers/http/about.rs b/src/handlers/http/about.rs
index 675d0182e..ee26e95d5 100644
--- a/src/handlers/http/about.rs
+++ b/src/handlers/http/about.rs
@@ -63,7 +63,8 @@ pub async fn about() -> Json<Value> {
     let commit = current_release.commit_hash;
     let deployment_id = meta.deployment_id.to_string();
     let mode = PARSEABLE.get_server_mode_string();
-    let staging = if PARSEABLE.options.mode == Mode::Query {
+    let staging = if PARSEABLE.options.mode == Mode::Query || PARSEABLE.options.mode == Mode::Prism
+    {
         "".to_string()
     } else {
         PARSEABLE.options.staging_dir().display().to_string()
diff --git a/src/handlers/http/query.rs b/src/handlers/http/query.rs
index bab407f28..8ea9c316b 100644
--- a/src/handlers/http/query.rs
+++ b/src/handlers/http/query.rs
@@ -45,7 +45,6 @@ use crate::query::{execute, CountsRequest, CountsResponse, Query as LogicalQuery
 use crate::query::{TableScanVisitor, QUERY_SESSION};
 use crate::rbac::Users;
 use crate::response::{QueryResponse, TIME_ELAPSED_HEADER};
-use crate::storage::object_storage::commit_schema_to_storage;
 use crate::storage::ObjectStorageError;
 use crate::utils::actix::extract_session_key_from_req;
 use crate::utils::time::{TimeParseError, TimeRange};
@@ -173,12 +172,9 @@ pub async fn get_counts(
 }
 
 pub async fn update_schema_when_distributed(tables: &Vec<String>) -> Result<(), EventError> {
-    if PARSEABLE.options.mode == Mode::Query {
+    if PARSEABLE.options.mode == Mode::Query || PARSEABLE.options.mode == Mode::Prism {
         for table in tables {
             if let Ok(new_schema) = fetch_schema(table).await {
-                // commit schema merges the schema internally and updates the schema in storage.
-                commit_schema_to_storage(table, new_schema.clone()).await?;
-
                 commit_schema(table, Arc::new(new_schema))?;
             }
         }
diff --git a/src/parseable/mod.rs b/src/parseable/mod.rs
index 05551818c..d52613dfe 100644
--- a/src/parseable/mod.rs
+++ b/src/parseable/mod.rs
@@ -214,6 +214,7 @@ impl Parseable {
     pub async fn check_or_load_stream(&self, stream_name: &str) -> bool {
         !self.streams.contains(stream_name)
             && (self.options.mode != Mode::Query
+                || self.options.mode != Mode::Prism
                 || !self
                     .create_stream_and_schema_from_storage(stream_name)
                     .await
@@ -521,7 +522,7 @@ impl Parseable {
         let stream_in_memory_dont_update =
             self.streams.contains(stream_name) && !update_stream_flag;
         let stream_in_storage_only_for_query_node = !self.streams.contains(stream_name)     // check if stream in storage only if not in memory
-            && self.options.mode == Mode::Query                                                   // and running in query mode
+            && (self.options.mode == Mode::Query  || self.options.mode == Mode::Prism)                                                // and running in query mode
             && self
                 .create_stream_and_schema_from_storage(stream_name)
                 .await?;
diff --git a/src/query/mod.rs b/src/query/mod.rs
index 6486b4836..0608a9459 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -467,7 +467,7 @@ pub async fn get_manifest_list(
     let mut merged_snapshot: Snapshot = Snapshot::default();
 
     // get a list of manifests
-    if PARSEABLE.options.mode == Mode::Query {
+    if PARSEABLE.options.mode == Mode::Query || PARSEABLE.options.mode == Mode::Prism {
         let path = RelativePathBuf::from_iter([stream_name, STREAM_ROOT_DIRECTORY]);
         let obs = glob_storage
             .get_objects(
diff --git a/src/query/stream_schema_provider.rs b/src/query/stream_schema_provider.rs
index 6b6219f91..d015bc71d 100644
--- a/src/query/stream_schema_provider.rs
+++ b/src/query/stream_schema_provider.rs
@@ -503,7 +503,7 @@ impl TableProvider for StandardTableProvider {
             .await?;
         };
         let mut merged_snapshot = Snapshot::default();
-        if PARSEABLE.options.mode == Mode::Query {
+        if PARSEABLE.options.mode == Mode::Query || PARSEABLE.options.mode == Mode::Prism {
             let path = RelativePathBuf::from_iter([&self.stream, STREAM_ROOT_DIRECTORY]);
             let obs = glob_storage
                 .get_objects(
diff --git a/src/utils/arrow/flight.rs b/src/utils/arrow/flight.rs
index c8d2dacf2..180d352d5 100644
--- a/src/utils/arrow/flight.rs
+++ b/src/utils/arrow/flight.rs
@@ -133,7 +133,8 @@ pub fn send_to_ingester(start: i64, end: i64) -> bool {
     );
     let time_filters =
         extract_primary_filter(&[Expr::BinaryExpr(ex1), Expr::BinaryExpr(ex2)], &None);
-    PARSEABLE.options.mode == Mode::Query && is_within_staging_window(&time_filters)
+    (PARSEABLE.options.mode == Mode::Query || PARSEABLE.options.mode == Mode::Prism)
+        && is_within_staging_window(&time_filters)
 }
 
 fn lit_timestamp_milli(time: i64) -> Expr {

From 8c164ff661061dbcc27942bc78deed68524889cc Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Wed, 9 Apr 2025 03:30:20 -0400
Subject: [PATCH 19/42] update logic for check or load stream

---
 src/parseable/mod.rs       | 15 ++++++++-------
 src/prism/logstream/mod.rs | 15 +++------------
 2 files changed, 11 insertions(+), 19 deletions(-)

diff --git a/src/parseable/mod.rs b/src/parseable/mod.rs
index d52613dfe..feafd4de7 100644
--- a/src/parseable/mod.rs
+++ b/src/parseable/mod.rs
@@ -212,13 +212,14 @@ impl Parseable {
 
     /// Checks for the stream in memory, or loads it from storage when in distributed mode
     pub async fn check_or_load_stream(&self, stream_name: &str) -> bool {
-        !self.streams.contains(stream_name)
-            && (self.options.mode != Mode::Query
-                || self.options.mode != Mode::Prism
-                || !self
-                    .create_stream_and_schema_from_storage(stream_name)
-                    .await
-                    .unwrap_or_default())
+        if self.streams.contains(stream_name) {
+            return true;
+        }
+        (self.options.mode == Mode::Query || self.options.mode == Mode::Prism)
+            && self
+                .create_stream_and_schema_from_storage(stream_name)
+                .await
+                .unwrap_or_default()
     }
 
     // validate the storage, if the proper path for staging directory is provided
diff --git a/src/prism/logstream/mod.rs b/src/prism/logstream/mod.rs
index b03de0454..c8e1015c4 100644
--- a/src/prism/logstream/mod.rs
+++ b/src/prism/logstream/mod.rs
@@ -86,7 +86,7 @@ pub async fn get_prism_logstream_info(
 
 async fn get_stream_schema_helper(stream_name: &str) -> Result<Arc<Schema>, StreamError> {
     // Ensure parseable is aware of stream in distributed mode
-    if PARSEABLE.check_or_load_stream(stream_name).await {
+    if !PARSEABLE.check_or_load_stream(stream_name).await {
         return Err(StreamNotFound(stream_name.to_owned()).into());
     }
 
@@ -152,7 +152,7 @@ async fn get_stream_info_helper(stream_name: &str) -> Result<StreamInfo, StreamE
     // For query mode, if the stream not found in memory map,
     //check if it exists in the storage
     //create stream and schema from storage
-    if PARSEABLE.check_or_load_stream(stream_name).await {
+    if !PARSEABLE.check_or_load_stream(stream_name).await {
         return Err(StreamNotFound(stream_name.to_owned()).into());
     }
 
@@ -289,7 +289,7 @@ impl PrismDatasetRequest {
         }
 
         // Skip streams that don't exist
-        if !self.stream_exists(&stream).await {
+        if !PARSEABLE.check_or_load_stream(&stream).await {
             return Ok(None);
         }
 
@@ -311,15 +311,6 @@ impl PrismDatasetRequest {
         }
     }
 
-    async fn stream_exists(&self, stream: &str) -> bool {
-        if PARSEABLE.check_or_load_stream(stream).await {
-            warn!("Stream not found: {stream}");
-            false
-        } else {
-            true
-        }
-    }
-
     async fn build_dataset_response(
         &self,
         stream: String,

From 143e3f2837aa6225353e9f930f11037866a57855 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Wed, 9 Apr 2025 05:18:48 -0400
Subject: [PATCH 20/42] update logic for load stream

---
 src/handlers/http/logstream.rs | 18 +++++++++---------
 src/logstream/mod.rs           |  4 ++--
 src/storage/store_metadata.rs  |  6 +++++-
 3 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/src/handlers/http/logstream.rs b/src/handlers/http/logstream.rs
index 6d095d7b0..d4949b61b 100644
--- a/src/handlers/http/logstream.rs
+++ b/src/handlers/http/logstream.rs
@@ -47,7 +47,7 @@ use tracing::warn;
 pub async fn delete(stream_name: Path<String>) -> Result<impl Responder, StreamError> {
     let stream_name = stream_name.into_inner();
     // Error out if stream doesn't exist in memory, or in the case of query node, in storage as well
-    if PARSEABLE.check_or_load_stream(&stream_name).await {
+    if !PARSEABLE.check_or_load_stream(&stream_name).await {
         return Err(StreamNotFound(stream_name).into());
     }
 
@@ -124,7 +124,7 @@ pub async fn get_schema(stream_name: Path<String>) -> Result<impl Responder, Str
     let stream_name = stream_name.into_inner();
 
     // Ensure parseable is aware of stream in distributed mode
-    if PARSEABLE.check_or_load_stream(&stream_name).await {
+    if !PARSEABLE.check_or_load_stream(&stream_name).await {
         return Err(StreamNotFound(stream_name.clone()).into());
     }
 
@@ -160,7 +160,7 @@ pub async fn get_retention(stream_name: Path<String>) -> Result<impl Responder,
     // For query mode, if the stream not found in memory map,
     //check if it exists in the storage
     //create stream and schema from storage
-    if PARSEABLE.check_or_load_stream(&stream_name).await {
+    if !PARSEABLE.check_or_load_stream(&stream_name).await {
         return Err(StreamNotFound(stream_name.clone()).into());
     }
 
@@ -180,7 +180,7 @@ pub async fn put_retention(
     // For query mode, if the stream not found in memory map,
     //check if it exists in the storage
     //create stream and schema from storage
-    if PARSEABLE.check_or_load_stream(&stream_name).await {
+    if !PARSEABLE.check_or_load_stream(&stream_name).await {
         return Err(StreamNotFound(stream_name).into());
     }
 
@@ -231,7 +231,7 @@ pub async fn get_stats(
     // For query mode, if the stream not found in memory map,
     //check if it exists in the storage
     //create stream and schema from storage
-    if PARSEABLE.check_or_load_stream(&stream_name).await {
+    if !PARSEABLE.check_or_load_stream(&stream_name).await {
         return Err(StreamNotFound(stream_name.clone()).into());
     }
 
@@ -289,7 +289,7 @@ pub async fn get_stream_info(stream_name: Path<String>) -> Result<impl Responder
     // For query mode, if the stream not found in memory map,
     //check if it exists in the storage
     //create stream and schema from storage
-    if PARSEABLE.check_or_load_stream(&stream_name).await {
+    if !PARSEABLE.check_or_load_stream(&stream_name).await {
         return Err(StreamNotFound(stream_name.clone()).into());
     }
 
@@ -349,7 +349,7 @@ pub async fn put_stream_hot_tier(
     // For query mode, if the stream not found in memory map,
     //check if it exists in the storage
     //create stream and schema from storage
-    if PARSEABLE.check_or_load_stream(&stream_name).await {
+    if !PARSEABLE.check_or_load_stream(&stream_name).await {
         return Err(StreamNotFound(stream_name).into());
     }
 
@@ -396,7 +396,7 @@ pub async fn get_stream_hot_tier(stream_name: Path<String>) -> Result<impl Respo
     // For query mode, if the stream not found in memory map,
     //check if it exists in the storage
     //create stream and schema from storage
-    if PARSEABLE.check_or_load_stream(&stream_name).await {
+    if !PARSEABLE.check_or_load_stream(&stream_name).await {
         return Err(StreamNotFound(stream_name.clone()).into());
     }
 
@@ -416,7 +416,7 @@ pub async fn delete_stream_hot_tier(
     // For query mode, if the stream not found in memory map,
     //check if it exists in the storage
     //create stream and schema from storage
-    if PARSEABLE.check_or_load_stream(&stream_name).await {
+    if !PARSEABLE.check_or_load_stream(&stream_name).await {
         return Err(StreamNotFound(stream_name).into());
     }
 
diff --git a/src/logstream/mod.rs b/src/logstream/mod.rs
index 9dc29e06b..844885fc6 100644
--- a/src/logstream/mod.rs
+++ b/src/logstream/mod.rs
@@ -27,7 +27,7 @@ use crate::{handlers::http::{logstream::error::StreamError, query::update_schema
 
 pub async fn get_stream_schema_helper(stream_name: &str) -> Result<Arc<Schema>, StreamError> {
     // Ensure parseable is aware of stream in distributed mode
-    if PARSEABLE.check_or_load_stream(&stream_name).await {
+    if !PARSEABLE.check_or_load_stream(&stream_name).await {
         return Err(StreamNotFound(stream_name.to_owned()).into());
     }
 
@@ -48,7 +48,7 @@ pub async fn get_stream_info_helper(stream_name: &str) -> Result<StreamInfo, Str
     // For query mode, if the stream not found in memory map,
     //check if it exists in the storage
     //create stream and schema from storage
-    if PARSEABLE.check_or_load_stream(&stream_name).await {
+    if !PARSEABLE.check_or_load_stream(&stream_name).await {
         return Err(StreamNotFound(stream_name.to_owned()).into());
     }
 
diff --git a/src/storage/store_metadata.rs b/src/storage/store_metadata.rs
index edcdb7e27..55fad1c06 100644
--- a/src/storage/store_metadata.rs
+++ b/src/storage/store_metadata.rs
@@ -129,9 +129,13 @@ pub async fn resolve_parseable_metadata(
             Err("Could not start the server because staging directory indicates stale data from previous deployment, please choose an empty staging directory and restart the server")
         }
         EnvChange::NewStaging(mut metadata) => {
+
+            if metadata.server_mode==Mode::Prism {
+                Err("Starting Parseable OSS is not permitted when Parseable Enterprise is enabled. Please restart the server with different storage")
+            }
             // if server is started in ingest mode,we need to make sure that query mode has been started
             // i.e the metadata is updated to reflect the server mode = Query
-            if metadata.server_mode== Mode::All && PARSEABLE.options.mode == Mode::Ingest {
+            else if metadata.server_mode== Mode::All && PARSEABLE.options.mode == Mode::Ingest {
                 Err("Starting Ingest Mode is not allowed, Since Query Server has not been started yet")
             } else {
                 create_dir_all(PARSEABLE.options.staging_dir())?;

From 30165c31cd378517eaf317616470df7b21ccb0fe Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Thu, 10 Apr 2025 04:58:49 -0400
Subject: [PATCH 21/42] revert restriction for enterprise to oss

---
 src/storage/store_metadata.rs | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/storage/store_metadata.rs b/src/storage/store_metadata.rs
index 55fad1c06..2152f3d07 100644
--- a/src/storage/store_metadata.rs
+++ b/src/storage/store_metadata.rs
@@ -130,12 +130,9 @@ pub async fn resolve_parseable_metadata(
         }
         EnvChange::NewStaging(mut metadata) => {
 
-            if metadata.server_mode==Mode::Prism {
-                Err("Starting Parseable OSS is not permitted when Parseable Enterprise is enabled. Please restart the server with different storage")
-            }
             // if server is started in ingest mode,we need to make sure that query mode has been started
             // i.e the metadata is updated to reflect the server mode = Query
-            else if metadata.server_mode== Mode::All && PARSEABLE.options.mode == Mode::Ingest {
+            if metadata.server_mode== Mode::All && PARSEABLE.options.mode == Mode::Ingest {
                 Err("Starting Ingest Mode is not allowed, Since Query Server has not been started yet")
             } else {
                 create_dir_all(PARSEABLE.options.staging_dir())?;

From c6159ff0c0a55a67fe8d8e6e129a82444d4410bb Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Thu, 10 Apr 2025 06:27:02 -0400
Subject: [PATCH 22/42] add comment for check or load stream

---
 src/parseable/mod.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/parseable/mod.rs b/src/parseable/mod.rs
index feafd4de7..bcab999cd 100644
--- a/src/parseable/mod.rs
+++ b/src/parseable/mod.rs
@@ -211,6 +211,8 @@ impl Parseable {
     }
 
     /// Checks for the stream in memory, or loads it from storage when in distributed mode
+    /// return true if stream exists in memory or loaded from storage
+    /// return false if stream doesn't exist in memory and not loaded from storage
     pub async fn check_or_load_stream(&self, stream_name: &str) -> bool {
         if self.streams.contains(stream_name) {
             return true;

From 7a3a66290b9d70f97d6d66aa77edbe063d7dd625 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Sat, 12 Apr 2025 05:41:15 -0400
Subject: [PATCH 23/42] self in cluster info

---
 src/handlers/http/cluster/mod.rs | 16 +++++-
 src/handlers/http/modal/mod.rs   | 93 +++-----------------------------
 src/option.rs                    | 12 +++++
 3 files changed, 34 insertions(+), 87 deletions(-)

diff --git a/src/handlers/http/cluster/mod.rs b/src/handlers/http/cluster/mod.rs
index 456cf5aa2..0a70572ca 100644
--- a/src/handlers/http/cluster/mod.rs
+++ b/src/handlers/http/cluster/mod.rs
@@ -540,7 +540,20 @@ pub async fn send_retention_cleanup_request(
 }
 
 pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
-    // Get ingestor and indexer metadata concurrently
+    let self_info = utils::ClusterInfo::new(
+        &PARSEABLE.options.address,
+        true,
+        PARSEABLE
+            .options
+            .staging_dir()
+            .to_string_lossy()
+            .to_string(),
+        PARSEABLE.storage.get_endpoint(),
+        None,
+        Some(String::from("200 OK")),
+        &PARSEABLE.options.mode.to_node_type(),
+    );
+    // Get querier, ingestor and indexer metadata concurrently
     let (querier_result, ingestor_result, indexer_result) = future::join3(
         get_node_info("querier"),
         get_node_info("ingestor"),
@@ -585,6 +598,7 @@ pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
     infos.extend(querier_infos?);
     infos.extend(ingestor_infos?);
     infos.extend(indexer_infos?);
+    infos.push(self_info);
 
     Ok(actix_web::HttpResponse::Ok().json(infos))
 }
diff --git a/src/handlers/http/modal/mod.rs b/src/handlers/http/modal/mod.rs
index 2bafcb792..7b0d81d90 100644
--- a/src/handlers/http/modal/mod.rs
+++ b/src/handlers/http/modal/mod.rs
@@ -200,14 +200,14 @@ pub async fn load_on_init() -> anyhow::Result<()> {
 }
 
 #[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Default)]
+#[serde(rename_all = "lowercase")]
 pub enum NodeType {
     #[default]
-    #[serde(rename = "ingestor")]
     Ingestor,
-    #[serde(rename = "indexer")]
     Indexer,
-    #[serde(rename = "querier")]
     Querier,
+    Prism,
+    All,
 }
 
 impl NodeType {
@@ -216,6 +216,8 @@ impl NodeType {
             NodeType::Ingestor => "ingestor",
             NodeType::Indexer => "indexer",
             NodeType::Querier => "querier",
+            NodeType::Prism => "prism",
+            NodeType::All => "all",
         }
     }
 
@@ -224,6 +226,8 @@ impl NodeType {
             NodeType::Ingestor => Mode::Ingest,
             NodeType::Indexer => Mode::Index,
             NodeType::Querier => Mode::Query,
+            NodeType::Prism => Mode::Prism,
+            NodeType::All => Mode::All,
         }
     }
 }
@@ -532,89 +536,6 @@ pub type IngestorMetadata = NodeMetadata;
 pub type IndexerMetadata = NodeMetadata;
 pub type QuerierMetadata = NodeMetadata;
 
-// Helper functions for creating specific node types
-pub fn create_ingestor_metadata(
-    port: String,
-    domain_name: String,
-    bucket_name: String,
-    username: &str,
-    password: &str,
-    ingestor_id: String,
-    flight_port: String,
-) -> NodeMetadata {
-    NodeMetadata::new(
-        port,
-        domain_name,
-        bucket_name,
-        username,
-        password,
-        ingestor_id,
-        flight_port,
-        NodeType::Ingestor,
-    )
-}
-
-pub fn load_ingestor_metadata(
-    options: &Options,
-    storage: &dyn ObjectStorageProvider,
-) -> Arc<NodeMetadata> {
-    NodeMetadata::load(options, storage, NodeType::Ingestor)
-}
-
-pub fn create_indexer_metadata(
-    port: String,
-    domain_name: String,
-    bucket_name: String,
-    username: &str,
-    password: &str,
-    indexer_id: String,
-    flight_port: String,
-) -> NodeMetadata {
-    NodeMetadata::new(
-        port,
-        domain_name,
-        bucket_name,
-        username,
-        password,
-        indexer_id,
-        flight_port,
-        NodeType::Indexer,
-    )
-}
-
-pub fn load_indexer_metadata(
-    options: &Options,
-    storage: &dyn ObjectStorageProvider,
-) -> Arc<NodeMetadata> {
-    NodeMetadata::load(options, storage, NodeType::Indexer)
-}
-
-pub fn create_querier_metadata(
-    port: String,
-    domain_name: String,
-    bucket_name: String,
-    username: &str,
-    password: &str,
-    querier_id: String,
-    flight_port: String,
-) -> NodeMetadata {
-    NodeMetadata::new(
-        port,
-        domain_name,
-        bucket_name,
-        username,
-        password,
-        querier_id,
-        flight_port,
-        NodeType::Querier,
-    )
-}
-pub fn load_querier_metadata(
-    options: &Options,
-    storage: &dyn ObjectStorageProvider,
-) -> Arc<NodeMetadata> {
-    NodeMetadata::load(options, storage, NodeType::Querier)
-}
 #[cfg(test)]
 mod test {
     use actix_web::body::MessageBody;
diff --git a/src/option.rs b/src/option.rs
index aca27bb49..26b5f4664 100644
--- a/src/option.rs
+++ b/src/option.rs
@@ -18,6 +18,8 @@
 use parquet::basic::{BrotliLevel, GzipLevel, ZstdLevel};
 use serde::{Deserialize, Serialize};
 
+use crate::handlers::http::modal::NodeType;
+
 #[derive(Debug, Default, Eq, PartialEq, Clone, Copy, Serialize, Deserialize)]
 pub enum Mode {
     Query,
@@ -41,6 +43,16 @@ impl Mode {
 
         Ok(())
     }
+
+    pub fn to_node_type(&self) -> NodeType {
+        match self {
+            Mode::Ingest => NodeType::Ingestor,
+            Mode::Index => NodeType::Indexer,
+            Mode::Query => NodeType::Querier,
+            Mode::Prism => NodeType::Prism,
+            Mode::All => NodeType::All,
+        }
+    }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]

From 22f763e553b2c19400e2c4de17dfb5f1e2229cbc Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Sat, 12 Apr 2025 06:20:50 -0400
Subject: [PATCH 24/42] add scheme to cluster info

---
 src/handlers/http/cluster/mod.rs | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/handlers/http/cluster/mod.rs b/src/handlers/http/cluster/mod.rs
index 0a70572ca..6c6528902 100644
--- a/src/handlers/http/cluster/mod.rs
+++ b/src/handlers/http/cluster/mod.rs
@@ -37,7 +37,9 @@ use serde_json::error::Error as SerdeError;
 use serde_json::{to_vec, Value as JsonValue};
 use tracing::{error, info, warn};
 use url::Url;
-use utils::{check_liveness, to_url_string, IngestionStats, QueriedStats, StorageStats};
+use utils::{
+    check_liveness, to_url_string, ClusterInfo, IngestionStats, QueriedStats, StorageStats,
+};
 
 use crate::handlers::http::ingest::ingest_internal_stream;
 use crate::metrics::prom_utils::Metrics;
@@ -540,8 +542,13 @@ pub async fn send_retention_cleanup_request(
 }
 
 pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
-    let self_info = utils::ClusterInfo::new(
-        &PARSEABLE.options.address,
+    let self_info = ClusterInfo::new(
+        format!(
+            "{}://{}",
+            PARSEABLE.options.get_scheme(),
+            PARSEABLE.options.address
+        )
+        .as_str(),
         true,
         PARSEABLE
             .options

From 142987767bb10b5b138f4e18183905107d7f9b20 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 14 Apr 2025 01:58:23 -0400
Subject: [PATCH 25/42] self metadata for cluster info and metrics

---
 src/handlers/http/cluster/mod.rs | 49 ++++++++++++++++----------------
 src/handlers/http/modal/mod.rs   |  1 +
 src/parseable/mod.rs             | 35 ++++++++++++++++++++++-
 3 files changed, 59 insertions(+), 26 deletions(-)

diff --git a/src/handlers/http/cluster/mod.rs b/src/handlers/http/cluster/mod.rs
index 6c6528902..ceaeac80f 100644
--- a/src/handlers/http/cluster/mod.rs
+++ b/src/handlers/http/cluster/mod.rs
@@ -37,9 +37,7 @@ use serde_json::error::Error as SerdeError;
 use serde_json::{to_vec, Value as JsonValue};
 use tracing::{error, info, warn};
 use url::Url;
-use utils::{
-    check_liveness, to_url_string, ClusterInfo, IngestionStats, QueriedStats, StorageStats,
-};
+use utils::{check_liveness, to_url_string, IngestionStats, QueriedStats, StorageStats};
 
 use crate::handlers::http::ingest::ingest_internal_stream;
 use crate::metrics::prom_utils::Metrics;
@@ -542,24 +540,6 @@ pub async fn send_retention_cleanup_request(
 }
 
 pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
-    let self_info = ClusterInfo::new(
-        format!(
-            "{}://{}",
-            PARSEABLE.options.get_scheme(),
-            PARSEABLE.options.address
-        )
-        .as_str(),
-        true,
-        PARSEABLE
-            .options
-            .staging_dir()
-            .to_string_lossy()
-            .to_string(),
-        PARSEABLE.storage.get_endpoint(),
-        None,
-        Some(String::from("200 OK")),
-        &PARSEABLE.options.mode.to_node_type(),
-    );
     // Get querier, ingestor and indexer metadata concurrently
     let (querier_result, ingestor_result, indexer_result) = future::join3(
         get_node_info("querier"),
@@ -592,21 +572,27 @@ pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
         })
         .map_err(|err| StreamError::Anyhow(err.into()))?;
 
+    let self_metadata = if let Some(metadata) = PARSEABLE.get_metadata() {
+        vec![metadata]
+    } else {
+        vec![]
+    };
+
     // Fetch info for both node types concurrently
-    let (querier_infos, ingestor_infos, indexer_infos) = future::join3(
+    let (querier_infos, ingestor_infos, indexer_infos, self_info) = future::join4(
         fetch_nodes_info(querier_metadata),
         fetch_nodes_info(ingestor_metadata),
         fetch_nodes_info(indexer_metadata),
+        fetch_nodes_info(self_metadata),
     )
     .await;
 
     // Combine results from both node types
     let mut infos = Vec::new();
+    infos.extend(self_info?);
     infos.extend(querier_infos?);
     infos.extend(ingestor_infos?);
     infos.extend(indexer_infos?);
-    infos.push(self_info);
-
     Ok(actix_web::HttpResponse::Ok().json(infos))
 }
 
@@ -885,6 +871,12 @@ async fn fetch_cluster_metrics() -> Result<Vec<Metrics>, PostError> {
     )
     .await;
 
+    let self_metadata = if let Some(metadata) = PARSEABLE.get_metadata() {
+        vec![metadata]
+    } else {
+        vec![]
+    };
+
     // Handle querier metadata result
     let querier_metadata: Vec<NodeMetadata> = querier_result.map_err(|err| {
         error!("Fatal: failed to get querier info: {:?}", err);
@@ -901,7 +893,8 @@ async fn fetch_cluster_metrics() -> Result<Vec<Metrics>, PostError> {
         PostError::Invalid(err)
     })?;
     // Fetch metrics from ingestors and indexers concurrently
-    let (querier_metrics, ingestor_metrics, indexer_metrics) = future::join3(
+    let (self_metrics, querier_metrics, ingestor_metrics, indexer_metrics) = future::join4(
+        fetch_nodes_metrics(self_metadata),
         fetch_nodes_metrics(querier_metadata),
         fetch_nodes_metrics(ingestor_metadata),
         fetch_nodes_metrics(indexer_metadata),
@@ -911,6 +904,12 @@ async fn fetch_cluster_metrics() -> Result<Vec<Metrics>, PostError> {
     // Combine all metrics
     let mut all_metrics = Vec::new();
 
+    // Add self metrics
+    match self_metrics {
+        Ok(metrics) => all_metrics.extend(metrics),
+        Err(err) => return Err(err),
+    }
+
     // Add querier metrics
     match querier_metrics {
         Ok(metrics) => all_metrics.extend(metrics),
diff --git a/src/handlers/http/modal/mod.rs b/src/handlers/http/modal/mod.rs
index 7b0d81d90..abbd8265e 100644
--- a/src/handlers/http/modal/mod.rs
+++ b/src/handlers/http/modal/mod.rs
@@ -535,6 +535,7 @@ impl Metadata for NodeMetadata {
 pub type IngestorMetadata = NodeMetadata;
 pub type IndexerMetadata = NodeMetadata;
 pub type QuerierMetadata = NodeMetadata;
+pub type PrismMetadata = NodeMetadata;
 
 #[cfg(test)]
 mod test {
diff --git a/src/parseable/mod.rs b/src/parseable/mod.rs
index bcab999cd..d162f343a 100644
--- a/src/parseable/mod.rs
+++ b/src/parseable/mod.rs
@@ -49,7 +49,7 @@ use crate::{
             logstream::error::{CreateStreamError, StreamError},
             modal::{
                 utils::logstream_utils::PutStreamHeaders, IndexerMetadata, IngestorMetadata,
-                NodeType, QuerierMetadata,
+                Metadata, NodeMetadata, NodeType, PrismMetadata, QuerierMetadata,
             },
         },
         STREAM_TYPE_KEY,
@@ -130,6 +130,8 @@ pub struct Parseable {
     /// Metadata and staging realting to each logstreams
     /// A globally shared mapping of `Streams` that parseable is aware of.
     pub streams: Streams,
+    ///Metadata associated only with a prism
+    pub prism_metadata: Option<Arc<PrismMetadata>>,
     /// Metadata associated only with a querier
     pub querier_metadata: Option<Arc<QuerierMetadata>>,
     /// Metadata associated only with an ingestor
@@ -147,6 +149,14 @@ impl Parseable {
         #[cfg(feature = "kafka")] kafka_config: KafkaConfig,
         storage: Arc<dyn ObjectStorageProvider>,
     ) -> Self {
+        let prism_metadata = match &options.mode {
+            Mode::Prism => Some(PrismMetadata::load(
+                &options,
+                storage.as_ref(),
+                NodeType::Prism,
+            )),
+            _ => None,
+        };
         let ingestor_metadata = match &options.mode {
             Mode::Ingest => Some(IngestorMetadata::load(
                 &options,
@@ -175,6 +185,7 @@ impl Parseable {
             options: Arc::new(options),
             storage,
             streams: Streams::default(),
+            prism_metadata,
             ingestor_metadata,
             indexer_metadata,
             querier_metadata,
@@ -183,6 +194,28 @@ impl Parseable {
         }
     }
 
+    /// Get the metadata for the current node based on its mode.
+    pub fn get_metadata(&self) -> Option<NodeMetadata> {
+        let meta_ref = match self.options.mode {
+            Mode::Ingest => self.ingestor_metadata.as_ref(),
+            Mode::Index => self.indexer_metadata.as_ref(),
+            Mode::Query => self.querier_metadata.as_ref(),
+            _ => return None,
+        };
+
+        let meta = meta_ref?;
+        let node_metadata = NodeMetadata {
+            version: meta.version.clone(),
+            node_id: meta.node_id.clone(),
+            port: meta.port.clone(),
+            domain_name: meta.domain_name.clone(),
+            bucket_name: meta.bucket_name.clone(),
+            token: meta.token.clone(),
+            flight_port: meta.flight_port.clone(),
+            node_type: meta.node_type().clone(),
+        };
+        Some(node_metadata)
+    }
     /// Try to get the handle of a stream in staging, if it doesn't exist return `None`.
     pub fn get_stream(&self, stream_name: &str) -> Result<StreamRef, StreamNotFound> {
         self.streams

From 2232af82661c646daf82c0d50767f54b3ca1007f Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 14 Apr 2025 02:13:01 -0400
Subject: [PATCH 26/42] update url fetch

---
 src/cli.rs | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/cli.rs b/src/cli.rs
index b302f3f4b..536e93e2d 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -492,7 +492,17 @@ impl Options {
                 }
                 (&self.querier_endpoint, "P_QUERIER_ENDPOINT")
             }
-            _ => panic!("Invalid mode"),
+            _ => {
+                return format!(
+                    "{}://{}",
+                    self.get_scheme(),
+                    self.address
+                )
+                .parse::<Url>() // if the value was improperly set, this will panic before hand
+                .unwrap_or_else(|err| {
+                    panic!("{err}, failed to parse `{}` as Url. Please set the environment variable `P_ADDR` to `<ip address>:<port>` without the scheme (e.g., 192.168.1.1:8000). Please refer to the documentation: https://logg.ing/env for more details.", self.address)
+                });
+            }
         };
 
         if endpoint.starts_with("http") {

From 1e4728f2c0c920458fb9f743961d536db9617411 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 14 Apr 2025 02:22:29 -0400
Subject: [PATCH 27/42] add prism metadata for node info

---
 src/parseable/mod.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/parseable/mod.rs b/src/parseable/mod.rs
index d162f343a..dea5c8a99 100644
--- a/src/parseable/mod.rs
+++ b/src/parseable/mod.rs
@@ -200,6 +200,7 @@ impl Parseable {
             Mode::Ingest => self.ingestor_metadata.as_ref(),
             Mode::Index => self.indexer_metadata.as_ref(),
             Mode::Query => self.querier_metadata.as_ref(),
+            Mode::Prism => self.prism_metadata.as_ref(),
             _ => return None,
         };
 

From 48b121c3f35ce7bdef99fd0e3d7a10792826d4a2 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 14 Apr 2025 03:01:01 -0400
Subject: [PATCH 28/42] refactor url parsing

---
 src/cli.rs | 140 +++++++++++++++++++++--------------------------------
 1 file changed, 55 insertions(+), 85 deletions(-)

diff --git a/src/cli.rs b/src/cli.rs
index 536e93e2d..5446c7700 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -447,107 +447,77 @@ impl Options {
         }
     }
 
-    /// TODO: refactor and document
     pub fn get_url(&self, mode: Mode) -> Url {
-        let (endpoint, env_var) = match mode {
-            Mode::Ingest => {
-                if self.ingestor_endpoint.is_empty() {
-                    return format!(
-                        "{}://{}",
-                        self.get_scheme(),
-                        self.address
-                    )
-                    .parse::<Url>() // if the value was improperly set, this will panic before hand
-                    .unwrap_or_else(|err| {
-                        panic!("{err}, failed to parse `{}` as Url. Please set the environment variable `P_ADDR` to `<ip address>:<port>` without the scheme (e.g., 192.168.1.1:8000). Please refer to the documentation: https://logg.ing/env for more details.", self.address)
-                    });
-                }
-                (&self.ingestor_endpoint, "P_INGESTOR_ENDPOINT")
-            }
-            Mode::Index => {
-                if self.indexer_endpoint.is_empty() {
-                    return format!(
-                        "{}://{}",
-                        self.get_scheme(),
-                        self.address
-                    )
-                    .parse::<Url>() // if the value was improperly set, this will panic before hand
-                    .unwrap_or_else(|err| {
-                        panic!("{err}, failed to parse `{}` as Url. Please set the environment variable `P_ADDR` to `<ip address>:<port>` without the scheme (e.g., 192.168.1.1:8000). Please refer to the documentation: https://logg.ing/env for more details.", self.address)
-                    });
-                }
-                (&self.indexer_endpoint, "P_INDEXER_ENDPOINT")
-            }
-            Mode::Query => {
-                if self.querier_endpoint.is_empty() {
-                    return format!(
-                        "{}://{}",
-                        self.get_scheme(),
-                        self.address
-                    )
-                    .parse::<Url>() // if the value was improperly set, this will panic before hand
-                    .unwrap_or_else(|err| {
-                        panic!("{err}, failed to parse `{}` as Url. Please set the environment variable `P_ADDR` to `<ip address>:<port>` without the scheme (e.g., 192.168.1.1:8000). Please refer to the documentation: https://logg.ing/env for more details.", self.address)
-                    });
-                }
-                (&self.querier_endpoint, "P_QUERIER_ENDPOINT")
-            }
-            _ => {
-                return format!(
-                    "{}://{}",
-                    self.get_scheme(),
-                    self.address
-                )
-                .parse::<Url>() // if the value was improperly set, this will panic before hand
-                .unwrap_or_else(|err| {
-                    panic!("{err}, failed to parse `{}` as Url. Please set the environment variable `P_ADDR` to `<ip address>:<port>` without the scheme (e.g., 192.168.1.1:8000). Please refer to the documentation: https://logg.ing/env for more details.", self.address)
-                });
-            }
+        let endpoint = match mode {
+            Mode::Ingest => self.get_endpoint(&self.ingestor_endpoint, "P_INGESTOR_ENDPOINT"),
+            Mode::Index => self.get_endpoint(&self.indexer_endpoint, "P_INDEXER_ENDPOINT"),
+            Mode::Query => self.get_endpoint(&self.querier_endpoint, "P_QUERIER_ENDPOINT"),
+            _ => return self.build_url(&self.address),
         };
 
-        if endpoint.starts_with("http") {
-            panic!("Invalid value `{}`, please set the environement variable `{env_var}` to `<ip address / DNS>:<port>` without the scheme (e.g., 192.168.1.1:8000 or example.com:8000). Please refer to the documentation: https://logg.ing/env for more details.", endpoint);
+        self.parse_endpoint(&endpoint)
+    }
+
+    fn get_endpoint(&self, endpoint: &str, env_var: &str) -> String {
+        if endpoint.is_empty() {
+            self.address.to_string()
+        } else {
+            if endpoint.starts_with("http") {
+                panic!(
+                    "Invalid value `{}`, please set the environment variable `{}` to `<ip address / DNS>:<port>` without the scheme (e.g., 192.168.1.1:8000 or example.com:8000). Please refer to the documentation: https://logg.ing/env for more details.",
+                    endpoint, env_var
+                );
+            }
+            endpoint.to_string()
         }
+    }
 
-        let addr_from_env = endpoint.split(':').collect::<Vec<&str>>();
+    fn parse_endpoint(&self, endpoint: &str) -> Url {
+        let addr_parts: Vec<&str> = endpoint.split(':').collect();
 
-        if addr_from_env.len() != 2 {
-            panic!("Invalid value `{}`, please set the environement variable `{env_var}` to `<ip address / DNS>:<port>` without the scheme (e.g., 192.168.1.1:8000 or example.com:8000). Please refer to the documentation: https://logg.ing/env for more details.", endpoint);
+        if addr_parts.len() != 2 {
+            panic!(
+                "Invalid value `{}`, please set the environment variable to `<ip address / DNS>:<port>` without the scheme (e.g., 192.168.1.1:8000 or example.com:8000). Please refer to the documentation: https://logg.ing/env for more details.",
+                endpoint
+            );
         }
 
-        let mut hostname = addr_from_env[0].to_string();
-        let mut port = addr_from_env[1].to_string();
-
-        // if the env var value fits the pattern $VAR_NAME:$VAR_NAME
-        // fetch the value from the specified env vars
-        if hostname.starts_with('$') {
-            let var_hostname = hostname[1..].to_string();
-            hostname = env::var(&var_hostname).unwrap_or_default();
+        let hostname = self.resolve_env_var(addr_parts[0]);
+        let port = self.resolve_env_var(addr_parts[1]);
 
-            if hostname.is_empty() {
-                panic!("The environement variable `{}` is not set, please set as <ip address / DNS> without the scheme (e.g., 192.168.1.1 or example.com). Please refer to the documentation: https://logg.ing/env for more details.", var_hostname);
-            }
-            if hostname.starts_with("http") {
-                panic!("Invalid value `{}`, please set the environement variable `{}` to `<ip address / DNS>` without the scheme (e.g., 192.168.1.1 or example.com). Please refer to the documentation: https://logg.ing/env for more details.", hostname, var_hostname);
-            } else {
-                hostname = format!("{}://{}", self.get_scheme(), hostname);
-            }
-        }
+        self.build_url(&format!("{}:{}", hostname, port))
+    }
 
-        if port.starts_with('$') {
-            let var_port = port[1..].to_string();
-            port = env::var(&var_port).unwrap_or_default();
+    fn resolve_env_var(&self, value: &str) -> String {
+        if let Some(env_var) = value.strip_prefix('$') {
+            let resolved_value = env::var(env_var).unwrap_or_else(|_| {
+                panic!(
+                    "The environment variable `{}` is not set. Please set it to a valid value. Refer to the documentation: https://logg.ing/env for more details.",
+                    env_var
+                );
+            });
 
-            if port.is_empty() {
+            if resolved_value.starts_with("http") {
                 panic!(
-                    "Port is not set in the environement variable `{}`. Please refer to the documentation: https://logg.ing/env for more details.",
-                    var_port
+                    "Invalid value `{}`, please set the environment variable `{}` to `<ip address / DNS>` without the scheme (e.g., 192.168.1.1 or example.com). Please refer to the documentation: https://logg.ing/env for more details.",
+                    resolved_value, env_var
                 );
             }
+
+            resolved_value
+        } else {
+            value.to_string()
         }
+    }
 
-        format!("{}://{}:{}", self.get_scheme(), hostname, port)
+    fn build_url(&self, address: &str) -> Url {
+        format!("{}://{}", self.get_scheme(), address)
             .parse::<Url>()
-            .expect("Valid URL")
+            .unwrap_or_else(|err| {
+                panic!(
+                    "{err}, failed to parse `{}` as Url. Please set the environment variable `P_ADDR` to `<ip address>:<port>` without the scheme (e.g., 192.168.1.1:8000). Please refer to the documentation: https://logg.ing/env for more details.",
+                    address
+                );
+            })
     }
 }

From 6f0b6239c603cde3351e26fb881171163daaff82 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 14 Apr 2025 04:42:51 -0400
Subject: [PATCH 29/42] correct logic to delete node from cluster

---
 src/handlers/http/cluster/mod.rs | 38 ++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 16 deletions(-)

diff --git a/src/handlers/http/cluster/mod.rs b/src/handlers/http/cluster/mod.rs
index ceaeac80f..8a7bad026 100644
--- a/src/handlers/http/cluster/mod.rs
+++ b/src/handlers/http/cluster/mod.rs
@@ -54,7 +54,9 @@ use crate::HTTP_CLIENT;
 use super::base_path_without_preceding_slash;
 use super::ingest::PostError;
 use super::logstream::error::StreamError;
-use super::modal::{IndexerMetadata, IngestorMetadata, Metadata, NodeMetadata, QuerierMetadata};
+use super::modal::{
+    IndexerMetadata, IngestorMetadata, Metadata, NodeMetadata, NodeType, QuerierMetadata,
+};
 use super::rbac::RBACError;
 use super::role::RoleError;
 
@@ -718,37 +720,41 @@ pub async fn remove_node(node_url: Path<String>) -> Result<impl Responder, PostE
 
     // Delete ingestor metadata
     let removed_ingestor =
-        remove_node_metadata::<IngestorMetadata>(&object_store, &domain_name).await?;
+        remove_node_metadata::<IngestorMetadata>(&object_store, &domain_name, NodeType::Ingestor)
+            .await?;
 
     // Delete indexer metadata
     let removed_indexer =
-        remove_node_metadata::<IndexerMetadata>(&object_store, &domain_name).await?;
+        remove_node_metadata::<IndexerMetadata>(&object_store, &domain_name, NodeType::Indexer)
+            .await?;
 
     // Delete querier metadata
     let removed_querier =
-        remove_node_metadata::<QuerierMetadata>(&object_store, &domain_name).await?;
-
-    let msg = if removed_ingestor || removed_indexer || removed_querier {
-        format!("node {} removed successfully", domain_name)
-    } else {
-        format!("node {} is not found", domain_name)
-    };
-
-    info!("{}", &msg);
-    Ok((msg, StatusCode::OK))
+        remove_node_metadata::<QuerierMetadata>(&object_store, &domain_name, NodeType::Querier)
+            .await?;
+
+    if removed_ingestor || removed_indexer || removed_querier {
+        return Ok((
+            format!("node {} removed successfully", domain_name),
+            StatusCode::OK,
+        ));
+    }
+    Err(PostError::Invalid(anyhow::anyhow!(
+        "node {} not found",
+        domain_name
+    )))
 }
 
 // Helper function to remove a specific type of node metadata
 async fn remove_node_metadata<T: Metadata + DeserializeOwned + Default>(
     object_store: &Arc<dyn ObjectStorage>,
     domain_name: &str,
+    node_type: NodeType,
 ) -> Result<bool, PostError> {
-    let node_type = T::default().node_type().to_string();
-
     let metadatas = object_store
         .get_objects(
             Some(&RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY)),
-            Box::new(move |file_name| file_name.starts_with(&node_type)),
+            Box::new(move |file_name| file_name.starts_with(&node_type.to_string())),
         )
         .await?;
 

From 7bd0a3864b110230cbaec5c072a8c575d6faf818 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 14 Apr 2025 05:57:09 -0400
Subject: [PATCH 30/42] add comments, remove hard coded values for node types

---
 src/analytics.rs                              |  15 ++-
 src/catalog/mod.rs                            |  13 ++-
 src/cli.rs                                    |  11 ++
 src/handlers/airplane.rs                      |   4 +-
 src/handlers/http/about.rs                    |   8 +-
 src/handlers/http/cluster/mod.rs              | 104 +++++++++++-------
 src/handlers/http/mod.rs                      |   4 +-
 src/handlers/http/modal/mod.rs                |   3 +-
 .../http/modal/query/querier_logstream.rs     |   7 +-
 src/handlers/http/query.rs                    |   3 +
 src/parseable/mod.rs                          |   7 +-
 11 files changed, 116 insertions(+), 63 deletions(-)

diff --git a/src/analytics.rs b/src/analytics.rs
index 5c71ab371..b9b3e425f 100644
--- a/src/analytics.rs
+++ b/src/analytics.rs
@@ -36,7 +36,7 @@ use crate::{
         http::{
             base_path_without_preceding_slash,
             cluster::{self, utils::check_liveness},
-            modal::NodeMetadata,
+            modal::{NodeMetadata, NodeType},
         },
         STREAM_NAME_HEADER_KEY,
     },
@@ -116,7 +116,9 @@ impl Report {
         let mut active_queries = 0;
         let mut inactive_queries = 0;
 
-        let indexer_infos: Vec<NodeMetadata> = cluster::get_node_info("indexer").await?;
+        // check liveness of indexers
+        // get the count of active and inactive indexers
+        let indexer_infos: Vec<NodeMetadata> = cluster::get_node_info(NodeType::Indexer).await?;
         for indexer in indexer_infos {
             if check_liveness(&indexer.domain_name).await {
                 active_indexers += 1;
@@ -125,7 +127,9 @@ impl Report {
             }
         }
 
-        let query_infos: Vec<NodeMetadata> = cluster::get_node_info("query").await?;
+        // check liveness of queriers
+        // get the count of active and inactive queriers
+        let query_infos: Vec<NodeMetadata> = cluster::get_node_info(NodeType::Querier).await?;
         for query in query_infos {
             if check_liveness(&query.domain_name).await {
                 active_queries += 1;
@@ -254,11 +258,14 @@ async fn fetch_ingestors_metrics(
     let mut vec = vec![];
     let mut active_ingestors = 0u64;
     let mut offline_ingestors = 0u64;
+
+    // for OSS, Query mode fetches the analytics report
+    // for Enterprise, Prism mode fetches the analytics report
     if PARSEABLE.options.mode == Mode::Query || PARSEABLE.options.mode == Mode::Prism {
         // send analytics for ingest servers
 
         // ingestor infos should be valid here, if not some thing is wrong
-        let ingestor_infos: Vec<NodeMetadata> = cluster::get_node_info("ingestor").await?;
+        let ingestor_infos: Vec<NodeMetadata> = cluster::get_node_info(NodeType::Ingestor).await?;
 
         for im in ingestor_infos {
             if !check_liveness(&im.domain_name).await {
diff --git a/src/catalog/mod.rs b/src/catalog/mod.rs
index 23f732053..8db1a49b4 100644
--- a/src/catalog/mod.rs
+++ b/src/catalog/mod.rs
@@ -30,7 +30,10 @@ use crate::{
     event::DEFAULT_TIMESTAMP_KEY,
     handlers::{
         self,
-        http::{base_path_without_preceding_slash, modal::NodeMetadata},
+        http::{
+            base_path_without_preceding_slash,
+            modal::{NodeMetadata, NodeType},
+        },
     },
     metrics::{EVENTS_INGESTED_DATE, EVENTS_INGESTED_SIZE_DATE, EVENTS_STORAGE_SIZE_DATE},
     option::Mode,
@@ -338,6 +341,10 @@ pub async fn remove_manifest_from_snapshot(
         meta.first_event_at = None;
         storage.put_snapshot(stream_name, meta.snapshot).await?;
     }
+
+    // retention is initiated from the querier
+    // request is forwarded to all ingestors to clean up their manifests
+    // no action required for the Index or Prism nodes
     match PARSEABLE.options.mode {
         Mode::All | Mode::Ingest => {
             Ok(get_first_event(storage.clone(), stream_name, Vec::new()).await?)
@@ -359,7 +366,6 @@ pub async fn get_first_event(
 ) -> Result<Option<String>, ObjectStorageError> {
     let mut first_event_at: String = String::default();
     match PARSEABLE.options.mode {
-        Mode::Index | Mode::Prism => unimplemented!(),
         Mode::All | Mode::Ingest => {
             // get current snapshot
             let stream_first_event = PARSEABLE.get_stream(stream_name)?.get_first_event();
@@ -410,7 +416,7 @@ pub async fn get_first_event(
         }
         Mode::Query => {
             let ingestor_metadata: Vec<NodeMetadata> =
-                handlers::http::cluster::get_node_info("ingestor")
+                handlers::http::cluster::get_node_info(NodeType::Ingestor)
                     .await
                     .map_err(|err| {
                         error!("Fatal: failed to get ingestor info: {:?}", err);
@@ -440,6 +446,7 @@ pub async fn get_first_event(
             }
             first_event_at = ingestors_first_event_at.iter().min().unwrap().to_string();
         }
+        _ => {}
     }
 
     Ok(Some(first_event_at))
diff --git a/src/cli.rs b/src/cli.rs
index 5446c7700..976e96969 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -447,6 +447,8 @@ impl Options {
         }
     }
 
+    /// get the address of the server
+    /// based on the mode
     pub fn get_url(&self, mode: Mode) -> Url {
         let endpoint = match mode {
             Mode::Ingest => self.get_endpoint(&self.ingestor_endpoint, "P_INGESTOR_ENDPOINT"),
@@ -458,6 +460,8 @@ impl Options {
         self.parse_endpoint(&endpoint)
     }
 
+    /// get the endpoint for the server
+    /// if env var is empty, use the address, else use the env var
     fn get_endpoint(&self, endpoint: &str, env_var: &str) -> String {
         if endpoint.is_empty() {
             self.address.to_string()
@@ -472,6 +476,9 @@ impl Options {
         }
     }
 
+    /// parse the endpoint to get the address and port
+    /// if the address is an env var, resolve it
+    /// if the port is an env var, resolve it
     fn parse_endpoint(&self, endpoint: &str) -> Url {
         let addr_parts: Vec<&str> = endpoint.split(':').collect();
 
@@ -488,6 +495,9 @@ impl Options {
         self.build_url(&format!("{}:{}", hostname, port))
     }
 
+    /// resolve the env var
+    /// if the env var is not set, panic
+    /// if the env var is set, return the value
     fn resolve_env_var(&self, value: &str) -> String {
         if let Some(env_var) = value.strip_prefix('$') {
             let resolved_value = env::var(env_var).unwrap_or_else(|_| {
@@ -510,6 +520,7 @@ impl Options {
         }
     }
 
+    /// build the url from the address
     fn build_url(&self, address: &str) -> Url {
         format!("{}://{}", self.get_scheme(), address)
             .parse::<Url>()
diff --git a/src/handlers/airplane.rs b/src/handlers/airplane.rs
index cea0f9778..8a5551c2f 100644
--- a/src/handlers/airplane.rs
+++ b/src/handlers/airplane.rs
@@ -34,7 +34,7 @@ use tonic::transport::{Identity, Server, ServerTlsConfig};
 use tonic_web::GrpcWebLayer;
 
 use crate::handlers::http::cluster::get_node_info;
-use crate::handlers::http::modal::NodeMetadata;
+use crate::handlers::http::modal::{NodeMetadata, NodeType};
 use crate::handlers::http::query::{into_query, update_schema_when_distributed};
 use crate::handlers::livetail::cross_origin_config;
 use crate::metrics::QUERY_EXECUTE_TIME;
@@ -180,7 +180,7 @@ impl FlightService for AirServiceImpl {
             })
             .to_string();
 
-            let ingester_metadatas: Vec<NodeMetadata> = get_node_info("ingestor")
+            let ingester_metadatas: Vec<NodeMetadata> = get_node_info(NodeType::Ingestor)
                 .await
                 .map_err(|err| Status::failed_precondition(err.to_string()))?;
             let mut minute_result: Vec<RecordBatch> = vec![];
diff --git a/src/handlers/http/about.rs b/src/handlers/http/about.rs
index ee26e95d5..57bf0197a 100644
--- a/src/handlers/http/about.rs
+++ b/src/handlers/http/about.rs
@@ -21,7 +21,6 @@ use serde_json::{json, Value};
 
 use crate::{
     about::{self, get_latest_release},
-    option::Mode,
     parseable::PARSEABLE,
     storage::StorageMetadata,
 };
@@ -63,12 +62,7 @@ pub async fn about() -> Json<Value> {
     let commit = current_release.commit_hash;
     let deployment_id = meta.deployment_id.to_string();
     let mode = PARSEABLE.get_server_mode_string();
-    let staging = if PARSEABLE.options.mode == Mode::Query || PARSEABLE.options.mode == Mode::Prism
-    {
-        "".to_string()
-    } else {
-        PARSEABLE.options.staging_dir().display().to_string()
-    };
+    let staging = PARSEABLE.options.staging_dir().display().to_string();
     let grpc_port = PARSEABLE.options.grpc_port;
 
     let store_endpoint = PARSEABLE.storage.get_endpoint();
diff --git a/src/handlers/http/cluster/mod.rs b/src/handlers/http/cluster/mod.rs
index 8a7bad026..0231fde4b 100644
--- a/src/handlers/http/cluster/mod.rs
+++ b/src/handlers/http/cluster/mod.rs
@@ -75,10 +75,11 @@ pub async fn sync_streams_with_ingestors(
     for (key, value) in headers.iter() {
         reqwest_headers.insert(key.clone(), value.clone());
     }
-    let ingestor_infos: Vec<NodeMetadata> = get_node_info("ingestor").await.map_err(|err| {
-        error!("Fatal: failed to get ingestor info: {:?}", err);
-        StreamError::Anyhow(err)
-    })?;
+    let ingestor_infos: Vec<NodeMetadata> =
+        get_node_info(NodeType::Ingestor).await.map_err(|err| {
+            error!("Fatal: failed to get ingestor info: {:?}", err);
+            StreamError::Anyhow(err)
+        })?;
 
     for ingestor in ingestor_infos {
         if !utils::check_liveness(&ingestor.domain_name).await {
@@ -123,10 +124,11 @@ pub async fn sync_users_with_roles_with_ingestors(
     username: &String,
     role: &HashSet<String>,
 ) -> Result<(), RBACError> {
-    let ingestor_infos: Vec<NodeMetadata> = get_node_info("ingestor").await.map_err(|err| {
-        error!("Fatal: failed to get ingestor info: {:?}", err);
-        RBACError::Anyhow(err)
-    })?;
+    let ingestor_infos: Vec<NodeMetadata> =
+        get_node_info(NodeType::Ingestor).await.map_err(|err| {
+            error!("Fatal: failed to get ingestor info: {:?}", err);
+            RBACError::Anyhow(err)
+        })?;
 
     let role = to_vec(&role.clone()).map_err(|err| {
         error!("Fatal: failed to serialize role: {:?}", err);
@@ -173,10 +175,11 @@ pub async fn sync_users_with_roles_with_ingestors(
 
 // forward the delete user request to all ingestors to keep them in sync
 pub async fn sync_user_deletion_with_ingestors(username: &String) -> Result<(), RBACError> {
-    let ingestor_infos: Vec<NodeMetadata> = get_node_info("ingestor").await.map_err(|err| {
-        error!("Fatal: failed to get ingestor info: {:?}", err);
-        RBACError::Anyhow(err)
-    })?;
+    let ingestor_infos: Vec<NodeMetadata> =
+        get_node_info(NodeType::Ingestor).await.map_err(|err| {
+            error!("Fatal: failed to get ingestor info: {:?}", err);
+            RBACError::Anyhow(err)
+        })?;
 
     for ingestor in ingestor_infos.iter() {
         if !utils::check_liveness(&ingestor.domain_name).await {
@@ -220,10 +223,11 @@ pub async fn sync_user_creation_with_ingestors(
     user: User,
     role: &Option<HashSet<String>>,
 ) -> Result<(), RBACError> {
-    let ingestor_infos: Vec<NodeMetadata> = get_node_info("ingestor").await.map_err(|err| {
-        error!("Fatal: failed to get ingestor info: {:?}", err);
-        RBACError::Anyhow(err)
-    })?;
+    let ingestor_infos: Vec<NodeMetadata> =
+        get_node_info(NodeType::Ingestor).await.map_err(|err| {
+            error!("Fatal: failed to get ingestor info: {:?}", err);
+            RBACError::Anyhow(err)
+        })?;
 
     let mut user = user.clone();
 
@@ -278,10 +282,11 @@ pub async fn sync_user_creation_with_ingestors(
 
 // forward the password reset request to all ingestors to keep them in sync
 pub async fn sync_password_reset_with_ingestors(username: &String) -> Result<(), RBACError> {
-    let ingestor_infos: Vec<NodeMetadata> = get_node_info("ingestor").await.map_err(|err| {
-        error!("Fatal: failed to get ingestor info: {:?}", err);
-        RBACError::Anyhow(err)
-    })?;
+    let ingestor_infos: Vec<NodeMetadata> =
+        get_node_info(NodeType::Ingestor).await.map_err(|err| {
+            error!("Fatal: failed to get ingestor info: {:?}", err);
+            RBACError::Anyhow(err)
+        })?;
 
     for ingestor in ingestor_infos.iter() {
         if !utils::check_liveness(&ingestor.domain_name).await {
@@ -326,10 +331,11 @@ pub async fn sync_role_update_with_ingestors(
     name: String,
     privileges: Vec<DefaultPrivilege>,
 ) -> Result<(), RoleError> {
-    let ingestor_infos: Vec<NodeMetadata> = get_node_info("ingestor").await.map_err(|err| {
-        error!("Fatal: failed to get ingestor info: {:?}", err);
-        RoleError::Anyhow(err)
-    })?;
+    let ingestor_infos: Vec<NodeMetadata> =
+        get_node_info(NodeType::Ingestor).await.map_err(|err| {
+            error!("Fatal: failed to get ingestor info: {:?}", err);
+            RoleError::Anyhow(err)
+        })?;
 
     for ingestor in ingestor_infos.iter() {
         if !utils::check_liveness(&ingestor.domain_name).await {
@@ -541,12 +547,13 @@ pub async fn send_retention_cleanup_request(
     Ok(first_event_at)
 }
 
+/// Fetches cluster information for all nodes (ingestor, indexer, and querier)
 pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
     // Get querier, ingestor and indexer metadata concurrently
     let (querier_result, ingestor_result, indexer_result) = future::join3(
-        get_node_info("querier"),
-        get_node_info("ingestor"),
-        get_node_info("indexer"),
+        get_node_info(NodeType::Querier),
+        get_node_info(NodeType::Ingestor),
+        get_node_info(NodeType::Indexer),
     )
     .await;
 
@@ -574,13 +581,14 @@ pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
         })
         .map_err(|err| StreamError::Anyhow(err.into()))?;
 
+    // Get self metadata
     let self_metadata = if let Some(metadata) = PARSEABLE.get_metadata() {
         vec![metadata]
     } else {
         vec![]
     };
 
-    // Fetch info for both node types concurrently
+    // Fetch info for all nodes concurrently
     let (querier_infos, ingestor_infos, indexer_infos, self_info) = future::join4(
         fetch_nodes_info(querier_metadata),
         fetch_nodes_info(ingestor_metadata),
@@ -589,7 +597,7 @@ pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
     )
     .await;
 
-    // Combine results from both node types
+    // Combine results from all node types
     let mut infos = Vec::new();
     infos.extend(self_info?);
     infos.extend(querier_infos?);
@@ -598,7 +606,9 @@ pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
     Ok(actix_web::HttpResponse::Ok().json(infos))
 }
 
-/// Fetches info for a single node (ingestor or indexer)
+/// Fetches info for a single node
+/// call the about endpoint of the node
+/// construct the ClusterInfo struct and return it
 async fn fetch_node_info<T: Metadata>(node: &T) -> Result<utils::ClusterInfo, StreamError> {
     let uri = Url::parse(&format!(
         "{}{}/about",
@@ -690,10 +700,15 @@ pub async fn get_cluster_metrics() -> Result<impl Responder, PostError> {
     Ok(actix_web::HttpResponse::Ok().json(dresses))
 }
 
-pub async fn get_node_info<T: Metadata + DeserializeOwned>(prefix: &str) -> anyhow::Result<Vec<T>> {
+/// get node info for a specific node type
+/// this is used to get the node info for ingestor, indexer and querier
+/// it will return the metadata for all nodes of that type
+pub async fn get_node_info<T: Metadata + DeserializeOwned>(
+    node_type: NodeType,
+) -> anyhow::Result<Vec<T>> {
     let store = PARSEABLE.storage.get_object_store();
     let root_path = RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY);
-    let prefix_owned = prefix.to_string(); // Create an owned copy of the prefix
+    let prefix_owned = node_type.to_string();
 
     let metadata = store
         .get_objects(
@@ -707,7 +722,11 @@ pub async fn get_node_info<T: Metadata + DeserializeOwned>(prefix: &str) -> anyh
 
     Ok(metadata)
 }
-
+/// remove a node from the cluster
+/// check liveness of the node
+/// if the node is live, return an error
+/// if the node is not live, remove the node from the cluster
+/// remove the node metadata from the object store
 pub async fn remove_node(node_url: Path<String>) -> Result<impl Responder, PostError> {
     let domain_name = to_url_string(node_url.into_inner());
 
@@ -745,7 +764,8 @@ pub async fn remove_node(node_url: Path<String>) -> Result<impl Responder, PostE
     )))
 }
 
-// Helper function to remove a specific type of node metadata
+/// Removes node metadata from the object store
+/// Returns true if the metadata was removed, false if it was not found
 async fn remove_node_metadata<T: Metadata + DeserializeOwned + Default>(
     object_store: &Arc<dyn ObjectStorage>,
     domain_name: &str,
@@ -783,7 +803,10 @@ async fn remove_node_metadata<T: Metadata + DeserializeOwned + Default>(
     }
 }
 
-/// Fetches metrics from a node (ingestor or indexer)
+/// Fetches metrics for a single node
+/// This function is used to fetch metrics from a single node
+/// It checks if the node is live and then fetches the metrics
+/// If the node is not live, it returns None
 async fn fetch_node_metrics<T>(node: &T) -> Result<Option<Metrics>, PostError>
 where
     T: Metadata + Send + Sync + 'static,
@@ -867,13 +890,16 @@ where
     Ok(metrics)
 }
 
-/// Main function to fetch all cluster metrics, parallelized and refactored
+/// Main function to fetch cluster metrics
+/// fetches node info for all nodes
+/// fetches metrics for all nodes
+/// combines all metrics into a single vector
 async fn fetch_cluster_metrics() -> Result<Vec<Metrics>, PostError> {
     // Get ingestor and indexer metadata concurrently
     let (querier_result, ingestor_result, indexer_result) = future::join3(
-        get_node_info("querier"),
-        get_node_info("ingestor"),
-        get_node_info("indexer"),
+        get_node_info(NodeType::Querier),
+        get_node_info(NodeType::Ingestor),
+        get_node_info(NodeType::Indexer),
     )
     .await;
 
diff --git a/src/handlers/http/mod.rs b/src/handlers/http/mod.rs
index ea19ed3be..3d674d20b 100644
--- a/src/handlers/http/mod.rs
+++ b/src/handlers/http/mod.rs
@@ -22,7 +22,7 @@ use arrow_schema::Schema;
 use cluster::get_node_info;
 use http::StatusCode;
 use itertools::Itertools;
-use modal::NodeMetadata;
+use modal::{NodeMetadata, NodeType};
 use serde_json::Value;
 
 use crate::{parseable::PARSEABLE, storage::STREAM_ROOT_DIRECTORY, HTTP_CLIENT};
@@ -110,7 +110,7 @@ pub async fn fetch_schema(stream_name: &str) -> anyhow::Result<arrow_schema::Sch
 pub async fn send_query_request_to_ingestor(query: &Query) -> anyhow::Result<Vec<Value>> {
     // send the query request to the ingestor
     let mut res = vec![];
-    let ima: Vec<NodeMetadata> = get_node_info("ingestor").await?;
+    let ima: Vec<NodeMetadata> = get_node_info(NodeType::Ingestor).await?;
 
     for im in ima.iter() {
         let uri = format!(
diff --git a/src/handlers/http/modal/mod.rs b/src/handlers/http/modal/mod.rs
index abbd8265e..0c39872b2 100644
--- a/src/handlers/http/modal/mod.rs
+++ b/src/handlers/http/modal/mod.rs
@@ -199,6 +199,7 @@ pub async fn load_on_init() -> anyhow::Result<()> {
     Ok(())
 }
 
+/// NodeType represents the type of node in the cluster
 #[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Default)]
 #[serde(rename_all = "lowercase")]
 pub enum NodeType {
@@ -531,7 +532,7 @@ impl Metadata for NodeMetadata {
     }
 }
 
-// Type aliases for backward compatibility
+// Aliases for different node types
 pub type IngestorMetadata = NodeMetadata;
 pub type IndexerMetadata = NodeMetadata;
 pub type QuerierMetadata = NodeMetadata;
diff --git a/src/handlers/http/modal/query/querier_logstream.rs b/src/handlers/http/modal/query/querier_logstream.rs
index 27837d91f..ebffbfb43 100644
--- a/src/handlers/http/modal/query/querier_logstream.rs
+++ b/src/handlers/http/modal/query/querier_logstream.rs
@@ -41,7 +41,7 @@ use crate::{
             utils::{merge_quried_stats, IngestionStats, QueriedStats, StorageStats},
         },
         logstream::{error::StreamError, get_stats_date},
-        modal::NodeMetadata,
+        modal::{NodeMetadata, NodeType},
     },
     hottier::HotTierManager,
     parseable::{StreamNotFound, PARSEABLE},
@@ -82,8 +82,9 @@ pub async fn delete(stream_name: Path<String>) -> Result<impl Responder, StreamE
         }
     }
 
-    let ingestor_metadata: Vec<NodeMetadata> =
-        cluster::get_node_info("ingestor").await.map_err(|err| {
+    let ingestor_metadata: Vec<NodeMetadata> = cluster::get_node_info(NodeType::Ingestor)
+        .await
+        .map_err(|err| {
             error!("Fatal: failed to get ingestor info: {:?}", err);
             err
         })?;
diff --git a/src/handlers/http/query.rs b/src/handlers/http/query.rs
index 8ea9c316b..368fe8b9c 100644
--- a/src/handlers/http/query.rs
+++ b/src/handlers/http/query.rs
@@ -172,6 +172,9 @@ pub async fn get_counts(
 }
 
 pub async fn update_schema_when_distributed(tables: &Vec<String>) -> Result<(), EventError> {
+    // if the mode is query or prism, we need to update the schema in memory
+    // no need to commit schema to storage
+    // as the schema is read from memory everytime
     if PARSEABLE.options.mode == Mode::Query || PARSEABLE.options.mode == Mode::Prism {
         for table in tables {
             if let Ok(new_schema) = fetch_schema(table).await {
diff --git a/src/parseable/mod.rs b/src/parseable/mod.rs
index dea5c8a99..5a199818f 100644
--- a/src/parseable/mod.rs
+++ b/src/parseable/mod.rs
@@ -558,8 +558,11 @@ impl Parseable {
 
         let stream_in_memory_dont_update =
             self.streams.contains(stream_name) && !update_stream_flag;
-        let stream_in_storage_only_for_query_node = !self.streams.contains(stream_name)     // check if stream in storage only if not in memory
-            && (self.options.mode == Mode::Query  || self.options.mode == Mode::Prism)                                                // and running in query mode
+        // check if stream in storage only if not in memory
+        // for Parseable OSS, create_update_stream is called only from query node
+        // for Parseable Enterprise, create_update_stream is called from prism node
+        let stream_in_storage_only_for_query_node = !self.streams.contains(stream_name)
+            && (self.options.mode == Mode::Query || self.options.mode == Mode::Prism)
             && self
                 .create_stream_and_schema_from_storage(stream_name)
                 .await?;

From c66fb74d2bf9f71514d71a7f444f3054d979155d Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 14 Apr 2025 07:11:04 -0400
Subject: [PATCH 31/42] delete from staging for ingestor

---
 src/handlers/http/modal/ingest/ingestor_logstream.rs | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/handlers/http/modal/ingest/ingestor_logstream.rs b/src/handlers/http/modal/ingest/ingestor_logstream.rs
index 4872a6476..27f914536 100644
--- a/src/handlers/http/modal/ingest/ingestor_logstream.rs
+++ b/src/handlers/http/modal/ingest/ingestor_logstream.rs
@@ -16,6 +16,8 @@
  *
  */
 
+use std::fs;
+
 use actix_web::{
     web::{Json, Path},
     HttpRequest, Responder,
@@ -69,6 +71,16 @@ pub async fn delete(stream_name: Path<String>) -> Result<impl Responder, StreamE
         return Err(StreamNotFound(stream_name.clone()).into());
     }
 
+    // Delete from staging
+    let stream_dir = PARSEABLE.get_or_create_stream(&stream_name);
+    if fs::remove_dir_all(&stream_dir.data_path).is_err() {
+        warn!(
+            "failed to delete local data for stream {}. Clean {} manually",
+            stream_name,
+            stream_dir.data_path.to_string_lossy()
+        )
+    }
+
     // Delete from memory
     PARSEABLE.streams.delete(&stream_name);
     stats::delete_stats(&stream_name, "json")

From 264ccf306aabe9e385e889dd1647ae4afdbc098f Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 14 Apr 2025 07:16:13 -0400
Subject: [PATCH 32/42] replaced to get_stream

---
 src/handlers/http/modal/ingest/ingestor_logstream.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/handlers/http/modal/ingest/ingestor_logstream.rs b/src/handlers/http/modal/ingest/ingestor_logstream.rs
index 27f914536..b5ab42a40 100644
--- a/src/handlers/http/modal/ingest/ingestor_logstream.rs
+++ b/src/handlers/http/modal/ingest/ingestor_logstream.rs
@@ -72,7 +72,7 @@ pub async fn delete(stream_name: Path<String>) -> Result<impl Responder, StreamE
     }
 
     // Delete from staging
-    let stream_dir = PARSEABLE.get_or_create_stream(&stream_name);
+    let stream_dir = PARSEABLE.get_stream(&stream_name)?;
     if fs::remove_dir_all(&stream_dir.data_path).is_err() {
         warn!(
             "failed to delete local data for stream {}. Clean {} manually",

From 70c03422f448a329f6d4e8f6c0a1b2c0368c1fa0 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 14 Apr 2025 23:02:47 -0400
Subject: [PATCH 33/42] remove node metadata from parseable struct, add as
 OnceCell

---
 src/handlers/http/cluster/mod.rs         |  26 +----
 src/handlers/http/modal/ingest_server.rs |  18 ++-
 src/handlers/http/modal/mod.rs           | 118 ++++++++++++-------
 src/handlers/http/modal/query_server.rs  |  15 ++-
 src/parseable/mod.rs                     | 142 ++++-------------------
 src/storage/object_storage.rs            |  22 ++--
 6 files changed, 137 insertions(+), 204 deletions(-)

diff --git a/src/handlers/http/cluster/mod.rs b/src/handlers/http/cluster/mod.rs
index 0231fde4b..5a7a676e5 100644
--- a/src/handlers/http/cluster/mod.rs
+++ b/src/handlers/http/cluster/mod.rs
@@ -581,25 +581,16 @@ pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
         })
         .map_err(|err| StreamError::Anyhow(err.into()))?;
 
-    // Get self metadata
-    let self_metadata = if let Some(metadata) = PARSEABLE.get_metadata() {
-        vec![metadata]
-    } else {
-        vec![]
-    };
-
     // Fetch info for all nodes concurrently
-    let (querier_infos, ingestor_infos, indexer_infos, self_info) = future::join4(
+    let (querier_infos, ingestor_infos, indexer_infos) = future::join3(
         fetch_nodes_info(querier_metadata),
         fetch_nodes_info(ingestor_metadata),
         fetch_nodes_info(indexer_metadata),
-        fetch_nodes_info(self_metadata),
     )
     .await;
 
     // Combine results from all node types
     let mut infos = Vec::new();
-    infos.extend(self_info?);
     infos.extend(querier_infos?);
     infos.extend(ingestor_infos?);
     infos.extend(indexer_infos?);
@@ -903,12 +894,6 @@ async fn fetch_cluster_metrics() -> Result<Vec<Metrics>, PostError> {
     )
     .await;
 
-    let self_metadata = if let Some(metadata) = PARSEABLE.get_metadata() {
-        vec![metadata]
-    } else {
-        vec![]
-    };
-
     // Handle querier metadata result
     let querier_metadata: Vec<NodeMetadata> = querier_result.map_err(|err| {
         error!("Fatal: failed to get querier info: {:?}", err);
@@ -925,8 +910,7 @@ async fn fetch_cluster_metrics() -> Result<Vec<Metrics>, PostError> {
         PostError::Invalid(err)
     })?;
     // Fetch metrics from ingestors and indexers concurrently
-    let (self_metrics, querier_metrics, ingestor_metrics, indexer_metrics) = future::join4(
-        fetch_nodes_metrics(self_metadata),
+    let (querier_metrics, ingestor_metrics, indexer_metrics) = future::join3(
         fetch_nodes_metrics(querier_metadata),
         fetch_nodes_metrics(ingestor_metadata),
         fetch_nodes_metrics(indexer_metadata),
@@ -936,12 +920,6 @@ async fn fetch_cluster_metrics() -> Result<Vec<Metrics>, PostError> {
     // Combine all metrics
     let mut all_metrics = Vec::new();
 
-    // Add self metrics
-    match self_metrics {
-        Ok(metrics) => all_metrics.extend(metrics),
-        Err(err) => return Err(err),
-    }
-
     // Add querier metrics
     match querier_metrics {
         Ok(metrics) => all_metrics.extend(metrics),
diff --git a/src/handlers/http/modal/ingest_server.rs b/src/handlers/http/modal/ingest_server.rs
index d1b271134..65eb81467 100644
--- a/src/handlers/http/modal/ingest_server.rs
+++ b/src/handlers/http/modal/ingest_server.rs
@@ -27,8 +27,9 @@ use bytes::Bytes;
 use relative_path::RelativePathBuf;
 use serde_json::Value;
 use tokio::sync::oneshot;
+use tokio::sync::OnceCell;
 
-use crate::option::Mode;
+use crate::handlers::http::modal::NodeType;
 use crate::{
     analytics,
     handlers::{
@@ -46,13 +47,14 @@ use crate::{
     sync, Server,
 };
 
+use super::IngestorMetadata;
 use super::{
     ingest::{ingestor_logstream, ingestor_rbac, ingestor_role},
     OpenIdClient, ParseableServer,
 };
 
 pub const INGESTOR_EXPECT: &str = "Ingestor Metadata should be set in ingestor mode";
-
+pub static INGESTOR_META: OnceCell<IngestorMetadata> = OnceCell::const_new();
 pub struct IngestServer;
 
 #[async_trait]
@@ -98,6 +100,15 @@ impl ParseableServer for IngestServer {
         prometheus: &PrometheusMetrics,
         shutdown_rx: oneshot::Receiver<()>,
     ) -> anyhow::Result<()> {
+        // write the ingestor metadata to storage
+        INGESTOR_META
+            .get_or_init(|| async {
+                IngestorMetadata::load_node_metadata(NodeType::Ingestor)
+                    .await
+                    .expect("Ingestor Metadata should be set in ingestor mode")
+            })
+            .await;
+
         PARSEABLE.storage.register_store_metrics(prometheus);
 
         migration::run_migration(&PARSEABLE).await?;
@@ -108,9 +119,6 @@ impl ParseableServer for IngestServer {
 
         tokio::spawn(airplane::server());
 
-        // write the ingestor metadata to storage
-        PARSEABLE.store_metadata(Mode::Ingest).await?;
-
         // Ingestors shouldn't have to deal with OpenId auth flow
         let result = self.start(shutdown_rx, prometheus.clone(), None).await;
         // Cancel sync jobs
diff --git a/src/handlers/http/modal/mod.rs b/src/handlers/http/modal/mod.rs
index 0c39872b2..29f6376a4 100644
--- a/src/handlers/http/modal/mod.rs
+++ b/src/handlers/http/modal/mod.rs
@@ -277,28 +277,92 @@ impl NodeMetadata {
         }
     }
 
-    /// Capture metadata information by either loading it from staging or starting fresh
-    pub fn load(
-        options: &Options,
-        storage: &dyn ObjectStorageProvider,
-        node_type: NodeType,
-    ) -> Arc<Self> {
-        let staging_path = options.staging_dir();
+    pub async fn load_node_metadata(node_type: NodeType) -> anyhow::Result<Self> {
+        let staging_path = PARSEABLE.options.staging_dir();
         let node_type_str = node_type.as_str();
 
         // Attempt to load metadata from staging
-        if let Some(mut meta) = Self::load_from_staging(staging_path, node_type_str, options) {
-            Self::update_metadata(&mut meta, options, node_type);
-            meta.put_on_disk(staging_path)
-                .expect("Couldn't write updated metadata to disk");
-            return Arc::new(meta);
+        if let Some(meta) = Self::load_from_staging(staging_path, node_type_str, &PARSEABLE.options)
+        {
+            return Self::process_and_store_metadata(meta, staging_path, node_type).await;
+        }
+
+        // Attempt to load metadata from storage
+        let storage_metas = Self::load_from_storage(node_type_str.to_string()).await;
+        let url = PARSEABLE.options.get_url(node_type.to_mode());
+        let port = url.port().unwrap_or(80).to_string();
+        let url = url.to_string();
+
+        for storage_meta in storage_metas {
+            if storage_meta.domain_name == url && storage_meta.port == port {
+                return Self::process_and_store_metadata(storage_meta, staging_path, node_type)
+                    .await;
+            }
         }
 
-        // If no metadata is found in staging, create a new one
-        let meta = Self::create_new_metadata(options, storage, node_type);
+        // If no metadata is found, create a new one
+        let meta = Self::create_new_metadata(&PARSEABLE.options, &*PARSEABLE.storage, node_type);
+        Self::store_new_metadata(meta, staging_path).await
+    }
+
+    /// Process and store metadata
+    async fn process_and_store_metadata(
+        mut meta: Self,
+        staging_path: &Path,
+        node_type: NodeType,
+    ) -> anyhow::Result<Self> {
+        Self::update_metadata(&mut meta, &PARSEABLE.options, node_type);
+        meta.put_on_disk(staging_path)
+            .expect("Couldn't write updated metadata to disk");
+
+        let path = meta.file_path();
+        let resource = serde_json::to_vec(&meta)?.into();
+        let store = PARSEABLE.storage.get_object_store();
+        store.put_object(&path, resource).await?;
+
+        Ok(meta)
+    }
+
+    /// Store new metadata
+    async fn store_new_metadata(meta: Self, staging_path: &Path) -> anyhow::Result<Self> {
         meta.put_on_disk(staging_path)
             .expect("Couldn't write new metadata to disk");
-        Arc::new(meta)
+
+        let path = meta.file_path();
+        let resource = serde_json::to_vec(&meta)?.into();
+        let store = PARSEABLE.storage.get_object_store();
+        store.put_object(&path, resource).await?;
+
+        Ok(meta)
+    }
+
+    async fn load_from_storage(node_type: String) -> Vec<NodeMetadata> {
+        let path = RelativePathBuf::from(PARSEABLE_ROOT_DIRECTORY);
+        let glob_storage = PARSEABLE.storage.get_object_store();
+        let obs = glob_storage
+            .get_objects(
+                Some(&path),
+                Box::new({
+                    let node_type = node_type.clone();
+                    move |file_name| file_name.contains(&node_type)
+                }),
+            )
+            .await;
+
+        let mut metadata = vec![];
+        if let Ok(obs) = obs {
+            for object in obs {
+                //convert to NodeMetadata
+                match serde_json::from_slice::<NodeMetadata>(&object) {
+                    Ok(node_metadata) => metadata.push(node_metadata),
+                    Err(e) => error!("Failed to deserialize NodeMetadata: {:?}", e),
+                }
+            }
+        } else {
+            error!("Couldn't read from storage");
+        }
+        // Return the metadata
+        metadata
     }
 
     /// Load metadata from the staging directory
@@ -467,30 +531,6 @@ impl NodeMetadata {
         Ok(metadata)
     }
 
-    pub async fn migrate(&self) -> anyhow::Result<Option<Self>> {
-        let imp = self.file_path();
-        let bytes = match PARSEABLE.storage.get_object_store().get_object(&imp).await {
-            Ok(bytes) => bytes,
-            Err(_) => {
-                return Ok(None);
-            }
-        };
-
-        let mut resource = Self::from_bytes(&bytes, PARSEABLE.options.flight_port)?;
-        resource.node_type.clone_from(&self.node_type);
-        let bytes = Bytes::from(serde_json::to_vec(&resource)?);
-
-        resource.put_on_disk(PARSEABLE.options.staging_dir())?;
-
-        PARSEABLE
-            .storage
-            .get_object_store()
-            .put_object(&imp, bytes)
-            .await?;
-
-        Ok(Some(resource))
-    }
-
     /// Puts the node info into the staging.
     ///
     /// This function takes the node info as a parameter and stores it in staging.
diff --git a/src/handlers/http/modal/query_server.rs b/src/handlers/http/modal/query_server.rs
index d4ce26f72..614d9d425 100644
--- a/src/handlers/http/modal/query_server.rs
+++ b/src/handlers/http/modal/query_server.rs
@@ -32,17 +32,17 @@ use actix_web::{web, Scope};
 use actix_web_prometheus::PrometheusMetrics;
 use async_trait::async_trait;
 use bytes::Bytes;
-use tokio::sync::oneshot;
+use tokio::sync::{oneshot, OnceCell};
 use tracing::info;
 
 use crate::parseable::PARSEABLE;
 use crate::Server;
 
 use super::query::{querier_ingest, querier_logstream, querier_rbac, querier_role};
-use super::{load_on_init, OpenIdClient, ParseableServer};
+use super::{load_on_init, NodeType, OpenIdClient, ParseableServer, QuerierMetadata};
 
 pub struct QueryServer;
-
+pub static QUERIER_META: OnceCell<QuerierMetadata> = OnceCell::const_new();
 #[async_trait]
 impl ParseableServer for QueryServer {
     // configure the api routes
@@ -99,7 +99,14 @@ impl ParseableServer for QueryServer {
         shutdown_rx: oneshot::Receiver<()>,
     ) -> anyhow::Result<()> {
         PARSEABLE.storage.register_store_metrics(prometheus);
-
+        // write the ingestor metadata to storage
+        QUERIER_META
+            .get_or_init(|| async {
+                QuerierMetadata::load_node_metadata(NodeType::Querier)
+                    .await
+                    .expect("Querier Metadata should be set in ingestor mode")
+            })
+            .await;
         migration::run_migration(&PARSEABLE).await?;
 
         //create internal stream at server start
diff --git a/src/parseable/mod.rs b/src/parseable/mod.rs
index 5a199818f..08e0feb76 100644
--- a/src/parseable/mod.rs
+++ b/src/parseable/mod.rs
@@ -47,10 +47,7 @@ use crate::{
             cluster::{sync_streams_with_ingestors, INTERNAL_STREAM_NAME},
             ingest::PostError,
             logstream::error::{CreateStreamError, StreamError},
-            modal::{
-                utils::logstream_utils::PutStreamHeaders, IndexerMetadata, IngestorMetadata,
-                Metadata, NodeMetadata, NodeType, PrismMetadata, QuerierMetadata,
-            },
+            modal::{ingest_server::INGESTOR_META, utils::logstream_utils::PutStreamHeaders},
         },
         STREAM_TYPE_KEY,
     },
@@ -130,14 +127,6 @@ pub struct Parseable {
     /// Metadata and staging realting to each logstreams
     /// A globally shared mapping of `Streams` that parseable is aware of.
     pub streams: Streams,
-    ///Metadata associated only with a prism
-    pub prism_metadata: Option<Arc<PrismMetadata>>,
-    /// Metadata associated only with a querier
-    pub querier_metadata: Option<Arc<QuerierMetadata>>,
-    /// Metadata associated only with an ingestor
-    pub ingestor_metadata: Option<Arc<IngestorMetadata>>,
-    /// Metadata associated only with an indexer
-    pub indexer_metadata: Option<Arc<IndexerMetadata>>,
     /// Used to configure the kafka connector
     #[cfg(feature = "kafka")]
     pub kafka_config: KafkaConfig,
@@ -149,74 +138,14 @@ impl Parseable {
         #[cfg(feature = "kafka")] kafka_config: KafkaConfig,
         storage: Arc<dyn ObjectStorageProvider>,
     ) -> Self {
-        let prism_metadata = match &options.mode {
-            Mode::Prism => Some(PrismMetadata::load(
-                &options,
-                storage.as_ref(),
-                NodeType::Prism,
-            )),
-            _ => None,
-        };
-        let ingestor_metadata = match &options.mode {
-            Mode::Ingest => Some(IngestorMetadata::load(
-                &options,
-                storage.as_ref(),
-                NodeType::Ingestor,
-            )),
-            _ => None,
-        };
-        let indexer_metadata = match &options.mode {
-            Mode::Index => Some(IndexerMetadata::load(
-                &options,
-                storage.as_ref(),
-                NodeType::Indexer,
-            )),
-            _ => None,
-        };
-        let querier_metadata = match &options.mode {
-            Mode::Query => Some(QuerierMetadata::load(
-                &options,
-                storage.as_ref(),
-                NodeType::Querier,
-            )),
-            _ => None,
-        };
         Parseable {
             options: Arc::new(options),
             storage,
             streams: Streams::default(),
-            prism_metadata,
-            ingestor_metadata,
-            indexer_metadata,
-            querier_metadata,
             #[cfg(feature = "kafka")]
             kafka_config,
         }
     }
-
-    /// Get the metadata for the current node based on its mode.
-    pub fn get_metadata(&self) -> Option<NodeMetadata> {
-        let meta_ref = match self.options.mode {
-            Mode::Ingest => self.ingestor_metadata.as_ref(),
-            Mode::Index => self.indexer_metadata.as_ref(),
-            Mode::Query => self.querier_metadata.as_ref(),
-            Mode::Prism => self.prism_metadata.as_ref(),
-            _ => return None,
-        };
-
-        let meta = meta_ref?;
-        let node_metadata = NodeMetadata {
-            version: meta.version.clone(),
-            node_id: meta.node_id.clone(),
-            port: meta.port.clone(),
-            domain_name: meta.domain_name.clone(),
-            bucket_name: meta.bucket_name.clone(),
-            token: meta.token.clone(),
-            flight_port: meta.flight_port.clone(),
-            node_type: meta.node_type().clone(),
-        };
-        Some(node_metadata)
-    }
     /// Try to get the handle of a stream in staging, if it doesn't exist return `None`.
     pub fn get_stream(&self, stream_name: &str) -> Result<StreamRef, StreamNotFound> {
         self.streams
@@ -233,14 +162,16 @@ impl Parseable {
             return staging;
         }
 
+        let ingestor_id = INGESTOR_META
+            .get()
+            .map(|ingestor_metadata| ingestor_metadata.get_node_id());
+
         // Gets write privileges only for creating the stream when it doesn't already exist.
         self.streams.get_or_create(
             self.options.clone(),
             stream_name.to_owned(),
             LogStreamMetadata::default(),
-            self.ingestor_metadata
-                .as_ref()
-                .map(|meta| meta.get_node_id()),
+            ingestor_id,
         )
     }
 
@@ -325,43 +256,6 @@ impl Parseable {
         }
     }
 
-    pub async fn store_metadata(&self, mode: Mode) -> anyhow::Result<()> {
-        // Get the appropriate metadata based on mode
-        let meta_ref = match mode {
-            Mode::Ingest => self.ingestor_metadata.as_ref(),
-            Mode::Index => self.indexer_metadata.as_ref(),
-            Mode::Query => self.querier_metadata.as_ref(),
-            _ => return Err(anyhow::anyhow!("Invalid mode")),
-        };
-
-        let Some(meta) = meta_ref else {
-            return Ok(());
-        };
-
-        // Get storage metadata
-        let storage_metadata = meta.migrate().await?;
-        let store = self.storage.get_object_store();
-        let path = meta.file_path();
-
-        // Process metadata and store it
-        if let Some(mut store_data) = storage_metadata {
-            // Update domain and port if needed
-            if store_data.domain_name != meta.domain_name {
-                store_data.domain_name.clone_from(&meta.domain_name);
-                store_data.port.clone_from(&meta.port);
-
-                let resource = Bytes::from(serde_json::to_vec(&store_data)?);
-                store.put_object(&path, resource).await?;
-            }
-        } else {
-            // Store new metadata
-            let resource = serde_json::to_vec(&meta)?.into();
-            store.put_object(&path, resource).await?;
-        }
-
-        Ok(())
-    }
-
     /// list all streams from storage
     /// if stream exists in storage, create stream and schema from storage
     /// and add it to the memory map
@@ -415,13 +309,16 @@ impl Parseable {
             schema_version,
             log_source,
         );
+        let ingestor_id = INGESTOR_META
+            .get()
+            .map(|ingestor_metadata| ingestor_metadata.get_node_id());
+
+        // Gets write privileges only for creating the stream when it doesn't already exist.
         self.streams.get_or_create(
             self.options.clone(),
-            stream_name.to_string(),
+            stream_name.to_owned(),
             metadata,
-            self.ingestor_metadata
-                .as_ref()
-                .map(|meta| meta.get_node_id()),
+            ingestor_id,
         );
 
         Ok(true)
@@ -723,13 +620,18 @@ impl Parseable {
                     SchemaVersion::V1, // New stream
                     log_source,
                 );
+                let ingestor_id = if let Some(ingestor_metadata) = INGESTOR_META.get() {
+                    Some(ingestor_metadata.get_node_id())
+                } else {
+                    None
+                };
+
+                // Gets write privileges only for creating the stream when it doesn't already exist.
                 self.streams.get_or_create(
                     self.options.clone(),
-                    stream_name.to_string(),
+                    stream_name.to_owned(),
                     metadata,
-                    self.ingestor_metadata
-                        .as_ref()
-                        .map(|meta| meta.get_node_id()),
+                    ingestor_id,
                 );
             }
             Err(err) => {
diff --git a/src/storage/object_storage.rs b/src/storage/object_storage.rs
index a41bb8447..49da6b8fd 100644
--- a/src/storage/object_storage.rs
+++ b/src/storage/object_storage.rs
@@ -48,6 +48,7 @@ use crate::correlation::{CorrelationConfig, CorrelationError};
 use crate::event::format::LogSource;
 use crate::event::format::LogSourceEntry;
 use crate::handlers::http::modal::ingest_server::INGESTOR_EXPECT;
+use crate::handlers::http::modal::ingest_server::INGESTOR_META;
 use crate::handlers::http::users::CORRELATION_DIR;
 use crate::handlers::http::users::{DASHBOARDS_DIR, FILTER_DIR, USERS_ROOT_DIR};
 use crate::metrics::storage::StorageMetrics;
@@ -896,10 +897,9 @@ pub fn to_bytes(any: &(impl ?Sized + serde::Serialize)) -> Bytes {
 
 pub fn schema_path(stream_name: &str) -> RelativePathBuf {
     if PARSEABLE.options.mode == Mode::Ingest {
-        let id = PARSEABLE
-            .ingestor_metadata
-            .as_ref()
-            .expect(INGESTOR_EXPECT)
+        let id = INGESTOR_META
+            .get()
+            .unwrap_or_else(|| panic!("{}", INGESTOR_EXPECT))
             .get_node_id();
         let file_name = format!(".ingestor.{id}{SCHEMA_FILE_NAME}");
 
@@ -912,10 +912,9 @@ pub fn schema_path(stream_name: &str) -> RelativePathBuf {
 #[inline(always)]
 pub fn stream_json_path(stream_name: &str) -> RelativePathBuf {
     if PARSEABLE.options.mode == Mode::Ingest {
-        let id = PARSEABLE
-            .ingestor_metadata
-            .as_ref()
-            .expect(INGESTOR_EXPECT)
+        let id = INGESTOR_META
+            .get()
+            .unwrap_or_else(|| panic!("{}", INGESTOR_EXPECT))
             .get_node_id();
         let file_name = format!(".ingestor.{id}{STREAM_METADATA_FILE_NAME}",);
         RelativePathBuf::from_iter([stream_name, STREAM_ROOT_DIRECTORY, &file_name])
@@ -962,10 +961,9 @@ pub fn alert_json_path(alert_id: Ulid) -> RelativePathBuf {
 pub fn manifest_path(prefix: &str) -> RelativePathBuf {
     match &PARSEABLE.options.mode {
         Mode::Ingest => {
-            let id = PARSEABLE
-                .ingestor_metadata
-                .as_ref()
-                .expect(INGESTOR_EXPECT)
+            let id = INGESTOR_META
+                .get()
+                .unwrap_or_else(|| panic!("{}", INGESTOR_EXPECT))
                 .get_node_id();
             let manifest_file_name = format!("ingestor.{id}.{MANIFEST_FILE}");
             RelativePathBuf::from_iter([prefix, &manifest_file_name])

From 0049e1b7696a1646f7868ceba87c864e3bf63a3b Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Mon, 14 Apr 2025 23:13:44 -0400
Subject: [PATCH 34/42] ingestor_id mapping

---
 src/parseable/mod.rs | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/parseable/mod.rs b/src/parseable/mod.rs
index 08e0feb76..83197e42e 100644
--- a/src/parseable/mod.rs
+++ b/src/parseable/mod.rs
@@ -620,11 +620,9 @@ impl Parseable {
                     SchemaVersion::V1, // New stream
                     log_source,
                 );
-                let ingestor_id = if let Some(ingestor_metadata) = INGESTOR_META.get() {
-                    Some(ingestor_metadata.get_node_id())
-                } else {
-                    None
-                };
+                let ingestor_id = INGESTOR_META
+                    .get()
+                    .map(|ingestor_metadata| ingestor_metadata.get_node_id());
 
                 // Gets write privileges only for creating the stream when it doesn't already exist.
                 self.streams.get_or_create(

From 87f98031600e6eaeb25b5f1f0705f7b89c37b690 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Tue, 15 Apr 2025 01:23:08 -0400
Subject: [PATCH 35/42] prism mode in cluster info and metrics

---
 src/handlers/http/cluster/mod.rs | 33 ++++++++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/src/handlers/http/cluster/mod.rs b/src/handlers/http/cluster/mod.rs
index 5a7a676e5..a503c309d 100644
--- a/src/handlers/http/cluster/mod.rs
+++ b/src/handlers/http/cluster/mod.rs
@@ -550,13 +550,22 @@ pub async fn send_retention_cleanup_request(
 /// Fetches cluster information for all nodes (ingestor, indexer, and querier)
 pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
     // Get querier, ingestor and indexer metadata concurrently
-    let (querier_result, ingestor_result, indexer_result) = future::join3(
+    let (prism_result, querier_result, ingestor_result, indexer_result) = future::join4(
+        get_node_info(NodeType::Prism),
         get_node_info(NodeType::Querier),
         get_node_info(NodeType::Ingestor),
         get_node_info(NodeType::Indexer),
     )
     .await;
 
+    // Handle prism metadata result
+    let prism_metadata: Vec<NodeMetadata> = prism_result
+        .map_err(|err| {
+            error!("Fatal: failed to get prism info: {:?}", err);
+            PostError::Invalid(err)
+        })
+        .map_err(|err| StreamError::Anyhow(err.into()))?;
+
     // Handle querier metadata result
     let querier_metadata: Vec<NodeMetadata> = querier_result
         .map_err(|err| {
@@ -582,7 +591,8 @@ pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
         .map_err(|err| StreamError::Anyhow(err.into()))?;
 
     // Fetch info for all nodes concurrently
-    let (querier_infos, ingestor_infos, indexer_infos) = future::join3(
+    let (prism_infos, querier_infos, ingestor_infos, indexer_infos) = future::join4(
+        fetch_nodes_info(prism_metadata),
         fetch_nodes_info(querier_metadata),
         fetch_nodes_info(ingestor_metadata),
         fetch_nodes_info(indexer_metadata),
@@ -591,6 +601,7 @@ pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
 
     // Combine results from all node types
     let mut infos = Vec::new();
+    infos.extend(prism_infos?);
     infos.extend(querier_infos?);
     infos.extend(ingestor_infos?);
     infos.extend(indexer_infos?);
@@ -887,13 +898,20 @@ where
 /// combines all metrics into a single vector
 async fn fetch_cluster_metrics() -> Result<Vec<Metrics>, PostError> {
     // Get ingestor and indexer metadata concurrently
-    let (querier_result, ingestor_result, indexer_result) = future::join3(
+    let (prism_result, querier_result, ingestor_result, indexer_result) = future::join4(
+        get_node_info(NodeType::Prism),
         get_node_info(NodeType::Querier),
         get_node_info(NodeType::Ingestor),
         get_node_info(NodeType::Indexer),
     )
     .await;
 
+    // Handle prism metadata result
+    let prism_metadata: Vec<NodeMetadata> = prism_result.map_err(|err| {
+        error!("Fatal: failed to get prism info: {:?}", err);
+        PostError::Invalid(err)
+    })?;
+
     // Handle querier metadata result
     let querier_metadata: Vec<NodeMetadata> = querier_result.map_err(|err| {
         error!("Fatal: failed to get querier info: {:?}", err);
@@ -910,7 +928,8 @@ async fn fetch_cluster_metrics() -> Result<Vec<Metrics>, PostError> {
         PostError::Invalid(err)
     })?;
     // Fetch metrics from ingestors and indexers concurrently
-    let (querier_metrics, ingestor_metrics, indexer_metrics) = future::join3(
+    let (prism_metrics, querier_metrics, ingestor_metrics, indexer_metrics) = future::join4(
+        fetch_nodes_metrics(prism_metadata),
         fetch_nodes_metrics(querier_metadata),
         fetch_nodes_metrics(ingestor_metadata),
         fetch_nodes_metrics(indexer_metadata),
@@ -920,6 +939,12 @@ async fn fetch_cluster_metrics() -> Result<Vec<Metrics>, PostError> {
     // Combine all metrics
     let mut all_metrics = Vec::new();
 
+    // Add prism metrics
+    match prism_metrics {
+        Ok(metrics) => all_metrics.extend(metrics),
+        Err(err) => return Err(err),
+    }
+
     // Add querier metrics
     match querier_metrics {
         Ok(metrics) => all_metrics.extend(metrics),

From ccf5ef2d3e5a0eb3e0d068807a726640b5df8c83 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Tue, 15 Apr 2025 02:19:52 -0400
Subject: [PATCH 36/42] convert node metadata to Arc

---
 src/handlers/http/modal/ingest_server.rs |  3 ++-
 src/handlers/http/modal/mod.rs           | 10 +++++-----
 src/handlers/http/modal/query_server.rs  |  3 ++-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/handlers/http/modal/ingest_server.rs b/src/handlers/http/modal/ingest_server.rs
index 65eb81467..e9751ee13 100644
--- a/src/handlers/http/modal/ingest_server.rs
+++ b/src/handlers/http/modal/ingest_server.rs
@@ -16,6 +16,7 @@
  *
  */
 
+use std::sync::Arc;
 use std::thread;
 
 use actix_web::web;
@@ -54,7 +55,7 @@ use super::{
 };
 
 pub const INGESTOR_EXPECT: &str = "Ingestor Metadata should be set in ingestor mode";
-pub static INGESTOR_META: OnceCell<IngestorMetadata> = OnceCell::const_new();
+pub static INGESTOR_META: OnceCell<Arc<IngestorMetadata>> = OnceCell::const_new();
 pub struct IngestServer;
 
 #[async_trait]
diff --git a/src/handlers/http/modal/mod.rs b/src/handlers/http/modal/mod.rs
index 29f6376a4..3504ef0fa 100644
--- a/src/handlers/http/modal/mod.rs
+++ b/src/handlers/http/modal/mod.rs
@@ -277,7 +277,7 @@ impl NodeMetadata {
         }
     }
 
-    pub async fn load_node_metadata(node_type: NodeType) -> anyhow::Result<Self> {
+    pub async fn load_node_metadata(node_type: NodeType) -> anyhow::Result<Arc<Self>> {
         let staging_path = PARSEABLE.options.staging_dir();
         let node_type_str = node_type.as_str();
 
@@ -310,7 +310,7 @@ impl NodeMetadata {
         mut meta: Self,
         staging_path: &Path,
         node_type: NodeType,
-    ) -> anyhow::Result<Self> {
+    ) -> anyhow::Result<Arc<Self>> {
         Self::update_metadata(&mut meta, &PARSEABLE.options, node_type);
         meta.put_on_disk(staging_path)
             .expect("Couldn't write updated metadata to disk");
@@ -320,11 +320,11 @@ impl NodeMetadata {
         let store = PARSEABLE.storage.get_object_store();
         store.put_object(&path, resource).await?;
 
-        Ok(meta)
+        Ok(Arc::new(meta))
     }
 
     /// Store new metadata
-    async fn store_new_metadata(meta: Self, staging_path: &Path) -> anyhow::Result<Self> {
+    async fn store_new_metadata(meta: Self, staging_path: &Path) -> anyhow::Result<Arc<Self>> {
         meta.put_on_disk(staging_path)
             .expect("Couldn't write new metadata to disk");
 
@@ -333,7 +333,7 @@ impl NodeMetadata {
         let store = PARSEABLE.storage.get_object_store();
         store.put_object(&path, resource).await?;
 
-        Ok(meta)
+        Ok(Arc::new(meta))
     }
 
     async fn load_from_storage(node_type: String) -> Vec<NodeMetadata> {
diff --git a/src/handlers/http/modal/query_server.rs b/src/handlers/http/modal/query_server.rs
index 614d9d425..b138a292c 100644
--- a/src/handlers/http/modal/query_server.rs
+++ b/src/handlers/http/modal/query_server.rs
@@ -16,6 +16,7 @@
  *
  */
 
+use std::sync::Arc;
 use std::thread;
 
 use crate::handlers::airplane;
@@ -42,7 +43,7 @@ use super::query::{querier_ingest, querier_logstream, querier_rbac, querier_role
 use super::{load_on_init, NodeType, OpenIdClient, ParseableServer, QuerierMetadata};
 
 pub struct QueryServer;
-pub static QUERIER_META: OnceCell<QuerierMetadata> = OnceCell::const_new();
+pub static QUERIER_META: OnceCell<Arc<QuerierMetadata>> = OnceCell::const_new();
 #[async_trait]
 impl ParseableServer for QueryServer {
     // configure the api routes

From a71731357bef0f842d383d9730874dbc62443638 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Tue, 15 Apr 2025 03:30:31 -0400
Subject: [PATCH 37/42] coderabbitai suggestions

---
 src/handlers/http/cluster/mod.rs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/handlers/http/cluster/mod.rs b/src/handlers/http/cluster/mod.rs
index a503c309d..64fe8dc1a 100644
--- a/src/handlers/http/cluster/mod.rs
+++ b/src/handlers/http/cluster/mod.rs
@@ -547,7 +547,7 @@ pub async fn send_retention_cleanup_request(
     Ok(first_event_at)
 }
 
-/// Fetches cluster information for all nodes (ingestor, indexer, and querier)
+/// Fetches cluster information for all nodes (ingestor, indexer, querier and prism)
 pub async fn get_cluster_info() -> Result<impl Responder, StreamError> {
     // Get querier, ingestor and indexer metadata concurrently
     let (prism_result, querier_result, ingestor_result, indexer_result) = future::join4(
@@ -703,7 +703,7 @@ pub async fn get_cluster_metrics() -> Result<impl Responder, PostError> {
 }
 
 /// get node info for a specific node type
-/// this is used to get the node info for ingestor, indexer and querier
+/// this is used to get the node info for ingestor, indexer, querier and prism
 /// it will return the metadata for all nodes of that type
 pub async fn get_node_info<T: Metadata + DeserializeOwned>(
     node_type: NodeType,
@@ -754,7 +754,11 @@ pub async fn remove_node(node_url: Path<String>) -> Result<impl Responder, PostE
         remove_node_metadata::<QuerierMetadata>(&object_store, &domain_name, NodeType::Querier)
             .await?;
 
-    if removed_ingestor || removed_indexer || removed_querier {
+    // Delete prism metadata
+    let removed_prism =
+        remove_node_metadata::<NodeMetadata>(&object_store, &domain_name, NodeType::Prism).await?;
+
+    if removed_ingestor || removed_indexer || removed_querier || removed_prism {
         return Ok((
             format!("node {} removed successfully", domain_name),
             StatusCode::OK,

From b043a757e4f7af9938cbfda6134b19fdfd605001 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Tue, 15 Apr 2025 07:52:13 -0400
Subject: [PATCH 38/42] ingestor analytics response check

---
 src/analytics.rs | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/analytics.rs b/src/analytics.rs
index b9b3e425f..9b8e6603e 100644
--- a/src/analytics.rs
+++ b/src/analytics.rs
@@ -287,10 +287,14 @@ async fn fetch_ingestors_metrics(
                 .send()
                 .await
                 .expect("should respond");
-
-            let data = serde_json::from_slice::<NodeMetrics>(&resp.bytes().await?)?;
-            vec.push(data);
-            active_ingestors += 1;
+            // check if the response is valid
+            if let Ok(data) = serde_json::from_slice::<NodeMetrics>(&resp.bytes().await?) {
+                active_ingestors += 1;
+                vec.push(data);
+            } else {
+                offline_ingestors += 1;
+                continue;
+            }
         }
 
         node_metrics.accumulate(&mut vec);

From 8af3a4cd94d86c0ee7ef570ccbbebee24acb6626 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Tue, 15 Apr 2025 08:44:28 -0400
Subject: [PATCH 39/42] remove redundant code

---
 src/handlers/http/cluster/mod.rs | 400 ++++++++++++++++---------------
 1 file changed, 203 insertions(+), 197 deletions(-)

diff --git a/src/handlers/http/cluster/mod.rs b/src/handlers/http/cluster/mod.rs
index 64fe8dc1a..8ea44debb 100644
--- a/src/handlers/http/cluster/mod.rs
+++ b/src/handlers/http/cluster/mod.rs
@@ -20,6 +20,7 @@ pub mod utils;
 
 use futures::{future, stream, StreamExt};
 use std::collections::HashSet;
+use std::future::Future;
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -64,6 +65,31 @@ pub const INTERNAL_STREAM_NAME: &str = "pmeta";
 
 const CLUSTER_METRICS_INTERVAL_SECONDS: Interval = clokwerk::Interval::Minutes(1);
 
+pub async fn for_each_live_ingestor<F, Fut, E>(api_fn: F) -> Result<(), E>
+where
+    F: Fn(NodeMetadata) -> Fut + Clone,
+    Fut: Future<Output = Result<(), E>>,
+    E: From<anyhow::Error>,
+{
+    let ingestor_infos: Vec<NodeMetadata> =
+        get_node_info(NodeType::Ingestor).await.map_err(|err| {
+            error!("Fatal: failed to get ingestor info: {:?}", err);
+            E::from(err)
+        })?;
+
+    // Process each live ingestor
+    for ingestor in ingestor_infos {
+        if !utils::check_liveness(&ingestor.domain_name).await {
+            warn!("Ingestor {} is not live", ingestor.domain_name);
+            continue;
+        }
+
+        api_fn(ingestor).await?;
+    }
+
+    Ok(())
+}
+
 // forward the create/update stream request to all ingestors to keep them in sync
 pub async fn sync_streams_with_ingestors(
     headers: HeaderMap,
@@ -75,70 +101,63 @@ pub async fn sync_streams_with_ingestors(
     for (key, value) in headers.iter() {
         reqwest_headers.insert(key.clone(), value.clone());
     }
-    let ingestor_infos: Vec<NodeMetadata> =
-        get_node_info(NodeType::Ingestor).await.map_err(|err| {
-            error!("Fatal: failed to get ingestor info: {:?}", err);
-            StreamError::Anyhow(err)
-        })?;
 
-    for ingestor in ingestor_infos {
-        if !utils::check_liveness(&ingestor.domain_name).await {
-            warn!("Ingestor {} is not live", ingestor.domain_name);
-            continue;
-        }
-        let url = format!(
-            "{}{}/logstream/{}/sync",
-            ingestor.domain_name,
-            base_path_without_preceding_slash(),
-            stream_name
-        );
-        let res = HTTP_CLIENT
-            .put(url)
-            .headers(reqwest_headers.clone())
-            .header(header::AUTHORIZATION, &ingestor.token)
-            .body(body.clone())
-            .send()
-            .await
-            .map_err(|err| {
-                error!(
-                    "Fatal: failed to forward upsert stream request to ingestor: {}\n Error: {:?}",
-                    ingestor.domain_name, err
-                );
-                StreamError::Network(err)
-            })?;
+    let body_clone = body.clone();
+    let stream_name = stream_name.to_string();
+    let reqwest_headers_clone = reqwest_headers.clone();
 
-        if !res.status().is_success() {
-            error!(
-                "failed to forward upsert stream request to ingestor: {}\nResponse Returned: {:?}",
+    for_each_live_ingestor(
+        move |ingestor| {
+            let url = format!(
+                "{}{}/logstream/{}/sync",
                 ingestor.domain_name,
-                res.text().await
+                base_path_without_preceding_slash(),
+                stream_name
             );
+            let headers = reqwest_headers_clone.clone();
+            let body = body_clone.clone();
+            async move {
+                let res = HTTP_CLIENT
+                    .put(url)
+                    .headers(headers)
+                    .header(header::AUTHORIZATION, &ingestor.token)
+                    .body(body)
+                    .send()
+                    .await
+                    .map_err(|err| {
+                        error!(
+                            "Fatal: failed to forward upsert stream request to ingestor: {}\n Error: {:?}",
+                            ingestor.domain_name, err
+                        );
+                        StreamError::Network(err)
+                    })?;
+
+                if !res.status().is_success() {
+                    error!(
+                        "failed to forward upsert stream request to ingestor: {}\nResponse Returned: {:?}",
+                        ingestor.domain_name,
+                        res.text().await
+                    );
+                }
+                Ok(())
+            }
         }
-    }
-
-    Ok(())
+    ).await
 }
 
 // forward the role update request to all ingestors to keep them in sync
 pub async fn sync_users_with_roles_with_ingestors(
-    username: &String,
+    username: &str,
     role: &HashSet<String>,
 ) -> Result<(), RBACError> {
-    let ingestor_infos: Vec<NodeMetadata> =
-        get_node_info(NodeType::Ingestor).await.map_err(|err| {
-            error!("Fatal: failed to get ingestor info: {:?}", err);
-            RBACError::Anyhow(err)
-        })?;
-
-    let role = to_vec(&role.clone()).map_err(|err| {
+    let role_data = to_vec(&role.clone()).map_err(|err| {
         error!("Fatal: failed to serialize role: {:?}", err);
         RBACError::SerdeError(err)
     })?;
-    for ingestor in ingestor_infos.iter() {
-        if !utils::check_liveness(&ingestor.domain_name).await {
-            warn!("Ingestor {} is not live", ingestor.domain_name);
-            continue;
-        }
+
+    let username = username.to_owned();
+
+    for_each_live_ingestor(move |ingestor| {
         let url = format!(
             "{}{}/user/{}/role/sync",
             ingestor.domain_name,
@@ -146,46 +165,43 @@ pub async fn sync_users_with_roles_with_ingestors(
             username
         );
 
-        let res = HTTP_CLIENT
-            .put(url)
-            .header(header::AUTHORIZATION, &ingestor.token)
-            .header(header::CONTENT_TYPE, "application/json")
-            .body(role.clone())
-            .send()
-            .await
-            .map_err(|err| {
+        let role_data = role_data.clone();
+
+        async move {
+            let res = HTTP_CLIENT
+                .put(url)
+                .header(header::AUTHORIZATION, &ingestor.token)
+                .header(header::CONTENT_TYPE, "application/json")
+                .body(role_data)
+                .send()
+                .await
+                .map_err(|err| {
+                    error!(
+                        "Fatal: failed to forward request to ingestor: {}\n Error: {:?}",
+                        ingestor.domain_name, err
+                    );
+                    RBACError::Network(err)
+                })?;
+
+            if !res.status().is_success() {
                 error!(
-                    "Fatal: failed to forward request to ingestor: {}\n Error: {:?}",
-                    ingestor.domain_name, err
+                    "failed to forward request to ingestor: {}\nResponse Returned: {:?}",
+                    ingestor.domain_name,
+                    res.text().await
                 );
-                RBACError::Network(err)
-            })?;
+            }
 
-        if !res.status().is_success() {
-            error!(
-                "failed to forward request to ingestor: {}\nResponse Returned: {:?}",
-                ingestor.domain_name,
-                res.text().await
-            );
+            Ok(())
         }
-    }
-
-    Ok(())
+    })
+    .await
 }
 
 // forward the delete user request to all ingestors to keep them in sync
-pub async fn sync_user_deletion_with_ingestors(username: &String) -> Result<(), RBACError> {
-    let ingestor_infos: Vec<NodeMetadata> =
-        get_node_info(NodeType::Ingestor).await.map_err(|err| {
-            error!("Fatal: failed to get ingestor info: {:?}", err);
-            RBACError::Anyhow(err)
-        })?;
+pub async fn sync_user_deletion_with_ingestors(username: &str) -> Result<(), RBACError> {
+    let username = username.to_owned();
 
-    for ingestor in ingestor_infos.iter() {
-        if !utils::check_liveness(&ingestor.domain_name).await {
-            warn!("Ingestor {} is not live", ingestor.domain_name);
-            continue;
-        }
+    for_each_live_ingestor(move |ingestor| {
         let url = format!(
             "{}{}/user/{}/sync",
             ingestor.domain_name,
@@ -193,29 +209,32 @@ pub async fn sync_user_deletion_with_ingestors(username: &String) -> Result<(),
             username
         );
 
-        let res = HTTP_CLIENT
-            .delete(url)
-            .header(header::AUTHORIZATION, &ingestor.token)
-            .send()
-            .await
-            .map_err(|err| {
+        async move {
+            let res = HTTP_CLIENT
+                .delete(url)
+                .header(header::AUTHORIZATION, &ingestor.token)
+                .send()
+                .await
+                .map_err(|err| {
+                    error!(
+                        "Fatal: failed to forward request to ingestor: {}\n Error: {:?}",
+                        ingestor.domain_name, err
+                    );
+                    RBACError::Network(err)
+                })?;
+
+            if !res.status().is_success() {
                 error!(
-                    "Fatal: failed to forward request to ingestor: {}\n Error: {:?}",
-                    ingestor.domain_name, err
+                    "failed to forward request to ingestor: {}\nResponse Returned: {:?}",
+                    ingestor.domain_name,
+                    res.text().await
                 );
-                RBACError::Network(err)
-            })?;
+            }
 
-        if !res.status().is_success() {
-            error!(
-                "failed to forward request to ingestor: {}\nResponse Returned: {:?}",
-                ingestor.domain_name,
-                res.text().await
-            );
+            Ok(())
         }
-    }
-
-    Ok(())
+    })
+    .await
 }
 
 // forward the create user request to all ingestors to keep them in sync
@@ -223,12 +242,6 @@ pub async fn sync_user_creation_with_ingestors(
     user: User,
     role: &Option<HashSet<String>>,
 ) -> Result<(), RBACError> {
-    let ingestor_infos: Vec<NodeMetadata> =
-        get_node_info(NodeType::Ingestor).await.map_err(|err| {
-            error!("Fatal: failed to get ingestor info: {:?}", err);
-            RBACError::Anyhow(err)
-        })?;
-
     let mut user = user.clone();
 
     if let Some(role) = role {
@@ -236,16 +249,14 @@ pub async fn sync_user_creation_with_ingestors(
     }
     let username = user.username();
 
-    let user = to_vec(&user).map_err(|err| {
+    let user_data = to_vec(&user).map_err(|err| {
         error!("Fatal: failed to serialize user: {:?}", err);
         RBACError::SerdeError(err)
     })?;
 
-    for ingestor in ingestor_infos.iter() {
-        if !utils::check_liveness(&ingestor.domain_name).await {
-            warn!("Ingestor {} is not live", ingestor.domain_name);
-            continue;
-        }
+    let username = username.to_string();
+
+    for_each_live_ingestor(move |ingestor| {
         let url = format!(
             "{}{}/user/{}/sync",
             ingestor.domain_name,
@@ -253,46 +264,43 @@ pub async fn sync_user_creation_with_ingestors(
             username
         );
 
-        let res = HTTP_CLIENT
-            .post(url)
-            .header(header::AUTHORIZATION, &ingestor.token)
-            .header(header::CONTENT_TYPE, "application/json")
-            .body(user.clone())
-            .send()
-            .await
-            .map_err(|err| {
+        let user_data = user_data.clone();
+
+        async move {
+            let res = HTTP_CLIENT
+                .post(url)
+                .header(header::AUTHORIZATION, &ingestor.token)
+                .header(header::CONTENT_TYPE, "application/json")
+                .body(user_data)
+                .send()
+                .await
+                .map_err(|err| {
+                    error!(
+                        "Fatal: failed to forward request to ingestor: {}\n Error: {:?}",
+                        ingestor.domain_name, err
+                    );
+                    RBACError::Network(err)
+                })?;
+
+            if !res.status().is_success() {
                 error!(
-                    "Fatal: failed to forward request to ingestor: {}\n Error: {:?}",
-                    ingestor.domain_name, err
+                    "failed to forward request to ingestor: {}\nResponse Returned: {:?}",
+                    ingestor.domain_name,
+                    res.text().await
                 );
-                RBACError::Network(err)
-            })?;
+            }
 
-        if !res.status().is_success() {
-            error!(
-                "failed to forward request to ingestor: {}\nResponse Returned: {:?}",
-                ingestor.domain_name,
-                res.text().await
-            );
+            Ok(())
         }
-    }
-
-    Ok(())
+    })
+    .await
 }
 
 // forward the password reset request to all ingestors to keep them in sync
-pub async fn sync_password_reset_with_ingestors(username: &String) -> Result<(), RBACError> {
-    let ingestor_infos: Vec<NodeMetadata> =
-        get_node_info(NodeType::Ingestor).await.map_err(|err| {
-            error!("Fatal: failed to get ingestor info: {:?}", err);
-            RBACError::Anyhow(err)
-        })?;
+pub async fn sync_password_reset_with_ingestors(username: &str) -> Result<(), RBACError> {
+    let username = username.to_owned();
 
-    for ingestor in ingestor_infos.iter() {
-        if !utils::check_liveness(&ingestor.domain_name).await {
-            warn!("Ingestor {} is not live", ingestor.domain_name);
-            continue;
-        }
+    for_each_live_ingestor(move |ingestor| {
         let url = format!(
             "{}{}/user/{}/generate-new-password/sync",
             ingestor.domain_name,
@@ -300,30 +308,33 @@ pub async fn sync_password_reset_with_ingestors(username: &String) -> Result<(),
             username
         );
 
-        let res = HTTP_CLIENT
-            .post(url)
-            .header(header::AUTHORIZATION, &ingestor.token)
-            .header(header::CONTENT_TYPE, "application/json")
-            .send()
-            .await
-            .map_err(|err| {
+        async move {
+            let res = HTTP_CLIENT
+                .post(url)
+                .header(header::AUTHORIZATION, &ingestor.token)
+                .header(header::CONTENT_TYPE, "application/json")
+                .send()
+                .await
+                .map_err(|err| {
+                    error!(
+                        "Fatal: failed to forward request to ingestor: {}\n Error: {:?}",
+                        ingestor.domain_name, err
+                    );
+                    RBACError::Network(err)
+                })?;
+
+            if !res.status().is_success() {
                 error!(
-                    "Fatal: failed to forward request to ingestor: {}\n Error: {:?}",
-                    ingestor.domain_name, err
+                    "failed to forward request to ingestor: {}\nResponse Returned: {:?}",
+                    ingestor.domain_name,
+                    res.text().await
                 );
-                RBACError::Network(err)
-            })?;
+            }
 
-        if !res.status().is_success() {
-            error!(
-                "failed to forward request to ingestor: {}\nResponse Returned: {:?}",
-                ingestor.domain_name,
-                res.text().await
-            );
+            Ok(())
         }
-    }
-
-    Ok(())
+    })
+    .await
 }
 
 // forward the put role request to all ingestors to keep them in sync
@@ -331,17 +342,7 @@ pub async fn sync_role_update_with_ingestors(
     name: String,
     privileges: Vec<DefaultPrivilege>,
 ) -> Result<(), RoleError> {
-    let ingestor_infos: Vec<NodeMetadata> =
-        get_node_info(NodeType::Ingestor).await.map_err(|err| {
-            error!("Fatal: failed to get ingestor info: {:?}", err);
-            RoleError::Anyhow(err)
-        })?;
-
-    for ingestor in ingestor_infos.iter() {
-        if !utils::check_liveness(&ingestor.domain_name).await {
-            warn!("Ingestor {} is not live", ingestor.domain_name);
-            continue;
-        }
+    for_each_live_ingestor(move |ingestor| {
         let url = format!(
             "{}{}/role/{}/sync",
             ingestor.domain_name,
@@ -349,31 +350,36 @@ pub async fn sync_role_update_with_ingestors(
             name
         );
 
-        let res = HTTP_CLIENT
-            .put(url)
-            .header(header::AUTHORIZATION, &ingestor.token)
-            .header(header::CONTENT_TYPE, "application/json")
-            .json(&privileges)
-            .send()
-            .await
-            .map_err(|err| {
+        let privileges = privileges.clone();
+
+        async move {
+            let res = HTTP_CLIENT
+                .put(url)
+                .header(header::AUTHORIZATION, &ingestor.token)
+                .header(header::CONTENT_TYPE, "application/json")
+                .json(&privileges)
+                .send()
+                .await
+                .map_err(|err| {
+                    error!(
+                        "Fatal: failed to forward request to ingestor: {}\n Error: {:?}",
+                        ingestor.domain_name, err
+                    );
+                    RoleError::Network(err)
+                })?;
+
+            if !res.status().is_success() {
                 error!(
-                    "Fatal: failed to forward request to ingestor: {}\n Error: {:?}",
-                    ingestor.domain_name, err
+                    "failed to forward request to ingestor: {}\nResponse Returned: {:?}",
+                    ingestor.domain_name,
+                    res.text().await
                 );
-                RoleError::Network(err)
-            })?;
+            }
 
-        if !res.status().is_success() {
-            error!(
-                "failed to forward request to ingestor: {}\nResponse Returned: {:?}",
-                ingestor.domain_name,
-                res.text().await
-            );
+            Ok(())
         }
-    }
-
-    Ok(())
+    })
+    .await
 }
 
 pub fn fetch_daily_stats_from_ingestors(

From 1841a682d31c65738efa88bff4c78cfcf9e247cd Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Tue, 15 Apr 2025 08:54:50 -0400
Subject: [PATCH 40/42] process live ingestors in paralllel

---
 src/handlers/http/cluster/mod.rs | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/src/handlers/http/cluster/mod.rs b/src/handlers/http/cluster/mod.rs
index 8ea44debb..1de62919f 100644
--- a/src/handlers/http/cluster/mod.rs
+++ b/src/handlers/http/cluster/mod.rs
@@ -67,9 +67,9 @@ const CLUSTER_METRICS_INTERVAL_SECONDS: Interval = clokwerk::Interval::Minutes(1
 
 pub async fn for_each_live_ingestor<F, Fut, E>(api_fn: F) -> Result<(), E>
 where
-    F: Fn(NodeMetadata) -> Fut + Clone,
-    Fut: Future<Output = Result<(), E>>,
-    E: From<anyhow::Error>,
+    F: Fn(NodeMetadata) -> Fut + Clone + Send + Sync + 'static,
+    Fut: Future<Output = Result<(), E>> + Send,
+    E: From<anyhow::Error> + Send + Sync + 'static,
 {
     let ingestor_infos: Vec<NodeMetadata> =
         get_node_info(NodeType::Ingestor).await.map_err(|err| {
@@ -77,14 +77,25 @@ where
             E::from(err)
         })?;
 
-    // Process each live ingestor
+    let mut live_ingestors = Vec::new();
     for ingestor in ingestor_infos {
-        if !utils::check_liveness(&ingestor.domain_name).await {
+        if utils::check_liveness(&ingestor.domain_name).await {
+            live_ingestors.push(ingestor);
+        } else {
             warn!("Ingestor {} is not live", ingestor.domain_name);
-            continue;
         }
+    }
+
+    // Process all live ingestors in parallel
+    let results = futures::future::join_all(live_ingestors.into_iter().map(|ingestor| {
+        let api_fn = api_fn.clone();
+        async move { api_fn(ingestor).await }
+    }))
+    .await;
 
-        api_fn(ingestor).await?;
+    // collect results
+    for result in results {
+        result?;
     }
 
     Ok(())

From 04aade9b467b2d8d6b4237918400ea183659f33c Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <131262146+nikhilsinhaparseable@users.noreply.github.com>
Date: Tue, 15 Apr 2025 23:13:35 +0530
Subject: [PATCH 41/42] Update src/handlers/http/cluster/mod.rs

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Signed-off-by: Nikhil Sinha <131262146+nikhilsinhaparseable@users.noreply.github.com>
---
 src/handlers/http/cluster/mod.rs | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/handlers/http/cluster/mod.rs b/src/handlers/http/cluster/mod.rs
index 1de62919f..038a1654b 100644
--- a/src/handlers/http/cluster/mod.rs
+++ b/src/handlers/http/cluster/mod.rs
@@ -736,7 +736,15 @@ pub async fn get_node_info<T: Metadata + DeserializeOwned>(
         )
         .await?
         .iter()
-        .filter_map(|x| serde_json::from_slice::<T>(x).ok())
+        .filter_map(|x| {
+            match serde_json::from_slice::<T>(x) {
+                Ok(val) => Some(val),
+                Err(e) => {
+                    error!("Failed to parse node metadata: {:?}", e);
+                    None
+                }
+            }
+        })
         .collect();
 
     Ok(metadata)

From 891e38cded0a944626806794c4ef6119e448b250 Mon Sep 17 00:00:00 2001
From: Nikhil Sinha <nikhil.sinha@parseable.com>
Date: Wed, 16 Apr 2025 02:08:08 -0400
Subject: [PATCH 42/42] rename queries to queriers

---
 src/analytics.rs | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/analytics.rs b/src/analytics.rs
index 9b8e6603e..329a20632 100644
--- a/src/analytics.rs
+++ b/src/analytics.rs
@@ -77,8 +77,8 @@ pub struct Report {
     inactive_ingestors: u64,
     active_indexers: u64,
     inactive_indexers: u64,
-    active_queries: u64,
-    inactive_queries: u64,
+    active_queriers: u64,
+    inactive_queriers: u64,
     stream_count: usize,
     total_events_count: u64,
     total_json_bytes: u64,
@@ -113,8 +113,8 @@ impl Report {
         let ingestor_metrics = fetch_ingestors_metrics().await?;
         let mut active_indexers = 0;
         let mut inactive_indexers = 0;
-        let mut active_queries = 0;
-        let mut inactive_queries = 0;
+        let mut active_queriers = 0;
+        let mut inactive_queriers = 0;
 
         // check liveness of indexers
         // get the count of active and inactive indexers
@@ -132,9 +132,9 @@ impl Report {
         let query_infos: Vec<NodeMetadata> = cluster::get_node_info(NodeType::Querier).await?;
         for query in query_infos {
             if check_liveness(&query.domain_name).await {
-                active_queries += 1;
+                active_queriers += 1;
             } else {
-                inactive_queries += 1;
+                inactive_queriers += 1;
             }
         }
         Ok(Self {
@@ -154,8 +154,8 @@ impl Report {
             inactive_ingestors: ingestor_metrics.1,
             active_indexers,
             inactive_indexers,
-            active_queries,
-            inactive_queries,
+            active_queriers,
+            inactive_queriers,
             stream_count: ingestor_metrics.2,
             total_events_count: ingestor_metrics.3,
             total_json_bytes: ingestor_metrics.4,