fix: switched to tokio tasks (#862)

This PR updates the architecture to use tokio tasks 
instead of threads. This will allow better resource 
utilisation and performance. 

Refer: https://docs.rs/tokio/latest/tokio/task/index.html

Rewrote object_store_sync() & run_local_sync() in server's 
sync.rs to use tokio tasks but with same function signature. 
Updated the invoking code to match.

Rewrote init_scheduler() in server's storage/retention.rs to 
use tokio tasks but with same function signature. 
Updated the invoking code to match.

---------

Signed-off-by: Vishal <[email protected]>
Co-authored-by: Nitish Tiwari <[email protected]>

Author: vishalkrishnads

server/src/handlers/http/modal/ingest_server.rs

@@ -339,9 +339,10 @@ impl IngestServer {
 
         migration::run_migration(&CONFIG).await?;
 
-        let (localsync_handler, mut localsync_outbox, localsync_inbox) = sync::run_local_sync();
+        let (localsync_handler, mut localsync_outbox, localsync_inbox) =
+            sync::run_local_sync().await;
         let (mut remote_sync_handler, mut remote_sync_outbox, mut remote_sync_inbox) =
-            sync::object_store_sync();
+            sync::object_store_sync().await;
 
         tokio::spawn(airplane::server());
 
@@ -354,8 +355,12 @@ impl IngestServer {
                     // actix server finished .. stop other threads and stop the server
                     remote_sync_inbox.send(()).unwrap_or(());
                     localsync_inbox.send(()).unwrap_or(());
-                    localsync_handler.join().unwrap_or(());
-                    remote_sync_handler.join().unwrap_or(());
+                    if let Err(e) = localsync_handler.await {
+                        log::error!("Error joining remote_sync_handler: {:?}", e);
+                    }
+                    if let Err(e) = remote_sync_handler.await {
+                        log::error!("Error joining remote_sync_handler: {:?}", e);
+                    }
                     return e
                 },
                 _ = &mut localsync_outbox => {
@@ -365,8 +370,10 @@ impl IngestServer {
                 },
                 _ = &mut remote_sync_outbox => {
                     // remote_sync failed, this is recoverable by just starting remote_sync thread again
-                    remote_sync_handler.join().unwrap_or(());
-                    (remote_sync_handler, remote_sync_outbox, remote_sync_inbox) = sync::object_store_sync();
+                    if let Err(e) = remote_sync_handler.await {
+                        log::error!("Error joining remote_sync_handler: {:?}", e);
+                    }
+                    (remote_sync_handler, remote_sync_outbox, remote_sync_inbox) = sync::object_store_sync().await;
                 }
 
             };

server/src/handlers/http/modal/query_server.rs

@@ -105,8 +105,8 @@ impl ParseableServer for QueryServer {
     fn validate(&self) -> anyhow::Result<()> {
         if CONFIG.get_storage_mode_string() == "Local drive" {
             return Err(anyhow::anyhow!(
-                "Query Server cannot be started in local storage mode. Please start the server in a supported storage mode.",
-            ));
+                 "Query Server cannot be started in local storage mode. Please start the server in a supported storage mode.",
+             ));
         }
 
         Ok(())
@@ -188,13 +188,13 @@ impl QueryServer {
         if matches!(init_cluster_metrics_schedular(), Ok(())) {
             log::info!("Cluster metrics scheduler started successfully");
         }
-
         if let Some(hot_tier_manager) = HotTierManager::global() {
             hot_tier_manager.download_from_s3()?;
         };
-        let (localsync_handler, mut localsync_outbox, localsync_inbox) = sync::run_local_sync();
+        let (localsync_handler, mut localsync_outbox, localsync_inbox) =
+            sync::run_local_sync().await;
         let (mut remote_sync_handler, mut remote_sync_outbox, mut remote_sync_inbox) =
-            sync::object_store_sync();
+            sync::object_store_sync().await;
 
         tokio::spawn(airplane::server());
         let app = self.start(prometheus, CONFIG.parseable.openid.clone());
@@ -206,8 +206,12 @@ impl QueryServer {
                     // actix server finished .. stop other threads and stop the server
                     remote_sync_inbox.send(()).unwrap_or(());
                     localsync_inbox.send(()).unwrap_or(());
-                    localsync_handler.join().unwrap_or(());
-                    remote_sync_handler.join().unwrap_or(());
+                    if let Err(e) = localsync_handler.await {
+                        log::error!("Error joining localsync_handler: {:?}", e);
+                    }
+                    if let Err(e) = remote_sync_handler.await {
+                        log::error!("Error joining remote_sync_handler: {:?}", e);
+                    }
                     return e
                 },
                 _ = &mut localsync_outbox => {
@@ -217,8 +221,10 @@ impl QueryServer {
                 },
                 _ = &mut remote_sync_outbox => {
                     // remote_sync failed, this is recoverable by just starting remote_sync thread again
-                    remote_sync_handler.join().unwrap_or(());
-                    (remote_sync_handler, remote_sync_outbox, remote_sync_inbox) = sync::object_store_sync();
+                    if let Err(e) = remote_sync_handler.await {
+                        log::error!("Error joining remote_sync_handler: {:?}", e);
+                    }
+                    (remote_sync_handler, remote_sync_outbox, remote_sync_inbox) = sync::object_store_sync().await;
                 }
 
             };

server/src/handlers/http/modal/server.rs

@@ -533,9 +533,10 @@ impl Server {
 
         storage::retention::load_retention_from_global();
 
-        let (localsync_handler, mut localsync_outbox, localsync_inbox) = sync::run_local_sync();
+        let (localsync_handler, mut localsync_outbox, localsync_inbox) =
+            sync::run_local_sync().await;
         let (mut remote_sync_handler, mut remote_sync_outbox, mut remote_sync_inbox) =
-            sync::object_store_sync();
+            sync::object_store_sync().await;
 
         if CONFIG.parseable.send_analytics {
             analytics::init_analytics_scheduler()?;
@@ -553,8 +554,12 @@ impl Server {
                     // actix server finished .. stop other threads and stop the server
                     remote_sync_inbox.send(()).unwrap_or(());
                     localsync_inbox.send(()).unwrap_or(());
-                    localsync_handler.join().unwrap_or(());
-                    remote_sync_handler.join().unwrap_or(());
+                    if let Err(e) = localsync_handler.await {
+                        log::error!("Error joining remote_sync_handler: {:?}", e);
+                    }
+                    if let Err(e) = remote_sync_handler.await {
+                        log::error!("Error joining remote_sync_handler: {:?}", e);
+                    }
                     return e
                 },
                 _ = &mut localsync_outbox => {
@@ -564,8 +569,10 @@ impl Server {
                 },
                 _ = &mut remote_sync_outbox => {
                     // remote_sync failed, this is recoverable by just starting remote_sync thread again
-                    remote_sync_handler.join().unwrap_or(());
-                    (remote_sync_handler, remote_sync_outbox, remote_sync_inbox) = sync::object_store_sync();
+                    if let Err(e) = remote_sync_handler.await {
+                        log::error!("Error joining remote_sync_handler: {:?}", e);
+                    }
+                    (remote_sync_handler, remote_sync_outbox, remote_sync_inbox) = sync::object_store_sync().await;
                 }
             };
         }

server/src/storage/retention.rs

@@ -19,36 +19,28 @@
 use std::hash::Hash;
 use std::num::NonZeroU32;
 use std::sync::Mutex;
-use std::thread;
 use std::time::Duration;
 
 use clokwerk::AsyncScheduler;
 use clokwerk::Job;
 use clokwerk::TimeUnits;
 use derive_more::Display;
 use once_cell::sync::Lazy;
+use tokio::task::JoinHandle;
 
 use crate::metadata::STREAM_INFO;
 
-type SchedulerHandle = thread::JoinHandle<()>;
+type SchedulerHandle = JoinHandle<()>;
 
 static SCHEDULER_HANDLER: Lazy<Mutex<Option<SchedulerHandle>>> = Lazy::new(|| Mutex::new(None));
 
-fn async_runtime() -> tokio::runtime::Runtime {
-    tokio::runtime::Builder::new_current_thread()
-        .thread_name("retention-task-thread")
-        .enable_all()
-        .build()
-        .unwrap()
-}
-
 pub fn load_retention_from_global() {
     log::info!("loading retention for all streams");
     init_scheduler();
 }
 
 pub fn init_scheduler() {
-    log::info!("Setting up schedular");
+    log::info!("Setting up scheduler");
     let mut scheduler = AsyncScheduler::new();
     let func = move || async {
         //get retention every day at 12 am
@@ -57,22 +49,17 @@ pub fn init_scheduler() {
 
             match retention {
                 Ok(config) => {
-                    if config.is_none() {
-                        continue;
-                    }
-                    for Task { action, days, .. } in config.unwrap().tasks.into_iter() {
-                        match action {
-                            Action::Delete => {
-                                let stream = stream.to_string();
-                                thread::spawn(move || {
-                                    let rt = tokio::runtime::Runtime::new().unwrap();
-                                    rt.block_on(async {
-                                        // Run the asynchronous delete action
+                    if let Some(config) = config {
+                        for Task { action, days, .. } in config.tasks.into_iter() {
+                            match action {
+                                Action::Delete => {
+                                    let stream = stream.to_string();
+                                    tokio::spawn(async move {
                                         action::delete(stream.clone(), u32::from(days)).await;
                                     });
-                                });
-                            }
-                        };
+                                }
+                            };
+                        }
                     }
                 }
                 Err(err) => {
@@ -83,21 +70,17 @@ pub fn init_scheduler() {
     };
 
     // Execute once on startup
-    thread::spawn(move || {
-        let rt = async_runtime();
-        rt.block_on(func());
+    tokio::spawn(async move {
+        func().await;
     });
 
     scheduler.every(1.day()).at("00:00").run(func);
 
-    let scheduler_handler = thread::spawn(|| {
-        let rt = async_runtime();
-        rt.block_on(async move {
-            loop {
-                tokio::time::sleep(Duration::from_secs(10)).await;
-                scheduler.run_pending().await;
-            }
-        });
+    let scheduler_handler = tokio::spawn(async move {
+        loop {
+            tokio::time::sleep(Duration::from_secs(10)).await;
+            scheduler.run_pending().await;
+        }
     });
 
     *SCHEDULER_HANDLER.lock().unwrap() = Some(scheduler_handler);