bugfix and alerts retry

Author: parmesant

src/alerts/mod.rs

@@ -659,7 +659,7 @@ impl AlertConfig {
 
                 // validate condition config
                 let agg1 = &self.aggregates.aggregate_config[0];
-                let agg2 = &self.aggregates.aggregate_config[0];
+                let agg2 = &self.aggregates.aggregate_config[1];
 
                 validate_condition_config(&agg1.conditions)?;
                 validate_condition_config(&agg2.conditions)?;

src/alerts/target.rs

@@ -29,6 +29,7 @@ use http::{header::AUTHORIZATION, HeaderMap, HeaderValue};
 use humantime_serde::re::humantime;
 use reqwest::ClientBuilder;
 use tracing::{error, trace, warn};
+use url::Url;
 
 use super::ALERTS;
 
@@ -255,9 +256,9 @@ fn default_client_builder() -> ClientBuilder {
     ClientBuilder::new()
 }
 
-#[derive(Default, Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 pub struct SlackWebHook {
-    endpoint: String,
+    endpoint: Url,
 }
 
 #[async_trait]
@@ -279,7 +280,7 @@ impl CallableTarget for SlackWebHook {
             }
         };
 
-        if let Err(e) = client.post(&self.endpoint).json(&alert).send().await {
+        if let Err(e) = client.post(self.endpoint.clone()).json(&alert).send().await {
             error!("Couldn't make call to webhook, error: {}", e)
         }
     }
@@ -288,7 +289,7 @@ impl CallableTarget for SlackWebHook {
 #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct OtherWebHook {
-    endpoint: String,
+    endpoint: Url,
     #[serde(default)]
     headers: HashMap<String, String>,
     #[serde(default)]
@@ -314,7 +315,7 @@ impl CallableTarget for OtherWebHook {
         };
 
         let request = client
-            .post(&self.endpoint)
+            .post(self.endpoint.clone())
             .headers((&self.headers).try_into().expect("valid_headers"));
 
         if let Err(e) = request.body(alert).send().await {
@@ -326,7 +327,7 @@ impl CallableTarget for OtherWebHook {
 #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct AlertManager {
-    endpoint: String,
+    endpoint: Url,
     #[serde(default)]
     skip_tls_check: bool,
     #[serde(flatten)]
@@ -404,7 +405,7 @@ impl CallableTarget for AlertManager {
             }
         };
 
-        if let Err(e) = client.post(&self.endpoint).json(&alerts).send().await {
+        if let Err(e) = client.post(self.endpoint.clone()).json(&alerts).send().await {
             error!("Couldn't make call to alertmanager, error: {}", e)
         }
     }

src/sync.rs

@@ -246,16 +246,25 @@ pub async fn alert_runtime(mut rx: mpsc::Receiver<AlertTask>) -> Result<(), anyh
                 let alert = alert.clone();
                 let id = alert.id;
                 let handle = tokio::spawn(async move {
+                    let mut retry_counter = 0;
+                    let mut sleep_duration = alert.get_eval_frequency();
                     loop {
                         match alerts_utils::evaluate_alert(&alert).await {
-                            Ok(_) => {}
+                            Ok(_) => {
+                                retry_counter = 0;
+                            }
                             Err(err) => {
-                                error!("Error while evaluation- {err}");
-                                break;
+                                warn!("Error while evaluation- {}\nRetrying after sleeping for 1 minute", err);
+                                sleep_duration = 1;
+                                retry_counter += 1;
+
+                                if retry_counter > 3 {
+                                    error!("Alert with id {} failed to evaluate after 3 retries with err- {}", id, err);
+                                    break;
+                                }
                             }
                         }
-                        tokio::time::sleep(Duration::from_secs(alert.get_eval_frequency() * 60))
-                            .await;
+                        tokio::time::sleep(Duration::from_secs(sleep_duration * 60)).await;
                     }
                 });