async

Author: hackermondev

Cargo.lock

@@ -1,13 +1,16 @@
 [workspace]
+resolver = "2"
 members = [
     "src/security_api",
     "src/security_poller",
     "src/security_discord"
 ]
 
 [workspace.dependencies]
+anyhow = "1.0.93"
 tokio = { version = "1.38.0", features = ["full"] }
 serde = { version = "1.0.163", features = ["derive"] }
 serde_json = "1.0.96"
 log = "0.4.17"
-pretty_env_logger = "0.5.0"
+pretty_env_logger = "0.5.0"
+reqwest = { version = "0.12.9", default-features = false, features = ["json", "http2", "rustls-tls", "rustls-tls-webpki-roots"] }

Cargo.toml

@@ -1,4 +1,4 @@
-FROM rust:1.75 as builder
+FROM rust:1.82 as builder
 WORKDIR /usr/src/security
 
 COPY . .

Dockerfile

@@ -7,5 +7,5 @@ poller:
   handle: "" # HackerOne team handle
   session_token: "" # HackerOne session token (the "__Host-session" cookie), this is optional if you're using a public team
 
-  ### disable_reputation_polling: "true"
-  ### disable_hackactivity_polling: "true"
+  ### reputation_polling: "true"
+  ### hackactivity_polling: "true"

config.example.yaml

@@ -7,10 +7,12 @@ edition = "2021"
 
 [dependencies]
 graphql_client = "0.12.0"
-reqwest = { version = "0.11.18", features = ["blocking", "json"] }
+reqwest.workspace = true
 regex = "1.4.2"
 serde = "1.0.163"
 serde_json = "1.0.96"
 redis = "0.23.0"
 chrono = "0.4.24"
 nanoid = "0.4.0"
+anyhow.workspace = true
+deadpool-redis = "0.18.0"

src/security_api/Cargo.toml

@@ -1,8 +1,6 @@
 use graphql_client::GraphQLQuery;
 use regex::Regex;
-use reqwest::blocking::{Client, ClientBuilder};
-use reqwest::header;
-use std::error::Error;
+use reqwest::{header, Client, ClientBuilder};
 use std::time::Duration;
 
 #[derive(Clone)]
@@ -29,6 +27,7 @@ impl HackerOneClient {
             .user_agent("HackerOneTracker (+github.com/hackermondev/hackerone-tracker)")
             .default_headers(headers)
             .connect_timeout(Duration::from_secs(5))
+            .http2_prior_knowledge()
             .build()
             .unwrap();
 
@@ -49,25 +48,26 @@ fn extract_csrf_token(html: &str) -> Option<String> {
     None
 }
 
-pub fn get_hackerone_csrf_token(session_token: &str) -> Result<String, Box<dyn Error>> {
+pub async fn fetch_csrf_token(session_token: &str) -> Result<String, anyhow::Error> {
     let client = Client::new();
     let http_response = client
         .get("https://hackerone.com/bugs")
         .header("cookie", format!("__Host-session={};", session_token))
-        .send()?
-        .text()?;
+        .send().await?;
+
+    let http_response = http_response.error_for_status()?;
+    let http_response = http_response.text().await?;
 
     let token = extract_csrf_token(&http_response);
     if token.is_none() {
-        return Err("Could not extract token from page".into());
+        return Err(anyhow::Error::msg("unable to find CSRF token in page"));
     }
 
     Ok(token.unwrap())
 }
 
 // GraphQL types
 type DateTime = String;
-type DateInput = String;
 type URI = String;
 
 #[derive(GraphQLQuery, Debug)]
@@ -110,28 +110,21 @@ pub struct TeamNameHacktivityQuery;
 pub struct DiscoveryQuery;
 
 
-// impl Default for discovery_query::OrderDirection {
-//     fn default() -> Self {
-//         discovery_query::OrderDirection::popular
-//     }
-// }
-
-// Tests
 #[cfg(test)]
 mod tests {
     use super::*;
 
     #[test]
     fn can_extract_csrf_token() {
         let csrf_token = "hello_world";
-        let fake_page = format!(
+        let mock_page = format!(
             r#"
             <meta name="csrf-param" content="authenticity_token" />
             <meta name="csrf-token" content="{csrf_token}" />
         "#
         );
 
-        let extracted_token = extract_csrf_token(&fake_page);
+        let extracted_token = extract_csrf_token(&mock_page);
         assert_eq!(extracted_token.unwrap(), csrf_token);
     }
 }

src/security_api/src/hackerone.rs

@@ -1,18 +1,3 @@
 pub mod hackerone;
 pub mod models;
-pub mod redis;
-
-pub fn add(left: usize, right: usize) -> usize {
-    left + right
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn it_works() {
-        let result = add(2, 2);
-        assert_eq!(result, 4);
-    }
-}
+pub mod redis;

src/security_api/src/lib.rs

@@ -2,7 +2,7 @@ use chrono::NaiveDateTime;
 use nanoid::nanoid;
 use serde::{Deserialize, Serialize};
 
-#[derive(Debug, Deserialize, Serialize, Clone)]
+#[derive(Default, Debug, Deserialize, Serialize, Clone)]
 pub struct RepData {
     pub reputation: i64,
     pub rank: i64,
@@ -12,7 +12,7 @@ pub struct RepData {
     pub team_handle: Option<String>,
 }
 
-#[derive(Debug, Deserialize, Serialize)]
+#[derive(Default, Debug, Deserialize, Serialize)]
 pub struct RepDataQueueItem {
     pub id: Option<String>,
     pub diff: Vec<Vec<RepData>>,
@@ -24,7 +24,6 @@ pub struct RepDataQueueItem {
 
 impl RepDataQueueItem {
     pub fn create_id(&mut self) {
-        // TODO: get rid of nanoid, write a unique id func
         let id = nanoid!();
         self.id = Some(id);
     }
@@ -60,7 +59,6 @@ pub struct ReportsDataQueueItem {
 
 impl ReportsDataQueueItem {
     pub fn create_id(&mut self) {
-        // TODO: get rid of nanoid, write a unique id func
         let id = nanoid!();
         self.id = Some(id);
     }

src/security_api/src/models.rs

@@ -1,39 +1,48 @@
-pub use redis;
+use std::sync::LazyLock;
 
-pub fn open(url: &str) -> Result<redis::Client, Box<dyn std::error::Error>> {
-    let client = redis::Client::open(url)?;
-    Ok(client)
+pub use deadpool_redis::redis;
+use deadpool_redis::{self as deadpool, redis::AsyncCommands, Connection};
+
+static GLOBAL_REDIS_POOL: LazyLock<deadpool::Pool> = LazyLock::new(|| {
+    let config = get_config();
+    config.create_pool(Some(deadpool::Runtime::Tokio1)).unwrap()
+});
+
+pub fn get_connection() -> deadpool::Pool {
+    GLOBAL_REDIS_POOL.clone()
 }
 
-pub fn save_vec_to_set<'a, V: serde::Deserialize<'a> + serde::Serialize>(
-    name: String,
+pub fn get_config() -> deadpool::Config {
+    let url = std::env::var("REDIS_URL");
+    let url = url.expect("Redis connection URI required");
+    deadpool::Config::from_url(url)
+}
+
+pub async fn save_vec_to_set<'a, V: serde::Deserialize<'a> + serde::Serialize>(
+    name: &str,
     data: Vec<V>,
     overwrite: bool,
-    mut conn: &mut redis::Connection,
-) -> Result<(), Box<dyn std::error::Error>> {
+    redis: &mut Connection,
+) -> Result<(), anyhow::Error> {
     if overwrite {
-        redis::cmd("DEL").arg(&name).query(&mut conn)?;
+        redis.del::<_, ()>(&name).await?;
     }
 
     for i in data {
         let value_name = serde_json::to_string(&i)?;
-        redis::cmd("SADD")
-            .arg(&name)
-            .arg(value_name)
-            .query(&mut conn)?;
+        redis.sadd::<_, _, ()>(&name, value_name).await?;
     }
 
     Ok(())
 }
 
-pub fn load_set_to_vec(
-    name: String,
-    mut conn: &mut redis::Connection,
-) -> Result<Vec<String>, Box<dyn std::error::Error>> {
-    let set_values: Vec<String> = redis::cmd("SMEMBERS").arg(&name).query(&mut conn)?;
-
+pub async fn load_set_to_vec(
+    name: &str,
+    redis: &mut Connection,
+) -> Result<Vec<String>, anyhow::Error> {
+    let set_members = redis.smembers::<_, Vec<String>>(&name).await?;
     let mut result = Vec::new();
-    for mut value in set_values {
+    for mut value in set_members {
         if value.starts_with('"') {
             value = value[1..value.len() - 1].to_string();
         }

src/security_api/src/redis.rs

@@ -7,11 +7,14 @@ edition = "2021"
 
 [dependencies]
 clap = { version = "4.0", features = ["derive", "env"] }
-reqwest = { version = "0.11.18", features = ["blocking", "json"] }
+reqwest.workspace = true
 security_api = { path = "../security_api" }
 serde.workspace = true
 serde_json.workspace = true
 twilight-util = { version = "0.15.2", features = ["builder"] }
 twilight-model = "0.15.2"
 log.workspace = true
-pretty_env_logger.workspace = true
+pretty_env_logger.workspace = true
+tokio.workspace = true
+anyhow.workspace = true
+futures-util = "0.3.31"