chore: update depedencies (#782)

* chore: update depedencies

arrow-schema = "47.0.0" -> "51.0.0"
arrow-array =  "47.0.0" -> "51.0.0"
arrow-json = "47.0.0" -> "51.0.0"
arrow-ipc = "47.0.0" -> "51.0.0"
arrow-select = "47.0.0" -> "51.0.0"
parquet = "47.0.0" -> "51.0.0"
arrow-flight = "47.0.0" -> "51.0.0"
datafusion = "32.0.0" -> "37.1.0"
object_store = "0.7.0" -> "0.9.1"
tonic = "0.10.0" -> "0.11.0"
tonic-web = "0.10.0" -> "0.11.0"

* chore: update depedencies related to arrow

arrow-schema = "47.0.0" -> "51.0.0"
arrow-array =  "47.0.0" -> "51.0.0"
arrow-json = "47.0.0" -> "51.0.0"
arrow-ipc = "47.0.0" -> "51.0.0"
arrow-select = "47.0.0" -> "51.0.0"
parquet = "47.0.0" -> "51.0.0"
arrow-flight = "47.0.0" -> "51.0.0"
datafusion = "32.0.0" -> "37.1.0"
object_store = "0.7.0" -> "0.9.1"
tonic = "0.10.0" -> "0.11.0"
tonic-web = "0.10.0" -> "0.11.0"
thread-priority = "0.13.1" -> "1.0.0"
zip = "0.6" -> "1.1.1"
sysinfo = "0.29.6" -> "0.30.11"

* update dependencies second pass

actix-web = "4.3"-> "4.5.1"
actix-cors = "0.6" -> "0.7.0"
base64 = "0.21" -> "0.22.0"
cookie = "0.17.0" -> "0.18.1"
crossterm = "0.26" -> "0.27.0"
env_logger = "0.10" -> "0.11.3"
hostname = "0.3" -> "0.4.0"
http = "0.2" -> "0.2.7"
itertools = "0.10" -> "0.12.1"
reqwest = "0.11.18" -> "0.11.27"
rustls = "0.20" -> "0.22.4"
rustls-pemfile = "1.0" -> "2.1.2"
uptime_lib = "0.2.2" -> "0.3.0"
openid = "0.12.0" -> "0.14.0"
cargo_toml = "0.15" -> "0.20.1"
rstest = "0.16" -> "0.19.0"

* changes in behaviour on Null rows

arrow now infers null rows by mark existence of a field if it wasn't
known before. This was not done in previous version.

So schema for
`{"a": 1,"b": "hello","c": null}` is inferred to have all 3 columns ["a":
Number, "b": string, "c": null] previously it would have inferred it to
be `["a": Number, "b": String]

Modified the tests basic_object_with_null_into_rb and
arr_with_null_derive_schema_into_rb (temporary, going forward we should
delete them)

Author: Eshanatnight

Cargo.lock

@@ -9,25 +9,25 @@ build = "build.rs"
 
 [dependencies]
 ### apache arrow/datafusion dependencies
-arrow-schema = { version = "^47.0.0", features = ["serde"] }
-arrow-array = { version = "^47.0.0" }
-arrow-json = "^47.0.0"
-arrow-ipc = "^47.0.0"
-arrow-select = "^47.0.0"
-datafusion = "32.0.0"
-object_store = { version = "^0.7.0", features = ["cloud", "aws"] }
-parquet = "^47.0.0"
+arrow-schema = { version = "51.0.0", features = ["serde"] }
+arrow-array = { version = "51.0.0" }
+arrow-json = "51.0.0"
+arrow-ipc = "51.0.0"
+arrow-select = "51.0.0"
+datafusion = "37.1.0"
+object_store = { version = "0.9.1", features = ["cloud", "aws"] }
+parquet = "51.0.0"
 
 ### LiveTail server deps
-arrow-flight = "47.0.0"
-tonic = {version = "0.10.0", features = ["tls"] }
-tonic-web = "0.10.0"
+arrow-flight = "51.0.0"
+tonic = {version = "0.11.0", features = ["tls"] }
+tonic-web = "0.11.0"
 tower-http = { version = "0.4.4", features = ["cors"] }
 
 ### actix dependencies
 actix-web-httpauth = "0.8"
-actix-web = { version = "4.3", features = ["rustls"] }
-actix-cors = "0.6"
+actix-web = { version = "4.5.1", features = ["rustls-0_22"] }
+actix-cors = "0.7.0"
 actix-web-prometheus = { version = "0.1" }
 actix-web-static-files = "4.0"
 mime = "0.3.17"
@@ -36,11 +36,11 @@ mime = "0.3.17"
 anyhow = { version = "1.0", features = ["backtrace"] }
 argon2 = "0.5.0"
 async-trait = "0.1"
-base64 = "0.21"
+base64 = "0.22.0"
 bytes = "1.4"
 byteorder = "1.4.3"
 bzip2 = { version = "*", features = ["static"] }
-cookie = "0.17.0"
+cookie = "0.18.1"
 chrono = "0.4"
 chrono-humanize = "0.2"
 clap = { version = "4.1", default-features = false, features = [
@@ -53,51 +53,51 @@ clap = { version = "4.1", default-features = false, features = [
   "error-context",
 ] }
 clokwerk = "0.4"
-crossterm = "0.26"
+crossterm = "0.27.0"
 derive_more = "0.99"
-env_logger = "0.10"
+env_logger = "0.11.3"
 fs_extra = "1.3"
 futures = "0.3"
 futures-util = "0.3.28"
 hex = "0.4"
-hostname = "0.3"
-http = "0.2"
+hostname = "0.4.0"
+http = "0.2.7"
 humantime-serde = "1.1"
-itertools = "0.10"
+itertools = "0.12.1"
 log = "0.4"
 num_cpus = "1.15"
 once_cell = "1.17.1"
 prometheus = { version = "0.13", features = ["process"] }
 rand = "0.8"
 regex = "1.7.3"
 relative-path = { version = "1.7", features = ["serde"] }
-reqwest = { version = "0.11.18", default_features = false, features = [
+reqwest = { version = "0.11.27", default_features = false, features = [
   "rustls-tls",
   "json",
 ] }
-rustls = "0.20"
-rustls-pemfile = "1.0"
+rustls = "0.22.4"
+rustls-pemfile = "2.1.2"
 semver = "1.0"
 serde = { version = "1.0", features = ["rc"] }
 serde_json = "1.0"
 static-files = "0.2"
-sysinfo = "0.29.6"
+sysinfo = "0.30.11"
 thiserror = "1"
-thread-priority = "0.13.1"
+thread-priority = "1.0.0"
 tokio = { version = "1.28", default-features = false, features = [
   "sync",
   "macros",
   "fs",
 ] }
 tokio-stream = { version = "0.1", features = ["fs"] }
 ulid = { version = "1.0", features = ["serde"] }
-uptime_lib = "0.2.2"
+uptime_lib = "0.3.0"
 xxhash-rust = { version = "0.8", features = ["xxh3"] }
 xz2 = { version = "*", features = ["static"] }
 nom = "7.1.3"
 humantime = "2.1.0"
 human-size = "0.4"
-openid = { version = "0.12.0", default-features = false, features = ["rustls"] }
+openid = { version = "0.14.0", default-features = false, features = ["rustls"] }
 url = "2.4.0"
 http-auth-basic = "0.3.3"
 serde_repr = "0.1.17"
@@ -108,18 +108,18 @@ prometheus-parse = "0.2.5"
 sha2 = "0.10.8"
 
 [build-dependencies]
-cargo_toml = "0.15"
+cargo_toml = "0.20.1"
 sha1_smol = { version = "1.0", features = ["std"] }
 static-files = "0.2"
 ureq = "2.6"
 vergen = { version = "8.1", features = ["build", "git", "cargo", "gitcl"] }
-zip = { version = "0.6", default_features = false, features = ["deflate"] }
+zip = { version = "1.1.1", default_features = false, features = ["deflate"] }
 url = "2.4.0"
 prost-build = "0.12.3"
 
 [dev-dependencies]
 maplit = "1.0"
-rstest = "0.16"
+rstest = "0.19.0"
 
 [package.metadata.parseable_ui]
 assets-url = "https://github.com/parseablehq/console/releases/download/v0.7.0/build.zip"

server/Cargo.toml

@@ -22,7 +22,7 @@ use chrono_humanize::{Accuracy, Tense};
 use crossterm::style::Stylize;
 use std::env;
 use std::path::Path;
-use sysinfo::SystemExt;
+use sysinfo::System;
 use ulid::Ulid;
 
 use crate::analytics;
@@ -59,11 +59,7 @@ pub fn user_agent(uid: &Ulid) -> String {
         uid,
         current().released_version,
         current().commit_hash,
-        analytics::SYS_INFO
-            .lock()
-            .unwrap()
-            .name()
-            .unwrap_or_default(),
+        System::name().unwrap_or_default(),
         platform()
     )
 }

server/src/about.rs

@@ -34,7 +34,7 @@ use serde_json::Value;
 use std::collections::HashMap;
 use std::sync::Mutex;
 use std::time::Duration;
-use sysinfo::{CpuExt, System, SystemExt};
+use sysinfo::System;
 use ulid::Ulid;
 
 const ANALYTICS_SERVER_URL: &str = "https://analytics.parseable.io:80";
@@ -86,8 +86,8 @@ impl Report {
         let mut cpu_count = 0;
         let mut mem_total = 0;
         if let Ok(info) = SYS_INFO.lock() {
-            os_version = info.os_version().unwrap_or_default();
-            os_name = info.name().unwrap_or_default();
+            os_version = System::os_version().unwrap_or_default();
+            os_name = System::name().unwrap_or_default();
             cpu_count = info.cpus().len();
             mem_total = info.total_memory();
         }

server/src/analytics.rs

@@ -28,7 +28,7 @@ use crate::{
 };
 use crate::{handlers, Mode};
 use bytes::Bytes;
-use chrono::{DateTime, Local, NaiveDateTime, NaiveTime, Utc};
+use chrono::{DateTime, Local, NaiveTime, Utc};
 use relative_path::RelativePathBuf;
 use std::io::Error as IOError;
 pub mod column;
@@ -82,12 +82,8 @@ fn get_file_bounds(file: &manifest::File) -> (DateTime<Utc>, DateTime<Utc>) {
         .unwrap()
     {
         column::TypedStatistics::Int(stats) => (
-            NaiveDateTime::from_timestamp_millis(stats.min)
-                .unwrap()
-                .and_utc(),
-            NaiveDateTime::from_timestamp_millis(stats.max)
-                .unwrap()
-                .and_utc(),
+            DateTime::from_timestamp_millis(stats.min).unwrap(),
+            DateTime::from_timestamp_millis(stats.max).unwrap(),
         ),
         _ => unreachable!(),
     }

server/src/catalog.rs

@@ -112,28 +112,9 @@ impl StorageStats {
 }
 
 pub fn merge_quried_stats(stats: Vec<QueriedStats>) -> QueriedStats {
-    // get the actual creation time
-    // let min_creation_time = stats
-    //     .iter()
-    //     .map(|x| x.creation_time.parse::<DateTime<Utc>>().unwrap())
-    //     .min()
-    //     .unwrap(); // should never be None
-
     // get the stream name
     let stream_name = stats[1].stream.clone();
 
-    // get the first event at
-    // let min_first_event_at = stats
-    //     .iter()
-    //     .map(|x| match x.first_event_at.as_ref() {
-    // we can directly unwrap here because
-    // we are sure that the first_event_at is a valid date
-    //         Some(fea) => fea.parse::<DateTime<Utc>>().unwrap(),
-    //         None => Utc::now(), // current time ie the max time
-    //     })
-    //     .min()
-    //     .unwrap(); // should never be None
-
     let min_time = stats.iter().map(|x| x.time).min().unwrap_or_else(Utc::now);
 
     let cumulative_ingestion =

server/src/handlers/http/cluster/utils.rs

@@ -354,7 +354,7 @@ mod tests {
         .unwrap();
 
         assert_eq!(rb.num_rows(), 1);
-        assert_eq!(rb.num_columns(), 5);
+        assert_eq!(rb.num_columns(), 6);
         assert_eq!(
             rb.column_by_name("a").unwrap().as_int64_arr(),
             &Int64Array::from_iter([1])
@@ -663,7 +663,7 @@ mod tests {
         .unwrap();
 
         assert_eq!(rb.num_rows(), 3);
-        assert_eq!(rb.num_columns(), 5);
+        assert_eq!(rb.num_columns(), 6);
         assert_eq!(
             rb.column_by_name("a").unwrap().as_int64_arr(),
             &Int64Array::from(vec![Some(1), Some(1), Some(1)])

server/src/handlers/http/ingest.rs

@@ -93,7 +93,7 @@ impl ParseableServer for IngestServer {
 
         if let Some(config) = ssl {
             http_server
-                .bind_rustls(&CONFIG.parseable.address, config)?
+                .bind_rustls_0_22(&CONFIG.parseable.address, config)?
                 .run()
                 .await?;
         } else {

server/src/handlers/http/modal/ingest_server.rs

@@ -73,7 +73,7 @@ impl ParseableServer for QueryServer {
         let http_server = HttpServer::new(create_app_fn).workers(num_cpus::get());
         if let Some(config) = ssl {
             http_server
-                .bind_rustls(&CONFIG.parseable.address, config)?
+                .bind_rustls_0_22(&CONFIG.parseable.address, config)?
                 .run()
                 .await?;
         } else {

server/src/handlers/http/modal/query_server.rs

@@ -32,7 +32,7 @@ use crate::migration;
 use crate::rbac;
 use crate::storage;
 use crate::sync;
-use std::{fs::File, io::BufReader, sync::Arc};
+use std::sync::Arc;
 
 use actix_web::web::resource;
 use actix_web::Resource;
@@ -42,9 +42,6 @@ use actix_web_prometheus::PrometheusMetrics;
 use actix_web_static_files::ResourceFiles;
 use async_trait::async_trait;
 
-use rustls::{Certificate, PrivateKey, ServerConfig};
-use rustls_pemfile::{certs, pkcs8_private_keys};
-
 use crate::{
     handlers::http::{
         self, cross_origin_config, ingest, llm, logstream,
@@ -57,6 +54,7 @@ use crate::{
 
 // use super::generate;
 use super::generate;
+use super::ssl_acceptor::get_ssl_acceptor;
 use super::OpenIdClient;
 use super::ParseableServer;
 
@@ -89,45 +87,16 @@ impl ParseableServer for Server {
                 .wrap(cross_origin_config())
         };
 
-        let ssl_acceptor = match (
+        let ssl = get_ssl_acceptor(
             &CONFIG.parseable.tls_cert_path,
             &CONFIG.parseable.tls_key_path,
-        ) {
-            (Some(cert), Some(key)) => {
-                // init server config builder with safe defaults
-                let config = ServerConfig::builder()
-                    .with_safe_defaults()
-                    .with_no_client_auth();
-
-                // load TLS key/cert files
-                let cert_file = &mut BufReader::new(File::open(cert)?);
-                let key_file = &mut BufReader::new(File::open(key)?);
-
-                // convert files to key/cert objects
-                let cert_chain = certs(cert_file)?.into_iter().map(Certificate).collect();
-
-                let mut keys: Vec<PrivateKey> = pkcs8_private_keys(key_file)?
-                    .into_iter()
-                    .map(PrivateKey)
-                    .collect();
-
-                // exit if no keys could be parsed
-                if keys.is_empty() {
-                    anyhow::bail!("Could not locate PKCS 8 private keys.");
-                }
-
-                let server_config = config.with_single_cert(cert_chain, keys.remove(0))?;
-
-                Some(server_config)
-            }
-            (_, _) => None,
-        };
+        )?;
 
         // concurrent workers equal to number of cores on the cpu
         let http_server = HttpServer::new(create_app_fn).workers(num_cpus::get());
-        if let Some(config) = ssl_acceptor {
+        if let Some(config) = ssl {
             http_server
-                .bind_rustls(&CONFIG.parseable.address, config)?
+                .bind_rustls_0_22(&CONFIG.parseable.address, config)?
                 .run()
                 .await?;
         } else {