From 43326f9dd9c2886e6e65494d046056899f073b20 Mon Sep 17 00:00:00 2001 From: Richard Zak Date: Sun, 23 Feb 2025 22:02:41 -0500 Subject: [PATCH] feat: aws s3 as a file storage option Signed-off-by: Richard Zak --- Cargo.lock | 515 +++++++++++++++++++++++++++++++--- Cargo.toml | 3 + crates/server/Cargo.toml | 3 + crates/server/src/cloud.rs | 112 ++++++++ crates/server/src/db/mod.rs | 2 + crates/server/src/http/mod.rs | 3 + crates/server/src/lib.rs | 134 +++++---- src/cli/config.rs | 35 +++ src/cli/run.rs | 18 ++ src/cli/vt/mod.rs | 1 + src/main.rs | 3 + 11 files changed, 731 insertions(+), 98 deletions(-) create mode 100644 crates/server/src/cloud.rs diff --git a/Cargo.lock b/Cargo.lock index a045994c..7f7040c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -339,9 +339,9 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb812ffb58524bdd10860d7d974e2f01cc0950c2438a74ee5ec2e2280c6c4ffa" +checksum = "497c00e0fd83a72a79a39fcbd8e3e2f055d6f6c7e025f3b3d91f4f8e76527fb8" dependencies = [ "async-task", "concurrent-queue", @@ -491,6 +491,43 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "aws-config" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c478f5b10ce55c9a33f87ca3404ca92768b144fc1bfdede7c0121214a8283a25" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 1.3.1", + "time", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1541072f81945fa1251f8795ef6c92c4282d74d59f88498ae7d4bf00f0ebdad9" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + [[package]] name = "aws-lc-rs" version = "1.13.3" @@ -514,6 +551,288 @@ dependencies = [ "fs_extra", ] +[[package]] +name = "aws-runtime" +version = "1.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c034a1bc1d70e16e7f4e4caf7e9f7693e4c9c24cd91cf17c2a0b21abaebc7c8b" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.103.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af040a86ae4378b7ed2f62c83b36be1848709bbbf5757ec850d0e08596a26be9" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "lru", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.84.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91abcdbfb48c38a0419eb75e0eac772a4783a96750392680e4f3c25a8a0535b9" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084c34162187d39e3740cb635acd73c4e3a551a36146ad6fe8883c929c9f876c" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.3.1", + "percent-encoding", + "sha2", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.63.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dbef71cd3cf607deb5c407df52f7e589e6849b296874ee448977efbb6d0832b" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc-fast", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "182b03393e8c677347fb5705a04a9392695d47d20ef0a2f8cfe28c8e6b9b9778" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.62.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c4dacf2d38996cf729f55e7a762b30918229917eca115de45dfa8dfb97796c9" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.61.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a16e040799d29c17412943bdbf488fd75db04112d0c0d4b9290bacf5ae0014b9" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9364d5989ac4dd918e5cc4c4bdcc61c9be17dcd2586ea7f69e348fc7c6cab393" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d57c8b53a72d15c8e190475743acf34e4996685e346a3448dd54ef696fc6e0" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07f5e0fc8a6b3f2303f331b94504bbf754d85488f402d6f1dd7a6080f99afe56" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.3.1", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db87b96cb1b16c024980f133968d52882ca0daaee3a086c6decc500f6c99728" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b069d19bf01e46298eaedd7c6f283fe565a59263e53eebec945f3e6398f42390" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.8.4" @@ -524,8 +843,8 @@ dependencies = [ "axum-macros", "bytes", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-util", @@ -554,8 +873,8 @@ checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -585,8 +904,8 @@ dependencies = [ "arc-swap", "bytes", "fs-err", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "hyper", "hyper-util", "pin-project-lite", @@ -619,6 +938,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.8.0" @@ -689,9 +1018,9 @@ dependencies = [ [[package]] name = "bit_field" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" [[package]] name = "bitflags" @@ -864,6 +1193,16 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bytesize" version = "2.0.1" @@ -1362,6 +1701,34 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc-fast" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf62af4cc77d8fe1c22dde4e721d87f2f54056139d8c412e1366b740305f56f" +dependencies = [ + "crc", + "digest", + "libc", + "rand 0.9.2", + "regex", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -1884,7 +2251,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -2448,7 +2815,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.3.1", "indexmap", "slab", "tokio", @@ -2551,6 +2918,17 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.3.1" @@ -2562,6 +2940,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -2569,7 +2958,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.3.1", ] [[package]] @@ -2580,8 +2969,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "pin-project-lite", ] @@ -2626,8 +3015,8 @@ dependencies = [ "futures-channel", "futures-core", "h2", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -2644,7 +3033,7 @@ version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http", + "http 1.3.1", "hyper", "hyper-util", "rustls", @@ -2682,8 +3071,8 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "hyper", "ipnet", "libc", @@ -3061,9 +3450,9 @@ dependencies = [ [[package]] name = "image-webp" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6970fe7a5300b4b42e62c52efa0187540a5bef546c60edaf554ef595d2e6f0b" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" dependencies = [ "byteorder-lite", "quick-error", @@ -3388,14 +3777,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.53.3", ] [[package]] name = "liblzma" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "272b875472a046e39ff7408374a5a050b112d2142211a0f54a295c0bd1c3c757" +checksum = "10bf66f4598dc77ff96677c8e763655494f00ff9c1cf79e2eb5bb07bc31f807d" dependencies = [ "liblzma-sys", ] @@ -3550,6 +3939,15 @@ dependencies = [ "imgref", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -3743,6 +4141,9 @@ dependencies = [ "anyhow", "app-memory-usage-fetcher", "argon2", + "aws-config", + "aws-sdk-s3", + "aws-smithy-async", "axum", "axum-server", "base64", @@ -3755,7 +4156,7 @@ dependencies = [ "flate2", "fuzzyhash", "hex", - "http", + "http 1.3.1", "http-body-util", "human-hash", "humansize", @@ -4589,6 +4990,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "overload" version = "0.1.1" @@ -5077,9 +5484,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" dependencies = [ "bytes", "cfg_aliases", @@ -5088,7 +5495,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls", - "socket2 0.5.10", + "socket2 0.6.0", "thiserror 2.0.16", "tokio", "tracing", @@ -5097,9 +5504,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.12" +version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ "bytes", "getrandom 0.3.3", @@ -5118,16 +5525,16 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.0", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -5351,6 +5758,12 @@ dependencies = [ "regex-syntax 0.8.6", ] +[[package]] +name = "regex-lite" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943f41321c63ef1c92fd763bfe054d2668f7f225a5c29f0105903dc2fc04ba30" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -5383,8 +5796,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-rustls", @@ -5574,7 +5987,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -6380,7 +6793,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix 1.0.8", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -6791,8 +7204,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "iri-string", "pin-project-lite", "tokio", @@ -7067,6 +7480,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "usvg" version = "0.45.1" @@ -7190,6 +7609,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "vtable" version = "0.3.0" @@ -7535,7 +7960,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.60.2", ] [[package]] @@ -8146,6 +8571,12 @@ version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fd8403733700263c6eb89f192880191f1b83e332f7a20371ddcf421c4a337c7" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "xmlwriter" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 923c7876..28280d6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,6 +106,9 @@ anyhow = { version = "1.0", default-features = false } app-memory-usage-fetcher = { version = "0.2.1", default-features = false } argon2 = { version = "0.5.3", default-features = false } axum = { version = "0.8.4", default-features = false } +aws-config = { version = "1.8.5", default-features = false } +aws-sdk-s3 = { version = "1.103.0", default-features = false } +aws-smithy-async = { version = "1.2.5", default-features = false } axum-server = { version = "0.7.2", default-features = false } base64 = { version = "0.22.1", default-features = false } bytecount = { version = "0.6.9", default-features = false } diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index e25ffec1..a8e10077 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -28,6 +28,9 @@ aes-gcm = { workspace = true, features = ["aes", "alloc", "getrandom", "std"] } anyhow = { workspace = true, features = ["std"] } app-memory-usage-fetcher = { workspace = true } argon2 = { workspace = true, features = ["alloc", "password-hash", "std"] } +aws-config = { workspace = true, features = ["behavior-version-latest"] } +aws-sdk-s3 = { workspace = true } +aws-smithy-async = { workspace = true, features = ["rt-tokio"] } axum = { workspace = true, features = ["http1", "http2", "json", "macros", "tokio"] } axum-server = { workspace = true, features = ["tls-rustls", "rustls"] } base64 = { workspace = true, features = ["alloc", "std"] } diff --git a/crates/server/src/cloud.rs b/crates/server/src/cloud.rs new file mode 100644 index 00000000..b49e93cc --- /dev/null +++ b/crates/server/src/cloud.rs @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::fmt::{Debug, Formatter}; + +use anyhow::Result; +use aws_config::{AppName, BehaviorVersion, SdkConfig}; +use aws_sdk_s3::config::{Credentials, SharedCredentialsProvider}; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::Client; +use aws_smithy_async::rt::sleep::{SharedAsyncSleep, TokioSleep}; +use base64::prelude::BASE64_STANDARD; +use base64::Engine; +use sha2::{Digest, Sha256}; + +/// Cloud storage options +pub enum CloudStorage { + /// AWS S3 + S3(AWS), +} + +impl CloudStorage { + pub(crate) async fn write(&self, data: Vec, hashed_path: &str) -> Result<()> { + match self { + CloudStorage::S3(aws) => aws.write(data, hashed_path).await, + } + } + + pub(crate) async fn read(&self, hashed_path: &str) -> Result> { + match self { + CloudStorage::S3(aws) => aws.read(hashed_path).await, + } + } +} + +impl Debug for CloudStorage { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + CloudStorage::S3(_) => write!(f, "S3"), + } + } +} + +/// AWS S3 connection +#[derive(Debug)] +pub struct AWS { + /// AWS Client object + client: Client, + + /// S3 bucket name + bucket: String, +} + +impl AWS { + /// Create a new AWS connection + /// + /// # Panics + /// + /// The code will not panic despite the call to `unwrap()` + #[must_use] + pub fn new(access_key_id: String, secret_access_key: String, bucket: String) -> Self { + let creds = Credentials::new(access_key_id, secret_access_key, None, None, "malwaredb"); + let provider = SharedCredentialsProvider::new(creds); + let sleep_impl = SharedAsyncSleep::new(TokioSleep::new()); + let config = SdkConfig::builder() + .credentials_provider(provider) + .behavior_version(BehaviorVersion::latest()) + .sleep_impl(sleep_impl) + .app_name(AppName::new(constcat::concat!("malwaredb-v", crate::MDB_VERSION)).unwrap()) + .build(); + + Self { + client: Client::new(&config), + bucket, + } + } + + #[inline] + async fn write(&self, data: Vec, hashed_path: &str) -> Result<()> { + // Data might be compressed and/or encrypted, so re-hash for transmission to AWS + let mut hasher = Sha256::new(); + hasher.update(&data); + + let sha256_b64 = BASE64_STANDARD.encode(hasher.finalize()); + let bytes = ByteStream::from(data); + + self.client + .put_object() + .bucket(&self.bucket) + .checksum_sha256(sha256_b64) + .key(hashed_path) + .body(bytes) + .send() + .await?; + + Ok(()) + } + + #[inline] + async fn read(&self, hashed_path: &str) -> Result> { + Ok(self + .client + .get_object() + .bucket(&self.bucket) + .key(hashed_path) + .send() + .await? + .body + .collect() + .await? + .to_vec()) + } +} diff --git a/crates/server/src/db/mod.rs b/crates/server/src/db/mod.rs index 57b8085f..ad1fe82f 100644 --- a/crates/server/src/db/mod.rs +++ b/crates/server/src/db/mod.rs @@ -1010,6 +1010,7 @@ mod tests { }), cert: None, key: None, + cloud: None, }; let vt: VtUpdater = state.try_into().expect("failed to create VtUpdater"); @@ -1071,6 +1072,7 @@ mod tests { }), cert: None, key: None, + cloud: None, }; let sqlite_second = Sqlite::new(DB_FILE) diff --git a/crates/server/src/http/mod.rs b/crates/server/src/http/mod.rs index 46f7b4cd..79f6bff8 100644 --- a/crates/server/src/http/mod.rs +++ b/crates/server/src/http/mod.rs @@ -455,6 +455,7 @@ mod tests { vt_client: None, cert: None, key: None, + cloud: None, }; state @@ -516,6 +517,7 @@ mod tests { vt_client: None, cert: Some("../../testdata/server_ca_cert.pem".into()), key: Some("../../testdata/server_key.pem".into()), + cloud: None, } } else { State { @@ -536,6 +538,7 @@ mod tests { vt_client: None, cert: Some("../../testdata/server_cert.der".into()), key: Some("../../testdata/server_key.der".into()), + cloud: None, } }; diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index 2566a6a3..70692740 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -5,6 +5,9 @@ #![deny(clippy::all)] #![deny(clippy::pedantic)] +/// Cloud storage logic +pub mod cloud; + /// Cryptographic functionality for file storage pub mod crypto; @@ -21,6 +24,7 @@ pub mod utils; #[cfg(feature = "vt")] pub mod vt; +use crate::cloud::CloudStorage; use crate::crypto::FileEncryption; use crate::db::MDBConfig; use malwaredb_api::ServerInfo; @@ -87,6 +91,9 @@ pub struct State { /// Https private key file key: Option, + + /// AWS client and S3 bucket + cloud: Option, } impl State { @@ -107,6 +114,7 @@ impl State { key: Option, pg_cert: Option, #[cfg(feature = "vt")] vt_client: Option, + cloud: Option, ) -> Result { if let Some(dir) = &directory { if !dir.exists() { @@ -147,6 +155,7 @@ impl State { started: SystemTime::now(), cert, key, + cloud, }) } @@ -157,10 +166,13 @@ impl State { /// * If the file can't be written. /// * If a necessary sub-directory can't be created. pub async fn store_bytes(&self, data: &[u8]) -> Result { - if let Some(dest_path) = &self.directory { + if self.directory.is_none() && self.cloud.is_none() { + Ok(false) + } else { let mut hasher = Sha256::new(); hasher.update(data); - let sha256 = hex::encode(hasher.finalize()); + let sha256_bytes = hasher.finalize(); + let sha256 = hex::encode(sha256_bytes); // Trait `HashPath` needs to be re-worked so it can work with Strings. // This code below ends up making the String into ASCII representations of the hash @@ -173,16 +185,6 @@ impl State { sha256 ); - // The path which has the file name included, with the storage directory prepended. - //let hashed_path = result.hashed_path(3); - let mut dest_path = dest_path.clone(); - dest_path.push(hashed_path); - - // Remove the file name so we can just have the directory path. - let mut just_the_dir = dest_path.clone(); - just_the_dir.pop(); - std::fs::create_dir_all(just_the_dir)?; - let data = if self.db_config.compression { let buff = Cursor::new(data); let mut compressed = Vec::with_capacity(data.len() / 2); @@ -206,11 +208,23 @@ impl State { data }; - std::fs::write(dest_path, data)?; + if let Some(dest_path) = &self.directory { + // The path which has the file name included, with the storage directory prepended. + //let hashed_path = result.hashed_path(3); + let mut dest_path = dest_path.clone(); + dest_path.push(hashed_path); + + // Remove the file name so we can just have the directory path. + let mut just_the_dir = dest_path.clone(); + just_the_dir.pop(); + std::fs::create_dir_all(just_the_dir)?; + + std::fs::write(dest_path, data)?; + } else if let Some(cloud) = &self.cloud { + cloud.write(data, &hashed_path).await?; + } Ok(true) - } else { - Ok(false) } } @@ -222,53 +236,61 @@ impl State { /// * The file could not be read, maybe because it doesn't exist. /// * Failure to decrypt or decompress (corruption). pub async fn retrieve_bytes(&self, sha256: &String) -> Result> { - if let Some(dest_path) = &self.directory { - let path = format!( - "{}/{}/{}/{}", - &sha256[0..2], - &sha256[2..4], - &sha256[4..6], - sha256 - ); - // Trait `HashPath` needs to be re-worked so it can work with Strings. - // This code below ends up making the String into ASCII representations of the hash - // See: https://github.com/malwaredb/malwaredb-rs/issues/60 - //let path = sha256.as_bytes().iter().hashed_path(3); - let contents = std::fs::read(dest_path.join(path))?; + if self.directory.is_none() && self.cloud.is_none() { + bail!("files are not saved") + } - let contents = if self.keys.is_empty() { - // We don't have file encryption enabled - contents - } else { - let (key_id, nonce) = self.db_type.get_file_encryption_key_id(sha256).await?; - if let Some(key_id) = key_id { - if let Some(key) = self.keys.get(&key_id) { - key.decrypt(&contents, nonce)? - } else { - bail!("File was encrypted but we don't have tke key!") - } + // Trait `HashPath` needs to be re-worked so it can work with Strings. + // This code below ends up making the String into ASCII representations of the hash + // See: https://github.com/malwaredb/malwaredb-rs/issues/60 + //let path = sha256.as_bytes().iter().hashed_path(3); + + let path = format!( + "{}/{}/{}/{}", + &sha256[0..2], + &sha256[2..4], + &sha256[4..6], + sha256 + ); + + let contents = if let Some(dest_path) = &self.directory { + std::fs::read(dest_path.join(path))? + } else if let Some(cloud) = &self.cloud { + cloud.read(&path).await? + } else { + bail!("No AWS config and no local directory, shouldn't happen!") + }; + + let contents = if self.keys.is_empty() { + // We don't have file encryption enabled + contents + } else { + let (key_id, nonce) = self.db_type.get_file_encryption_key_id(sha256).await?; + if let Some(key_id) = key_id { + if let Some(key) = self.keys.get(&key_id) { + key.decrypt(&contents, nonce)? } else { - // File was not encrypted - contents + bail!("File was encrypted but we don't have tke key!") } - }; - - if contents.starts_with(&GZIP_MAGIC) { - let buff = Cursor::new(contents); - let mut decompressor = GzDecoder::new(buff); - let mut decompressed: Vec = vec![]; - decompressor.read_to_end(&mut decompressed)?; - Ok(decompressed) - } else if contents.starts_with(&ZSTD_MAGIC) { - let buff = Cursor::new(contents); - let mut decompressed: Vec = vec![]; - zstd::stream::copy_decode(buff, &mut decompressed)?; - Ok(decompressed) } else { - Ok(contents) + // File was not encrypted + contents } + }; + + if contents.starts_with(&GZIP_MAGIC) { + let buff = Cursor::new(contents); + let mut decompressor = GzDecoder::new(buff); + let mut decompressed: Vec = vec![]; + decompressor.read_to_end(&mut decompressed)?; + Ok(decompressed) + } else if contents.starts_with(&ZSTD_MAGIC) { + let buff = Cursor::new(contents); + let mut decompressed: Vec = vec![]; + zstd::stream::copy_decode(buff, &mut decompressed)?; + Ok(decompressed) } else { - bail!("files are not saved") + Ok(contents) } } diff --git a/src/cli/config.rs b/src/cli/config.rs index c5c2e0bd..306473cf 100644 --- a/src/cli/config.rs +++ b/src/cli/config.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +#[cfg(any(feature = "admin", feature = "admin-gui", feature = "vt"))] +use malwaredb_server::cloud::{CloudStorage, AWS}; #[cfg(any(feature = "admin", feature = "admin-gui", feature = "vt"))] use malwaredb_server::State; @@ -80,6 +82,21 @@ pub struct Config { #[cfg(feature = "vt")] #[serde(default, flatten)] pub vt_client: Option, + + /// AWS Access Key + #[arg(short, long)] + #[serde(default)] + pub access_key_id: Option, + + /// AWS Secret Access Key + #[arg(short, long)] + #[serde(default)] + pub secret_access_key: Option, + + /// AWS S3 bucket + #[arg(short, long)] + #[serde(default)] + pub bucket: Option, } const fn default_port() -> u16 { @@ -97,6 +114,20 @@ const fn default_max_upload_size() -> ByteSize { impl Config { #[cfg(any(feature = "admin", feature = "admin-gui", feature = "vt"))] pub async fn state(self) -> anyhow::Result { + let cloud = if self.access_key_id.is_some() + && self.secret_access_key.is_some() + && self.bucket.is_some() + { + let s3 = AWS::new( + self.access_key_id.unwrap(), + self.secret_access_key.unwrap(), + self.bucket.unwrap(), + ); + Some(CloudStorage::S3(s3)) + } else { + None + }; + #[allow(clippy::cast_possible_truncation)] // since we only aim to support 64-bit platforms State::new( self.port, @@ -109,6 +140,7 @@ impl Config { self.pg_cert, #[cfg(feature = "vt")] self.vt_client, + cloud, ) .await } @@ -206,6 +238,9 @@ impl Default for Config { cert: None, key: None, pg_cert: None, + access_key_id: None, + secret_access_key: None, + bucket: None, } } } diff --git a/src/cli/run.rs b/src/cli/run.rs index d38d4a06..efddf918 100644 --- a/src/cli/run.rs +++ b/src/cli/run.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 use super::config::Config; +use malwaredb_server::cloud::{CloudStorage, AWS}; use malwaredb_server::State; use std::process::ExitCode; @@ -28,6 +29,20 @@ impl Run { None => Config::from_found_files()?, }; + let cloud = if cfg.access_key_id.is_some() + && cfg.secret_access_key.is_some() + && cfg.bucket.is_some() + { + let s3 = AWS::new( + cfg.access_key_id.unwrap(), + cfg.secret_access_key.unwrap(), + cfg.bucket.unwrap(), + ); + Some(CloudStorage::S3(s3)) + } else { + None + }; + #[allow(clippy::cast_possible_truncation)] // since we aim to only support 64-bit platforms State::new( cfg.port, @@ -40,6 +55,7 @@ impl Run { cfg.pg_cert, #[cfg(feature = "vt")] cfg.vt_client, + cloud, ) .await } @@ -77,6 +93,7 @@ impl Load { } } +#[allow(clippy::large_enum_variant)] #[derive(Subcommand, Clone, Debug)] pub enum Subcommands { Load(Load), @@ -106,6 +123,7 @@ mod tests { None, #[cfg(feature = "vt")] None, + None, ) .await .unwrap(); diff --git a/src/cli/vt/mod.rs b/src/cli/vt/mod.rs index 9d8b7825..253b8e3c 100644 --- a/src/cli/vt/mod.rs +++ b/src/cli/vt/mod.rs @@ -35,6 +35,7 @@ impl VtOption { } /// Virus Total config either from disk or command line for VT operations outside the normal running of Malware DB +#[allow(clippy::large_enum_variant)] #[derive(Subcommand, Clone, Debug)] enum VtConfig { /// Load Virus Total config from a file diff --git a/src/main.rs b/src/main.rs index 0ade0ef3..07fb2d6a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,6 +35,9 @@ async fn main() -> anyhow::Result { "malwaredb_types", "deadpool_postgres", "postgres", + "aws_config", + "aws_sdk_s3", + "aws_smithy_async", #[cfg(feature = "sqlite")] "rusqlite", ]