Merge pull request #6834 from Turbo87/bye-bye-uploaders

Migrate remaining `Uploaders` code into `Storage` system

Author: Turbo87

src/admin/render_readmes.rs

@@ -1,8 +1,7 @@
 use crate::{
-    config, db,
+    db,
     models::Version,
     schema::{crates, readme_renderings, versions},
-    uploaders::Uploader,
 };
 use anyhow::{anyhow, Context};
 use std::{io::Read, path::Path, sync::Arc, thread};
@@ -40,7 +39,6 @@ pub struct Opts {
 }
 
 pub fn run(opts: Opts) -> anyhow::Result<()> {
-    let base_config = Arc::new(config::Base::from_environment());
     let storage = Arc::new(Storage::from_environment());
     let conn = &mut db::oneoff_connection().unwrap();
 
@@ -109,11 +107,10 @@ pub fn run(opts: Opts) -> anyhow::Result<()> {
                 .context("Couldn't record rendering time")?;
 
             let client = client.clone();
-            let base_config = base_config.clone();
             let storage = storage.clone();
             let handle = thread::spawn::<_, anyhow::Result<()>>(move || {
                 println!("[{}-{}] Rendering README...", krate_name, version.num);
-                let readme = get_readme(base_config.uploader(), &client, &version, &krate_name)?;
+                let readme = get_readme(&storage, &client, &version, &krate_name)?;
                 if !readme.is_empty() {
                     let rt = tokio::runtime::Builder::new_current_thread()
                         .enable_all()
@@ -143,19 +140,14 @@ pub fn run(opts: Opts) -> anyhow::Result<()> {
 
 /// Renders the readme of an uploaded crate version.
 fn get_readme(
-    uploader: &Uploader,
+    storage: &Storage,
     client: &Client,
     version: &Version,
     krate_name: &str,
 ) -> anyhow::Result<String> {
     let pkg_name = format!("{}-{}", krate_name, version.num);
 
-    let location = uploader.crate_location(krate_name, &version.num.to_string());
-
-    let location = match uploader {
-        Uploader::S3 { .. } => location,
-        Uploader::Local => format!("http://localhost:8888/{location}"),
-    };
+    let location = storage.crate_location(krate_name, &version.num.to_string());
 
     let mut extra_headers = header::HeaderMap::new();
     extra_headers.insert(

src/background_jobs.rs

@@ -10,7 +10,6 @@ use crate::db::ConnectionPool;
 use crate::storage::Storage;
 use crate::swirl::errors::EnqueueError;
 use crate::swirl::PerformError;
-use crate::uploaders::Uploader;
 use crate::worker;
 use crate::worker::cloudfront::CloudFront;
 use crate::worker::fastly::Fastly;
@@ -294,7 +293,6 @@ pub struct RenderAndUploadReadmeJob {
 
 pub struct Environment {
     index: Arc<Mutex<Repository>>,
-    pub uploader: Uploader,
     http_client: AssertUnwindSafe<Client>,
     cloudfront: Option<CloudFront>,
     fastly: Option<Fastly>,
@@ -304,15 +302,13 @@ pub struct Environment {
 impl Environment {
     pub fn new(
         index: Repository,
-        uploader: Uploader,
         http_client: Client,
         cloudfront: Option<CloudFront>,
         fastly: Option<Fastly>,
         storage: Arc<Storage>,
     ) -> Self {
         Self::new_shared(
             Arc::new(Mutex::new(index)),
-            uploader,
             http_client,
             cloudfront,
             fastly,
@@ -322,15 +318,13 @@ impl Environment {
 
     pub fn new_shared(
         index: Arc<Mutex<Repository>>,
-        uploader: Uploader,
         http_client: Client,
         cloudfront: Option<CloudFront>,
         fastly: Option<Fastly>,
         storage: Arc<Storage>,
     ) -> Self {
         Self {
             index,
-            uploader,
             http_client: AssertUnwindSafe(http_client),
             cloudfront,
             fastly,

src/bin/background-worker.rs

@@ -40,7 +40,6 @@ fn main() {
     info!("Booting runner");
 
     let config = config::Server::default();
-    let uploader = config.base.uploader();
 
     if config.db.are_all_read_only() {
         loop {
@@ -83,14 +82,7 @@ fn main() {
         .build()
         .expect("Couldn't build client");
 
-    let environment = Environment::new_shared(
-        repository,
-        uploader.clone(),
-        client,
-        cloudfront,
-        fastly,
-        storage,
-    );
+    let environment = Environment::new_shared(repository, client, cloudfront, fastly, storage);
 
     let environment = Arc::new(Some(environment));
 

src/config/base.rs

@@ -1,18 +1,11 @@
 //! Base configuration options
 //!
 //! - `HEROKU`: Is this instance of crates_io:: currently running on Heroku.
-//! - `S3_BUCKET`: The S3 bucket used to store crate files. If not present during development,
-//!    crates_io:: will fall back to a local uploader.
-//! - `S3_REGION`: The region in which the bucket was created. Optional if US standard.
-//! - `AWS_ACCESS_KEY`: The access key to interact with S3.
-//! - `AWS_SECRET_KEY`: The secret key to interact with S3.
-//! - `S3_CDN`: Optional CDN configuration for building public facing URLs.
 
-use crate::{env, uploaders::Uploader, Env};
+use crate::Env;
 
 pub struct Base {
     pub env: Env,
-    pub uploader: Uploader,
 }
 
 impl Base {
@@ -22,82 +15,6 @@ impl Base {
             _ => Env::Development,
         };
 
-        let uploader = if env == Env::Production {
-            // `env` panics if these vars are not set, and in production for a primary instance,
-            // that's what we want since we don't want to be able to start the server if the
-            // server doesn't know where to upload crates.
-            Self::s3_panic_if_missing_keys()
-        } else if dotenvy::var("S3_BUCKET").is_ok() {
-            // If we've set the `S3_BUCKET` variable to any value, use all of the values
-            // for the related S3 environment variables and configure the app to upload to
-            // and read from S3 like production does. All values except for bucket are
-            // optional, like production read-only mirrors.
-            info!("Using S3 uploader");
-            Self::s3_maybe_read_only()
-        } else {
-            // If we don't set the `S3_BUCKET` variable, we'll use a development-only
-            // uploader that makes it possible to run and publish to a locally-running
-            // crates.io instance without needing to set up an account and a bucket in S3.
-            info!("Using local uploader, crate files will be in the local_uploads directory");
-            Uploader::Local
-        };
-
-        Self { env, uploader }
-    }
-
-    pub fn uploader(&self) -> &Uploader {
-        &self.uploader
-    }
-
-    fn s3_panic_if_missing_keys() -> Uploader {
-        let index_bucket = match dotenvy::var("S3_INDEX_BUCKET") {
-            Ok(name) => Some(Box::new(s3::Bucket::new(
-                name,
-                dotenvy::var("S3_INDEX_REGION")
-                    .map_or_else(|_err| s3::Region::Default, s3::Region::Region),
-                env("AWS_ACCESS_KEY"),
-                env("AWS_SECRET_KEY"),
-                "https",
-            ))),
-            Err(_) => None,
-        };
-        Uploader::S3 {
-            bucket: Box::new(s3::Bucket::new(
-                env("S3_BUCKET"),
-                dotenvy::var("S3_REGION")
-                    .map_or_else(|_err| s3::Region::Default, s3::Region::Region),
-                env("AWS_ACCESS_KEY"),
-                env("AWS_SECRET_KEY"),
-                "https",
-            )),
-            index_bucket,
-            cdn: dotenvy::var("S3_CDN").ok(),
-        }
-    }
-
-    fn s3_maybe_read_only() -> Uploader {
-        let index_bucket = match dotenvy::var("S3_INDEX_BUCKET") {
-            Ok(name) => Some(Box::new(s3::Bucket::new(
-                name,
-                dotenvy::var("S3_INDEX_REGION")
-                    .map_or_else(|_err| s3::Region::Default, s3::Region::Region),
-                dotenvy::var("AWS_ACCESS_KEY").unwrap_or_default(),
-                dotenvy::var("AWS_SECRET_KEY").unwrap_or_default(),
-                "https",
-            ))),
-            Err(_) => None,
-        };
-        Uploader::S3 {
-            bucket: Box::new(s3::Bucket::new(
-                env("S3_BUCKET"),
-                dotenvy::var("S3_REGION")
-                    .map_or_else(|_err| s3::Region::Default, s3::Region::Region),
-                dotenvy::var("AWS_ACCESS_KEY").unwrap_or_default(),
-                dotenvy::var("AWS_SECRET_KEY").unwrap_or_default(),
-                "https",
-            )),
-            index_bucket,
-            cdn: dotenvy::var("S3_CDN").ok(),
-        }
+        Self { env }
     }
 }

src/config/server.rs

@@ -3,7 +3,7 @@ use ipnetwork::IpNetwork;
 use oauth2::{ClientId, ClientSecret};
 
 use crate::publish_rate_limit::PublishRateLimit;
-use crate::{env, env_optional, uploaders::Uploader, Env};
+use crate::{env, env_optional, Env};
 
 use super::base::Base;
 use super::database_pools::DatabasePools;
@@ -196,10 +196,6 @@ impl Server {
     pub fn env(&self) -> Env {
         self.base.env
     }
-
-    pub fn uploader(&self) -> &Uploader {
-        self.base.uploader()
-    }
 }
 
 pub(crate) fn domain_name() -> String {

src/controllers/krate/metadata.rs

@@ -312,7 +312,7 @@ pub async fn readme(
     Path((crate_name, version)): Path<(String, String)>,
     req: Parts,
 ) -> Response {
-    let redirect_url = app.config.uploader().readme_location(&crate_name, &version);
+    let redirect_url = app.storage.readme_location(&crate_name, &version);
     if req.wants_json() {
         Json(json!({ "url": redirect_url })).into_response()
     } else {

src/controllers/version/downloads.rs

@@ -125,7 +125,7 @@ pub async fn download(
         .await?
     };
 
-    let redirect_url = app.config.uploader().crate_location(&crate_name, &version);
+    let redirect_url = app.storage.crate_location(&crate_name, &version);
     if wants_json {
         Ok(Json(json!({ "url": redirect_url })).into_response())
     } else {

src/lib.rs

@@ -23,7 +23,7 @@ extern crate serde_json;
 #[macro_use]
 extern crate tracing;
 
-pub use crate::{app::App, email::Emails, uploaders::Uploader};
+pub use crate::{app::App, email::Emails};
 use std::str::FromStr;
 use std::sync::Arc;
 
@@ -52,7 +52,6 @@ pub mod sql;
 pub mod ssh;
 pub mod swirl;
 mod test_util;
-pub mod uploaders;
 pub mod util;
 pub mod worker;