Move index API failure handling to docbuilder

Author: Nemo157

src/bin/cratesfyi.rs

@@ -463,7 +463,7 @@ impl DatabaseSubcommand {
                 db::update_crate_data_in_database(
                     &*ctx.conn()?,
                     &name,
-                    &index.api().get_crate_data(&name),
+                    &index.api().get_crate_data(&name)?,
                 )?;
             }
 

src/docbuilder/rustwide_builder.rs

@@ -7,6 +7,7 @@ use crate::db::{
 };
 use crate::docbuilder::{crates::crates_from_path, Limits};
 use crate::error::Result;
+use crate::index::api::ReleaseData;
 use crate::storage::CompressionAlgorithms;
 use crate::storage::Storage;
 use crate::utils::{copy_doc_dir, parse_rustc_version, CargoMetadata};
@@ -399,6 +400,15 @@ impl RustwideBuilder {
                 } else {
                     crate::web::metrics::NON_LIBRARY_BUILDS.inc();
                 }
+
+                let release_data = match doc_builder.index.api().get_release_data(name, version) {
+                    Ok(data) => data,
+                    Err(err) => {
+                        warn!("{:#?}", err);
+                        ReleaseData::default()
+                    }
+                };
+
                 let release_id = add_package_into_database(
                     &conn,
                     res.cargo_metadata.root(),
@@ -407,7 +417,7 @@ impl RustwideBuilder {
                     &res.target,
                     files_list,
                     successful_targets,
-                    &doc_builder.index.api().get_release_data(name, version),
+                    &release_data,
                     has_docs,
                     has_examples,
                     algs,
@@ -416,11 +426,10 @@ impl RustwideBuilder {
                 add_build_into_database(&conn, release_id, &res.result)?;
 
                 // Some crates.io crate data is mutable, so we proactively update it during a release
-                update_crate_data_in_database(
-                    &conn,
-                    name,
-                    &doc_builder.index.api().get_crate_data(name),
-                )?;
+                match doc_builder.index.api().get_crate_data(name) {
+                    Ok(crate_data) => update_crate_data_in_database(&conn, name, &crate_data)?,
+                    Err(err) => warn!("{:#?}", err),
+                }
 
                 doc_builder.add_to_cache(name, version);
                 Ok(res)

src/index/api.rs

@@ -1,6 +1,5 @@
 use chrono::{DateTime, Utc};
-use failure::err_msg;
-use log::warn;
+use failure::{err_msg, ResultExt};
 use reqwest::header::{HeaderValue, ACCEPT, USER_AGENT};
 use semver::Version;
 use serde::Deserialize;
@@ -32,6 +31,16 @@ pub(crate) struct ReleaseData {
     pub(crate) downloads: i32,
 }
 
+impl Default for ReleaseData {
+    fn default() -> ReleaseData {
+        ReleaseData {
+            release_time: Utc::now(),
+            yanked: false,
+            downloads: 0,
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct CrateOwner {
     pub(crate) avatar: String,
@@ -62,28 +71,24 @@ impl Api {
             .ok_or_else(|| err_msg("index is missing an api base url"))
     }
 
-    pub fn get_crate_data(&self, name: &str) -> CrateData {
-        let owners = self.get_owners(name).unwrap_or_else(|err| {
-            warn!("Failed to get owners for {}: {}", name, err);
-            Vec::new()
-        });
+    pub fn get_crate_data(&self, name: &str) -> Result<CrateData> {
+        let owners = self
+            .get_owners(name)
+            .context(format!("Failed to get owners for {}", name))?;
 
-        CrateData { owners }
+        Ok(CrateData { owners })
     }
 
-    pub(crate) fn get_release_data(&self, name: &str, version: &str) -> ReleaseData {
+    pub(crate) fn get_release_data(&self, name: &str, version: &str) -> Result<ReleaseData> {
         let (release_time, yanked, downloads) = self
             .get_release_time_yanked_downloads(name, version)
-            .unwrap_or_else(|err| {
-                warn!("Failed to get crate data for {}-{}: {}", name, version, err);
-                (Utc::now(), false, 0)
-            });
+            .context(format!("Failed to get crate data for {}-{}", name, version))?;
 
-        ReleaseData {
+        Ok(ReleaseData {
             release_time,
             yanked,
             downloads,
-        }
+        })
     }
 
     /// Get release_time, yanked and downloads from the registry's API