Merge remote-tracking branch 'upstream/pr/816'

Author: carols10cents

src/krate.rs

@@ -40,6 +40,10 @@ use util::{RequestUtils, CargoResult, internal, ChainError, human};
 use version::{EncodableVersion, NewVersion};
 use {Model, User, Keyword, Version, Category, Badge, Replica};
 
+/// Hosts in this blacklist are known to not be hosting documentation,
+/// and are possibly of malicious intent e.g. ad tracking networks, etc.
+const DOCUMENTATION_BLACKLIST: [&'static str; 1] = ["rust-ci.org"];
+
 #[derive(Debug, Insertable, Queryable, Identifiable, Associations, AsChangeset)]
 #[belongs_to(Crate)]
 #[primary_key(crate_id, date)]
@@ -538,6 +542,8 @@ impl Crate {
         let keyword_ids = keywords.map(|kws| kws.iter().map(|kw| kw.keyword.clone()).collect());
         let category_ids = categories.map(|cats| cats.iter().map(|cat| cat.slug.clone()).collect());
         let badges = badges.map(|bs| bs.into_iter().map(|b| b.encodable()).collect());
+        let documentation = Crate::remove_blacklisted_documentation_urls(documentation);
+
         EncodableCrate {
             id: name.clone(),
             name: name.clone(),
@@ -566,6 +572,34 @@ impl Crate {
         }
     }
 
+    /// Return `None` if the documentation URL host matches a blacklisted host
+    fn remove_blacklisted_documentation_urls(url: Option<String>) -> Option<String> {
+        // Handles if documentation URL is None
+        let url = match url {
+            Some(url) => url,
+            None => return None,
+        };
+
+        // Handles unsuccessful parsing of documentation URL
+        let parsed_url = match Url::parse(&url) {
+            Ok(parsed_url) => parsed_url,
+            Err(_) => return None,
+        };
+
+        // Extract host string from documentation URL
+        let url_host = match parsed_url.host_str() {
+            Some(url_host) => url_host,
+            None => return None,
+        };
+
+        // Match documentation URL host against blacklisted host array elements
+        if DOCUMENTATION_BLACKLIST.contains(&url_host) {
+            None
+        } else {
+            Some(url)
+        }
+    }
+
     pub fn max_version(&self, conn: &PgConnection) -> CargoResult<semver::Version> {
         use schema::versions::dsl::*;
 
@@ -1594,3 +1628,41 @@ pub fn reverse_dependencies(req: &mut Request) -> CargoResult<Response> {
 use diesel::types::{Text, Date};
 sql_function!(canon_crate_name, canon_crate_name_t, (x: Text) -> Text);
 sql_function!(to_char, to_char_t, (a: Date, b: Text) -> Text);
+
+#[cfg(test)]
+mod tests {
+    use super::Crate;
+
+    #[test]
+    fn documentation_blacklist_no_url_provided() {
+        assert_eq!(Crate::remove_blacklisted_documentation_urls(None), None);
+    }
+
+    #[test]
+    fn documentation_blacklist_invalid_url() {
+        assert_eq!(
+            Crate::remove_blacklisted_documentation_urls(Some(String::from("not a url"))),
+            None
+        );
+    }
+
+    #[test]
+    fn documentation_blacklist_url_contains_partial_match() {
+        assert_eq!(
+            Crate::remove_blacklisted_documentation_urls(
+                Some(String::from("http://rust-ci.organists.com")),
+            ),
+            Some(String::from("http://rust-ci.organists.com"))
+        );
+    }
+
+    #[test]
+    fn documentation_blacklist_blacklisted_url() {
+        assert_eq!(
+            Crate::remove_blacklisted_documentation_urls(Some(String::from(
+                "http://rust-ci.org/crate/crate-0.1/doc/crate-0.1",
+            ))),
+            None
+        );
+    }
+}

src/tests/krate.rs

@@ -2094,3 +2094,24 @@ fn test_default_sort_recent() {
     assert_eq!(json.crates[1].recent_downloads, Some(0));
     assert_eq!(json.crates[1].downloads, 20);
 }
+
+#[test]
+fn block_blacklisted_documentation_url() {
+    let (_b, app, middle) = ::app();
+
+    let _ = {
+        let conn = app.diesel_database.get().unwrap();
+        let u = ::new_user("foo").create_or_update(&conn).unwrap();
+        ::CrateBuilder::new("foo_bad_doc_url", u.id)
+            .documentation(
+                "http://rust-ci.org/foo/foo_bad_doc_url/doc/foo_bad_doc_url/",
+            )
+            .expect_build(&conn)
+    };
+
+    let mut req = ::req(app, Method::Get, "/api/v1/crates/foo_bad_doc_url");
+    let mut response = ok_resp!(middle.call(&mut req));
+    let json: CrateResponse = ::json(&mut response);
+
+    assert_eq!(json.krate.documentation, None);
+}