From 369b4f4c2b671fb3985ef2948544ff08eaa23de3 Mon Sep 17 00:00:00 2001 From: Justin Geibel Date: Wed, 11 Oct 2017 18:53:07 -0400 Subject: [PATCH 01/10] Move krate::index to krate::search::search --- src/krate/mod.rs | 218 +----------------------------------------- src/krate/search.rs | 227 ++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 2 +- 3 files changed, 230 insertions(+), 217 deletions(-) create mode 100644 src/krate/search.rs diff --git a/src/krate/mod.rs b/src/krate/mod.rs index 242d500e4f4..adca85ec613 100644 --- a/src/krate/mod.rs +++ b/src/krate/mod.rs @@ -13,7 +13,6 @@ use diesel::pg::upsert::*; use diesel::pg::Pg; use diesel::prelude::*; use diesel; -use diesel_full_text_search::*; use license_exprs; use hex::ToHex; use serde_json; @@ -30,7 +29,6 @@ use git; use keyword::{CrateKeyword, EncodableKeyword}; use owner::{rights, CrateOwner, EncodableOwner, Owner, OwnerKind, Rights, Team}; use crate_owner_invitation::NewCrateOwnerInvitation; -use pagination::Paginate; use render; use schema::*; use upload; @@ -40,6 +38,8 @@ use util::{human, internal, CargoResult, ChainError, RequestUtils}; use version::{EncodableVersion, NewVersion}; use {Badge, Category, Keyword, Replica, User, Version}; +pub mod search; + /// Hosts in this blacklist are known to not be hosting documentation, /// and are possibly of malicious intent e.g. ad tracking networks, etc. const DOCUMENTATION_BLACKLIST: [&'static str; 1] = ["rust-ci.org"]; @@ -562,220 +562,6 @@ impl Crate { } } -/// Handles the `GET /crates` route. -/// Returns a list of crates. Called in a variety of scenarios in the -/// front end, including: -/// - Alphabetical listing of crates -/// - List of crates under a specific owner -/// - Listing a user's followed crates -/// -/// Notes: -/// The different use cases this function covers is handled through passing -/// in parameters in the GET request. -/// -/// We would like to stop adding functionality in here. It was built like -/// this to keep the number of database queries low, though given Rust's -/// low performance overhead, this is a soft goal to have, and can afford -/// more database transactions if it aids understandability. -/// -/// All of the edge cases for this function are not currently covered -/// in testing, and if they fail, it is difficult to determine what -/// caused the break. In the future, we should look at splitting this -/// function out to cover the different use cases, and create unit tests -/// for them. -pub fn index(req: &mut Request) -> CargoResult { - use diesel::expression::{AsExpression, DayAndMonthIntervalDsl}; - use diesel::types::{BigInt, Bool, Nullable}; - use diesel::expression::functions::date_and_time::{date, now}; - use diesel::expression::sql_literal::sql; - - let conn = req.db_conn()?; - let (offset, limit) = req.pagination(10, 100)?; - let params = req.query(); - let sort = params - .get("sort") - .map(|s| &**s) - .unwrap_or("recent-downloads"); - - let recent_downloads = sql::>("SUM(crate_downloads.downloads)"); - - let mut query = crates::table - .left_join( - crate_downloads::table.on( - crates::id - .eq(crate_downloads::crate_id) - .and(crate_downloads::date.gt(date(now - 90.days()))), - ), - ) - .group_by(crates::id) - .select(( - ALL_COLUMNS, - AsExpression::::as_expression(false), - recent_downloads.clone(), - )) - .into_boxed(); - - if sort == "downloads" { - query = query.order(crates::downloads.desc()) - } else if sort == "recent-downloads" { - query = query.order(recent_downloads.clone().desc().nulls_last()) - } else { - query = query.order(crates::name.asc()) - } - - if let Some(q_string) = params.get("q") { - let sort = params.get("sort").map(|s| &**s).unwrap_or("relevance"); - let q = plainto_tsquery(q_string); - query = query.filter( - q.matches(crates::textsearchable_index_col) - .or(Crate::name_canonically_equals(q_string)), - ); - - query = query.select(( - ALL_COLUMNS, - Crate::name_canonically_equals(q_string), - recent_downloads.clone(), - )); - let perfect_match = Crate::name_canonically_equals(q_string).desc(); - if sort == "downloads" { - query = query.order((perfect_match, crates::downloads.desc())); - } else if sort == "recent-downloads" { - query = query.order(( - perfect_match, - recent_downloads.clone().desc().nulls_last(), - )); - } else { - let rank = ts_rank_cd(crates::textsearchable_index_col, q); - query = query.order((perfect_match, rank.desc())) - } - } - - if let Some(cat) = params.get("category") { - query = query.filter( - crates::id.eq_any( - crates_categories::table - .select(crates_categories::crate_id) - .inner_join(categories::table) - .filter( - categories::slug - .eq(cat) - .or(categories::slug.like(format!("{}::%", cat))), - ), - ), - ); - } - - if let Some(kw) = params.get("keyword") { - query = query.filter( - crates::id.eq_any( - crates_keywords::table - .select(crates_keywords::crate_id) - .inner_join(keywords::table) - .filter(::lower(keywords::keyword).eq(::lower(kw))), - ), - ); - } else if let Some(letter) = params.get("letter") { - let pattern = format!( - "{}%", - letter - .chars() - .next() - .unwrap() - .to_lowercase() - .collect::() - ); - query = query.filter(canon_crate_name(crates::name).like(pattern)); - } else if let Some(user_id) = params.get("user_id").and_then(|s| s.parse::().ok()) { - query = query.filter( - crates::id.eq_any( - crate_owners::table - .select(crate_owners::crate_id) - .filter(crate_owners::owner_id.eq(user_id)) - .filter(crate_owners::deleted.eq(false)) - .filter(crate_owners::owner_kind.eq(OwnerKind::User as i32)), - ), - ); - } else if let Some(team_id) = params.get("team_id").and_then(|s| s.parse::().ok()) { - query = query.filter( - crates::id.eq_any( - crate_owners::table - .select(crate_owners::crate_id) - .filter(crate_owners::owner_id.eq(team_id)) - .filter(crate_owners::deleted.eq(false)) - .filter(crate_owners::owner_kind.eq(OwnerKind::Team as i32)), - ), - ); - } else if params.get("following").is_some() { - query = query.filter( - crates::id.eq_any( - follows::table - .select(follows::crate_id) - .filter(follows::user_id.eq(req.user()?.id)), - ), - ); - } - - // The database query returns a tuple within a tuple , with the root - // tuple containing 3 items. - let data = query - .paginate(limit, offset) - .load::<((Crate, bool, Option), i64)>(&*conn)?; - let total = data.first().map(|&(_, t)| t).unwrap_or(0); - let crates = data.iter() - .map(|&((ref c, _, _), _)| c.clone()) - .collect::>(); - let perfect_matches = data.clone() - .into_iter() - .map(|((_, b, _), _)| b) - .collect::>(); - let recent_downloads = data.clone() - .into_iter() - .map(|((_, _, s), _)| s.unwrap_or(0)) - .collect::>(); - - let versions = Version::belonging_to(&crates) - .load::(&*conn)? - .grouped_by(&crates) - .into_iter() - .map(|versions| Version::max(versions.into_iter().map(|v| v.num))); - - let crates = versions - .zip(crates) - .zip(perfect_matches) - .zip(recent_downloads) - .map( - |(((max_version, krate), perfect_match), recent_downloads)| { - // FIXME: If we add crate_id to the Badge enum we can eliminate - // this N+1 - let badges = badges::table - .filter(badges::crate_id.eq(krate.id)) - .load::(&*conn)?; - Ok(krate.minimal_encodable( - max_version, - Some(badges), - perfect_match, - Some(recent_downloads), - )) - }, - ) - .collect::>()?; - - #[derive(Serialize)] - struct R { - crates: Vec, - meta: Meta, - } - #[derive(Serialize)] - struct Meta { - total: i64, - } - - Ok(req.json(&R { - crates: crates, - meta: Meta { total: total }, - })) -} - /// Handles the `GET /summary` route. pub fn summary(req: &mut Request) -> CargoResult { use diesel::expression::{date, now, sql, DayAndMonthIntervalDsl}; diff --git a/src/krate/search.rs b/src/krate/search.rs new file mode 100644 index 00000000000..3498ad01c57 --- /dev/null +++ b/src/krate/search.rs @@ -0,0 +1,227 @@ +use conduit::{Request, Response}; +use diesel::prelude::*; +use diesel_full_text_search::*; + +use db::RequestTransaction; +use owner::OwnerKind; +use pagination::Paginate; +use schema::*; +use user::RequestUser; +use util::{CargoResult, RequestUtils}; +use {Badge, Version}; + +use super::{canon_crate_name, Crate, EncodableCrate, ALL_COLUMNS}; + +/// Handles the `GET /crates` route. +/// Returns a list of crates. Called in a variety of scenarios in the +/// front end, including: +/// - Alphabetical listing of crates +/// - List of crates under a specific owner +/// - Listing a user's followed crates +/// +/// Notes: +/// The different use cases this function covers is handled through passing +/// in parameters in the GET request. +/// +/// We would like to stop adding functionality in here. It was built like +/// this to keep the number of database queries low, though given Rust's +/// low performance overhead, this is a soft goal to have, and can afford +/// more database transactions if it aids understandability. +/// +/// All of the edge cases for this function are not currently covered +/// in testing, and if they fail, it is difficult to determine what +/// caused the break. In the future, we should look at splitting this +/// function out to cover the different use cases, and create unit tests +/// for them. +pub fn search(req: &mut Request) -> CargoResult { + use diesel::expression::{AsExpression, DayAndMonthIntervalDsl}; + use diesel::types::{BigInt, Bool, Nullable}; + use diesel::expression::functions::date_and_time::{date, now}; + use diesel::expression::sql_literal::sql; + + let conn = req.db_conn()?; + let (offset, limit) = req.pagination(10, 100)?; + let params = req.query(); + let sort = params + .get("sort") + .map(|s| &**s) + .unwrap_or("recent-downloads"); + + let recent_downloads = sql::>("SUM(crate_downloads.downloads)"); + + let mut query = crates::table + .left_join( + crate_downloads::table.on( + crates::id + .eq(crate_downloads::crate_id) + .and(crate_downloads::date.gt(date(now - 90.days()))), + ), + ) + .group_by(crates::id) + .select(( + ALL_COLUMNS, + AsExpression::::as_expression(false), + recent_downloads.clone(), + )) + .into_boxed(); + + if sort == "downloads" { + query = query.order(crates::downloads.desc()) + } else if sort == "recent-downloads" { + query = query.order(recent_downloads.clone().desc().nulls_last()) + } else { + query = query.order(crates::name.asc()) + } + + if let Some(q_string) = params.get("q") { + let sort = params.get("sort").map(|s| &**s).unwrap_or("relevance"); + let q = plainto_tsquery(q_string); + query = query.filter( + q.matches(crates::textsearchable_index_col) + .or(Crate::name_canonically_equals(q_string)), + ); + + query = query.select(( + ALL_COLUMNS, + Crate::name_canonically_equals(q_string), + recent_downloads.clone(), + )); + let perfect_match = Crate::name_canonically_equals(q_string).desc(); + if sort == "downloads" { + query = query.order((perfect_match, crates::downloads.desc())); + } else if sort == "recent-downloads" { + query = query.order(( + perfect_match, + recent_downloads.clone().desc().nulls_last(), + )); + } else { + let rank = ts_rank_cd(crates::textsearchable_index_col, q); + query = query.order((perfect_match, rank.desc())) + } + } + + if let Some(cat) = params.get("category") { + query = query.filter( + crates::id.eq_any( + crates_categories::table + .select(crates_categories::crate_id) + .inner_join(categories::table) + .filter( + categories::slug + .eq(cat) + .or(categories::slug.like(format!("{}::%", cat))), + ), + ), + ); + } + + if let Some(kw) = params.get("keyword") { + query = query.filter( + crates::id.eq_any( + crates_keywords::table + .select(crates_keywords::crate_id) + .inner_join(keywords::table) + .filter(::lower(keywords::keyword).eq(::lower(kw))), + ), + ); + } else if let Some(letter) = params.get("letter") { + let pattern = format!( + "{}%", + letter + .chars() + .next() + .unwrap() + .to_lowercase() + .collect::() + ); + query = query.filter(canon_crate_name(crates::name).like(pattern)); + } else if let Some(user_id) = params.get("user_id").and_then(|s| s.parse::().ok()) { + query = query.filter( + crates::id.eq_any( + crate_owners::table + .select(crate_owners::crate_id) + .filter(crate_owners::owner_id.eq(user_id)) + .filter(crate_owners::deleted.eq(false)) + .filter(crate_owners::owner_kind.eq(OwnerKind::User as i32)), + ), + ); + } else if let Some(team_id) = params.get("team_id").and_then(|s| s.parse::().ok()) { + query = query.filter( + crates::id.eq_any( + crate_owners::table + .select(crate_owners::crate_id) + .filter(crate_owners::owner_id.eq(team_id)) + .filter(crate_owners::deleted.eq(false)) + .filter(crate_owners::owner_kind.eq(OwnerKind::Team as i32)), + ), + ); + } else if params.get("following").is_some() { + query = query.filter( + crates::id.eq_any( + follows::table + .select(follows::crate_id) + .filter(follows::user_id.eq(req.user()?.id)), + ), + ); + } + + // The database query returns a tuple within a tuple , with the root + // tuple containing 3 items. + let data = query + .paginate(limit, offset) + .load::<((Crate, bool, Option), i64)>(&*conn)?; + let total = data.first().map(|&(_, t)| t).unwrap_or(0); + let crates = data.iter() + .map(|&((ref c, _, _), _)| c.clone()) + .collect::>(); + let perfect_matches = data.clone() + .into_iter() + .map(|((_, b, _), _)| b) + .collect::>(); + let recent_downloads = data.clone() + .into_iter() + .map(|((_, _, s), _)| s.unwrap_or(0)) + .collect::>(); + + let versions = Version::belonging_to(&crates) + .load::(&*conn)? + .grouped_by(&crates) + .into_iter() + .map(|versions| Version::max(versions.into_iter().map(|v| v.num))); + + let crates = versions + .zip(crates) + .zip(perfect_matches) + .zip(recent_downloads) + .map( + |(((max_version, krate), perfect_match), recent_downloads)| { + // FIXME: If we add crate_id to the Badge enum we can eliminate + // this N+1 + let badges = badges::table + .filter(badges::crate_id.eq(krate.id)) + .load::(&*conn)?; + Ok(krate.minimal_encodable( + max_version, + Some(badges), + perfect_match, + Some(recent_downloads), + )) + }, + ) + .collect::>()?; + + #[derive(Serialize)] + struct R { + crates: Vec, + meta: Meta, + } + #[derive(Serialize)] + struct Meta { + total: i64, + } + + Ok(req.json(&R { + crates: crates, + meta: Meta { total: total }, + })) +} diff --git a/src/lib.rs b/src/lib.rs index c8dbcd37104..343ef7a0f08 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -138,7 +138,7 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder { let mut api_router = RouteBuilder::new(); // Route used by both `cargo search` and the frontend - api_router.get("/crates", C(krate::index)); + api_router.get("/crates", C(krate::search::search)); // Routes used by `cargo` api_router.put("/crates/new", C(krate::new)); From 915a0d0e8cf6549e4186a5b7e8e893d8cbe4f13b Mon Sep 17 00:00:00 2001 From: Justin Geibel Date: Wed, 11 Oct 2017 19:01:17 -0400 Subject: [PATCH 02/10] Move krate::new to krate::publish::publish --- src/krate/mod.rs | 216 +---------------------------------------- src/krate/publish.rs | 224 +++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 2 +- 3 files changed, 229 insertions(+), 213 deletions(-) create mode 100644 src/krate/publish.rs diff --git a/src/krate/mod.rs b/src/krate/mod.rs index adca85ec613..9356f63f509 100644 --- a/src/krate/mod.rs +++ b/src/krate/mod.rs @@ -1,7 +1,5 @@ use std::ascii::AsciiExt; use std::cmp; -use std::collections::HashMap; -use std::sync::Arc; use chrono::{NaiveDate, NaiveDateTime}; use conduit::{Request, Response}; @@ -14,7 +12,6 @@ use diesel::pg::Pg; use diesel::prelude::*; use diesel; use license_exprs; -use hex::ToHex; use serde_json; use semver; use url::Url; @@ -23,22 +20,19 @@ use app::{App, RequestApp}; use badge::EncodableBadge; use category::{CrateCategory, EncodableCategory}; use db::RequestTransaction; -use dependency::{self, EncodableDependency, ReverseDependency}; +use dependency::{EncodableDependency, ReverseDependency}; use download::{EncodableVersionDownload, VersionDownload}; -use git; use keyword::{CrateKeyword, EncodableKeyword}; use owner::{rights, CrateOwner, EncodableOwner, Owner, OwnerKind, Rights, Team}; use crate_owner_invitation::NewCrateOwnerInvitation; -use render; use schema::*; -use upload; use user::RequestUser; -use util::{read_fill, read_le_u32}; -use util::{human, internal, CargoResult, ChainError, RequestUtils}; -use version::{EncodableVersion, NewVersion}; +use util::{human, CargoResult, RequestUtils}; +use version::EncodableVersion; use {Badge, Category, Keyword, Replica, User, Version}; pub mod search; +pub mod publish; /// Hosts in this blacklist are known to not be hosting documentation, /// and are possibly of malicious intent e.g. ad tracking networks, etc. @@ -713,208 +707,6 @@ pub fn show(req: &mut Request) -> CargoResult { ) } -/// Handles the `PUT /crates/new` route. -/// Used by `cargo publish` to publish a new crate or to publish a new version of an -/// existing crate. -/// -/// Currently blocks the HTTP thread, perhaps some function calls can spawn new -/// threads and return completion or error through other methods a `cargo publish -/// --status` command, via crates.io's front end, or email. -pub fn new(req: &mut Request) -> CargoResult { - let app = Arc::clone(req.app()); - let (new_crate, user) = parse_new_headers(req)?; - - let name = &*new_crate.name; - let vers = &*new_crate.vers; - let features = new_crate - .features - .iter() - .map(|(k, v)| { - ( - k[..].to_string(), - v.iter().map(|v| v[..].to_string()).collect(), - ) - }) - .collect::>>(); - let keywords = new_crate - .keywords - .as_ref() - .map(|kws| kws.iter().map(|kw| &**kw).collect()) - .unwrap_or_else(Vec::new); - - let categories = new_crate.categories.as_ref().map(|s| &s[..]).unwrap_or(&[]); - let categories: Vec<_> = categories.iter().map(|k| &**k).collect(); - - let conn = req.db_conn()?; - // Create a transaction on the database, if there are no errors, - // commit the transactions to record a new or updated crate. - conn.transaction(|| { - // Persist the new crate, if it doesn't already exist - let persist = NewCrate { - name: name, - description: new_crate.description.as_ref().map(|s| &**s), - homepage: new_crate.homepage.as_ref().map(|s| &**s), - documentation: new_crate.documentation.as_ref().map(|s| &**s), - readme: new_crate.readme.as_ref().map(|s| &**s), - repository: new_crate.repository.as_ref().map(|s| &**s), - license: new_crate.license.as_ref().map(|s| &**s), - max_upload_size: None, - }; - - let license_file = new_crate.license_file.as_ref().map(|s| &**s); - let krate = persist.create_or_update(&conn, license_file, user.id)?; - - let owners = krate.owners(&conn)?; - if rights(req.app(), &owners, &user)? < Rights::Publish { - return Err(human( - "this crate exists but you don't seem to be an owner. \ - If you believe this is a mistake, perhaps you need \ - to accept an invitation to be an owner before \ - publishing.", - )); - } - - if krate.name != name { - return Err(human( - &format_args!("crate was previously named `{}`", krate.name), - )); - } - - let length = req.content_length() - .chain_error(|| human("missing header: Content-Length"))?; - let max = krate - .max_upload_size - .map(|m| m as u64) - .unwrap_or(app.config.max_upload_size); - if length > max { - return Err(human(&format_args!("max upload size is: {}", max))); - } - - // This is only redundant for now. Eventually the duplication will be removed. - let license = new_crate.license.clone(); - - // Persist the new version of this crate - let version = NewVersion::new(krate.id, vers, &features, license, license_file)? - .save(&conn, &new_crate.authors)?; - - // Link this new version to all dependencies - let git_deps = dependency::add_dependencies(&conn, &new_crate.deps, version.id)?; - - // Update all keywords for this crate - Keyword::update_crate(&conn, &krate, &keywords)?; - - // Update all categories for this crate, collecting any invalid categories - // in order to be able to warn about them - let ignored_invalid_categories = Category::update_crate(&conn, &krate, &categories)?; - - // Update all badges for this crate, collecting any invalid badges in - // order to be able to warn about them - let ignored_invalid_badges = Badge::update_crate(&conn, &krate, new_crate.badges.as_ref())?; - let max_version = krate.max_version(&conn)?; - - // Render the README for this crate - let readme = match new_crate.readme.as_ref() { - Some(readme) => Some(render::markdown_to_html(&**readme)?), - None => None, - }; - - // Upload the crate, return way to delete the crate from the server - // If the git commands fail below, we shouldn't keep the crate on the - // server. - let (cksum, mut crate_bomb, mut readme_bomb) = - app.config - .uploader - .upload_crate(req, &krate, readme, max, vers)?; - version.record_readme_rendering(&conn)?; - - // Register this crate in our local git repo. - let git_crate = git::Crate { - name: name.to_string(), - vers: vers.to_string(), - cksum: cksum.to_hex(), - features: features, - deps: git_deps, - yanked: Some(false), - }; - git::add_crate(&**req.app(), &git_crate).chain_error(|| { - internal(&format_args!( - "could not add crate `{}` to the git repo", - name - )) - })?; - - // Now that we've come this far, we're committed! - crate_bomb.path = None; - readme_bomb.path = None; - - #[derive(Serialize)] - struct Warnings<'a> { - invalid_categories: Vec<&'a str>, - invalid_badges: Vec<&'a str>, - } - let warnings = Warnings { - invalid_categories: ignored_invalid_categories, - invalid_badges: ignored_invalid_badges, - }; - - #[derive(Serialize)] - struct R<'a> { - #[serde(rename = "crate")] krate: EncodableCrate, - warnings: Warnings<'a>, - } - Ok(req.json(&R { - krate: krate.minimal_encodable(max_version, None, false, None), - warnings: warnings, - })) - }) -} - -/// Used by the `krate::new` function. -/// -/// This function parses the JSON headers to interpret the data and validates -/// the data during and after the parsing. Returns crate metadata and user -/// information. -fn parse_new_headers(req: &mut Request) -> CargoResult<(upload::NewCrate, User)> { - // Read the json upload request - let amt = u64::from(read_le_u32(req.body())?); - let max = req.app().config.max_upload_size; - if amt > max { - return Err(human(&format_args!("max upload size is: {}", max))); - } - let mut json = vec![0; amt as usize]; - read_fill(req.body(), &mut json)?; - let json = String::from_utf8(json).map_err(|_| human("json body was not valid utf-8"))?; - let new: upload::NewCrate = serde_json::from_str(&json) - .map_err(|e| human(&format_args!("invalid upload request: {}", e)))?; - - // Make sure required fields are provided - fn empty(s: Option<&String>) -> bool { - s.map_or(true, |s| s.is_empty()) - } - let mut missing = Vec::new(); - - if empty(new.description.as_ref()) { - missing.push("description"); - } - if empty(new.license.as_ref()) && empty(new.license_file.as_ref()) { - missing.push("license"); - } - if new.authors.iter().all(|s| s.is_empty()) { - missing.push("authors"); - } - if !missing.is_empty() { - return Err(human(&format_args!( - "missing or empty metadata fields: {}. Please \ - see http://doc.crates.io/manifest.html#package-metadata for \ - how to upload metadata", - missing.join(", ") - ))); - } - - let user = req.user()?; - Ok((new, user.clone())) -} - /// Handles the `GET /crates/:crate_id/:version/download` route. /// This returns a URL to the location where the crate is stored. pub fn download(req: &mut Request) -> CargoResult { diff --git a/src/krate/publish.rs b/src/krate/publish.rs new file mode 100644 index 00000000000..d600bc46c2a --- /dev/null +++ b/src/krate/publish.rs @@ -0,0 +1,224 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use conduit::{Request, Response}; +use diesel::prelude::*; +use hex::ToHex; +use serde_json; + +use app::RequestApp; +use db::RequestTransaction; +use dependency; +use git; +use owner::{rights, Rights}; +use render; +use upload; +use user::RequestUser; +use util::{read_fill, read_le_u32}; +use util::{human, internal, CargoResult, ChainError, RequestUtils}; +use version::NewVersion; +use {Badge, Category, Keyword, User}; + +use super::{EncodableCrate, NewCrate}; + +/// Handles the `PUT /crates/new` route. +/// Used by `cargo publish` to publish a new crate or to publish a new version of an +/// existing crate. +/// +/// Currently blocks the HTTP thread, perhaps some function calls can spawn new +/// threads and return completion or error through other methods a `cargo publish +/// --status` command, via crates.io's front end, or email. +pub fn publish(req: &mut Request) -> CargoResult { + let app = Arc::clone(req.app()); + let (new_crate, user) = parse_new_headers(req)?; + + let name = &*new_crate.name; + let vers = &*new_crate.vers; + let features = new_crate + .features + .iter() + .map(|(k, v)| { + ( + k[..].to_string(), + v.iter().map(|v| v[..].to_string()).collect(), + ) + }) + .collect::>>(); + let keywords = new_crate + .keywords + .as_ref() + .map(|kws| kws.iter().map(|kw| &**kw).collect()) + .unwrap_or_else(Vec::new); + + let categories = new_crate.categories.as_ref().map(|s| &s[..]).unwrap_or(&[]); + let categories: Vec<_> = categories.iter().map(|k| &**k).collect(); + + let conn = req.db_conn()?; + // Create a transaction on the database, if there are no errors, + // commit the transactions to record a new or updated crate. + conn.transaction(|| { + // Persist the new crate, if it doesn't already exist + let persist = NewCrate { + name: name, + description: new_crate.description.as_ref().map(|s| &**s), + homepage: new_crate.homepage.as_ref().map(|s| &**s), + documentation: new_crate.documentation.as_ref().map(|s| &**s), + readme: new_crate.readme.as_ref().map(|s| &**s), + repository: new_crate.repository.as_ref().map(|s| &**s), + license: new_crate.license.as_ref().map(|s| &**s), + max_upload_size: None, + }; + + let license_file = new_crate.license_file.as_ref().map(|s| &**s); + let krate = persist.create_or_update(&conn, license_file, user.id)?; + + let owners = krate.owners(&conn)?; + if rights(req.app(), &owners, &user)? < Rights::Publish { + return Err(human( + "this crate exists but you don't seem to be an owner. \ + If you believe this is a mistake, perhaps you need \ + to accept an invitation to be an owner before \ + publishing.", + )); + } + + if krate.name != name { + return Err(human( + &format_args!("crate was previously named `{}`", krate.name), + )); + } + + let length = req.content_length() + .chain_error(|| human("missing header: Content-Length"))?; + let max = krate + .max_upload_size + .map(|m| m as u64) + .unwrap_or(app.config.max_upload_size); + if length > max { + return Err(human(&format_args!("max upload size is: {}", max))); + } + + // This is only redundant for now. Eventually the duplication will be removed. + let license = new_crate.license.clone(); + + // Persist the new version of this crate + let version = NewVersion::new(krate.id, vers, &features, license, license_file)? + .save(&conn, &new_crate.authors)?; + + // Link this new version to all dependencies + let git_deps = dependency::add_dependencies(&conn, &new_crate.deps, version.id)?; + + // Update all keywords for this crate + Keyword::update_crate(&conn, &krate, &keywords)?; + + // Update all categories for this crate, collecting any invalid categories + // in order to be able to warn about them + let ignored_invalid_categories = Category::update_crate(&conn, &krate, &categories)?; + + // Update all badges for this crate, collecting any invalid badges in + // order to be able to warn about them + let ignored_invalid_badges = Badge::update_crate(&conn, &krate, new_crate.badges.as_ref())?; + let max_version = krate.max_version(&conn)?; + + // Render the README for this crate + let readme = match new_crate.readme.as_ref() { + Some(readme) => Some(render::markdown_to_html(&**readme)?), + None => None, + }; + + // Upload the crate, return way to delete the crate from the server + // If the git commands fail below, we shouldn't keep the crate on the + // server. + let (cksum, mut crate_bomb, mut readme_bomb) = + app.config + .uploader + .upload_crate(req, &krate, readme, max, vers)?; + version.record_readme_rendering(&conn)?; + + // Register this crate in our local git repo. + let git_crate = git::Crate { + name: name.to_string(), + vers: vers.to_string(), + cksum: cksum.to_hex(), + features: features, + deps: git_deps, + yanked: Some(false), + }; + git::add_crate(&**req.app(), &git_crate).chain_error(|| { + internal(&format_args!( + "could not add crate `{}` to the git repo", + name + )) + })?; + + // Now that we've come this far, we're committed! + crate_bomb.path = None; + readme_bomb.path = None; + + #[derive(Serialize)] + struct Warnings<'a> { + invalid_categories: Vec<&'a str>, + invalid_badges: Vec<&'a str>, + } + let warnings = Warnings { + invalid_categories: ignored_invalid_categories, + invalid_badges: ignored_invalid_badges, + }; + + #[derive(Serialize)] + struct R<'a> { + #[serde(rename = "crate")] krate: EncodableCrate, + warnings: Warnings<'a>, + } + Ok(req.json(&R { + krate: krate.minimal_encodable(max_version, None, false, None), + warnings: warnings, + })) + }) +} + +/// Used by the `krate::new` function. +/// +/// This function parses the JSON headers to interpret the data and validates +/// the data during and after the parsing. Returns crate metadata and user +/// information. +fn parse_new_headers(req: &mut Request) -> CargoResult<(upload::NewCrate, User)> { + // Read the json upload request + let amt = u64::from(read_le_u32(req.body())?); + let max = req.app().config.max_upload_size; + if amt > max { + return Err(human(&format_args!("max upload size is: {}", max))); + } + let mut json = vec![0; amt as usize]; + read_fill(req.body(), &mut json)?; + let json = String::from_utf8(json).map_err(|_| human("json body was not valid utf-8"))?; + let new: upload::NewCrate = serde_json::from_str(&json) + .map_err(|e| human(&format_args!("invalid upload request: {}", e)))?; + + // Make sure required fields are provided + fn empty(s: Option<&String>) -> bool { + s.map_or(true, |s| s.is_empty()) + } + let mut missing = Vec::new(); + + if empty(new.description.as_ref()) { + missing.push("description"); + } + if empty(new.license.as_ref()) && empty(new.license_file.as_ref()) { + missing.push("license"); + } + if new.authors.iter().all(|s| s.is_empty()) { + missing.push("authors"); + } + if !missing.is_empty() { + return Err(human(&format_args!( + "missing or empty metadata fields: {}. Please \ + see http://doc.crates.io/manifest.html#package-metadata for \ + how to upload metadata", + missing.join(", ") + ))); + } + + let user = req.user()?; + Ok((new, user.clone())) +} diff --git a/src/lib.rs b/src/lib.rs index 343ef7a0f08..957a87d250f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -141,7 +141,7 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder { api_router.get("/crates", C(krate::search::search)); // Routes used by `cargo` - api_router.put("/crates/new", C(krate::new)); + api_router.put("/crates/new", C(krate::publish::publish)); api_router.get("/crates/:crate_id/owners", C(krate::owners)); api_router.put("/crates/:crate_id/owners", C(krate::add_owners)); api_router.delete("/crates/:crate_id/owners", C(krate::remove_owners)); From 72dddcdfe02be343e6777c97e323359d5c000254 Mon Sep 17 00:00:00 2001 From: Justin Geibel Date: Wed, 11 Oct 2017 19:08:45 -0400 Subject: [PATCH 03/10] Move all ownership related routes to krate::owner::* --- src/krate/mod.rs | 132 +--------------------------------------- src/krate/owners.rs | 143 ++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 10 ++-- 3 files changed, 150 insertions(+), 135 deletions(-) create mode 100644 src/krate/owners.rs diff --git a/src/krate/mod.rs b/src/krate/mod.rs index 9356f63f509..4d75bdc3cd5 100644 --- a/src/krate/mod.rs +++ b/src/krate/mod.rs @@ -12,7 +12,6 @@ use diesel::pg::Pg; use diesel::prelude::*; use diesel; use license_exprs; -use serde_json; use semver; use url::Url; @@ -23,7 +22,7 @@ use db::RequestTransaction; use dependency::{EncodableDependency, ReverseDependency}; use download::{EncodableVersionDownload, VersionDownload}; use keyword::{CrateKeyword, EncodableKeyword}; -use owner::{rights, CrateOwner, EncodableOwner, Owner, OwnerKind, Rights, Team}; +use owner::{CrateOwner, Owner, OwnerKind}; use crate_owner_invitation::NewCrateOwnerInvitation; use schema::*; use user::RequestUser; @@ -33,6 +32,7 @@ use {Badge, Category, Keyword, Replica, User, Version}; pub mod search; pub mod publish; +pub mod owners; /// Hosts in this blacklist are known to not be hosting documentation, /// and are possibly of malicious intent e.g. ad tracking networks, etc. @@ -914,134 +914,6 @@ pub fn versions(req: &mut Request) -> CargoResult { Ok(req.json(&R { versions: versions })) } -/// Handles the `GET /crates/:crate_id/owners` route. -pub fn owners(req: &mut Request) -> CargoResult { - let crate_name = &req.params()["crate_id"]; - let conn = req.db_conn()?; - let krate = Crate::by_name(crate_name).first::(&*conn)?; - let owners = krate - .owners(&conn)? - .into_iter() - .map(Owner::encodable) - .collect(); - - #[derive(Serialize)] - struct R { - users: Vec, - } - Ok(req.json(&R { users: owners })) -} - -/// Handles the `GET /crates/:crate_id/owner_team` route. -pub fn owner_team(req: &mut Request) -> CargoResult { - let crate_name = &req.params()["crate_id"]; - let conn = req.db_conn()?; - let krate = Crate::by_name(crate_name).first::(&*conn)?; - let owners = Team::owning(&krate, &conn)? - .into_iter() - .map(Owner::encodable) - .collect(); - - #[derive(Serialize)] - struct R { - teams: Vec, - } - Ok(req.json(&R { teams: owners })) -} - -/// Handles the `GET /crates/:crate_id/owner_user` route. -pub fn owner_user(req: &mut Request) -> CargoResult { - let crate_name = &req.params()["crate_id"]; - let conn = req.db_conn()?; - let krate = Crate::by_name(crate_name).first::(&*conn)?; - let owners = User::owning(&krate, &conn)? - .into_iter() - .map(Owner::encodable) - .collect(); - - #[derive(Serialize)] - struct R { - users: Vec, - } - Ok(req.json(&R { users: owners })) -} - -/// Handles the `PUT /crates/:crate_id/owners` route. -pub fn add_owners(req: &mut Request) -> CargoResult { - modify_owners(req, true) -} - -/// Handles the `DELETE /crates/:crate_id/owners` route. -pub fn remove_owners(req: &mut Request) -> CargoResult { - modify_owners(req, false) -} - -fn modify_owners(req: &mut Request, add: bool) -> CargoResult { - let mut body = String::new(); - req.body().read_to_string(&mut body)?; - - let user = req.user()?; - let conn = req.db_conn()?; - let krate = Crate::by_name(&req.params()["crate_id"]).first::(&*conn)?; - let owners = krate.owners(&conn)?; - - match rights(req.app(), &owners, user)? { - Rights::Full => {} - // Yes! - Rights::Publish => { - return Err(human("team members don't have permission to modify owners")); - } - Rights::None => { - return Err(human("only owners have permission to modify owners")); - } - } - - #[derive(Deserialize)] - struct Request { - // identical, for back-compat (owners preferred) - users: Option>, - owners: Option>, - } - - let request: Request = serde_json::from_str(&body).map_err(|_| human("invalid json request"))?; - - let logins = request - .owners - .or(request.users) - .ok_or_else(|| human("invalid json request"))?; - - let mut msgs = Vec::new(); - - for login in &logins { - if add { - if owners.iter().any(|owner| owner.login() == *login) { - return Err(human(&format_args!("`{}` is already an owner", login))); - } - let msg = krate.owner_add(req.app(), &conn, user, login)?; - msgs.push(msg); - } else { - // Removing the team that gives you rights is prevented because - // team members only have Rights::Publish - if owners.len() == 1 { - return Err(human("cannot remove the sole owner of a crate")); - } - krate.owner_remove(req.app(), &conn, user, login)?; - } - } - - let comma_sep_msg = msgs.join(","); - - #[derive(Serialize)] - struct R { - ok: bool, - msg: String, - } - Ok(req.json(&R { - ok: true, - msg: comma_sep_msg, - })) -} - /// Handles the `GET /crates/:crate_id/reverse_dependencies` route. pub fn reverse_dependencies(req: &mut Request) -> CargoResult { use diesel::expression::dsl::any; diff --git a/src/krate/owners.rs b/src/krate/owners.rs new file mode 100644 index 00000000000..36e9ee595bd --- /dev/null +++ b/src/krate/owners.rs @@ -0,0 +1,143 @@ +//! All routes related to managing owners of a crate + +use conduit::{Request, Response}; +use conduit_router::RequestParams; +use diesel::prelude::*; +use serde_json; + +use app::RequestApp; +use db::RequestTransaction; +use owner::{rights, EncodableOwner, Owner, Rights, Team}; +use user::RequestUser; +use util::{human, CargoResult, RequestUtils}; +use User; + +use super::Crate; + +/// Handles the `GET /crates/:crate_id/owners` route. +pub fn owners(req: &mut Request) -> CargoResult { + let crate_name = &req.params()["crate_id"]; + let conn = req.db_conn()?; + let krate = Crate::by_name(crate_name).first::(&*conn)?; + let owners = krate + .owners(&conn)? + .into_iter() + .map(Owner::encodable) + .collect(); + + #[derive(Serialize)] + struct R { + users: Vec, + } + Ok(req.json(&R { users: owners })) +} + +/// Handles the `GET /crates/:crate_id/owner_team` route. +pub fn owner_team(req: &mut Request) -> CargoResult { + let crate_name = &req.params()["crate_id"]; + let conn = req.db_conn()?; + let krate = Crate::by_name(crate_name).first::(&*conn)?; + let owners = Team::owning(&krate, &conn)? + .into_iter() + .map(Owner::encodable) + .collect(); + + #[derive(Serialize)] + struct R { + teams: Vec, + } + Ok(req.json(&R { teams: owners })) +} + +/// Handles the `GET /crates/:crate_id/owner_user` route. +pub fn owner_user(req: &mut Request) -> CargoResult { + let crate_name = &req.params()["crate_id"]; + let conn = req.db_conn()?; + let krate = Crate::by_name(crate_name).first::(&*conn)?; + let owners = User::owning(&krate, &conn)? + .into_iter() + .map(Owner::encodable) + .collect(); + + #[derive(Serialize)] + struct R { + users: Vec, + } + Ok(req.json(&R { users: owners })) +} + +/// Handles the `PUT /crates/:crate_id/owners` route. +pub fn add_owners(req: &mut Request) -> CargoResult { + modify_owners(req, true) +} + +/// Handles the `DELETE /crates/:crate_id/owners` route. +pub fn remove_owners(req: &mut Request) -> CargoResult { + modify_owners(req, false) +} + +fn modify_owners(req: &mut Request, add: bool) -> CargoResult { + let mut body = String::new(); + req.body().read_to_string(&mut body)?; + + let user = req.user()?; + let conn = req.db_conn()?; + let krate = Crate::by_name(&req.params()["crate_id"]).first::(&*conn)?; + let owners = krate.owners(&conn)?; + + match rights(req.app(), &owners, user)? { + Rights::Full => {} + // Yes! + Rights::Publish => { + return Err(human("team members don't have permission to modify owners")); + } + Rights::None => { + return Err(human("only owners have permission to modify owners")); + } + } + + #[derive(Deserialize)] + struct Request { + // identical, for back-compat (owners preferred) + users: Option>, + owners: Option>, + } + + let request: Request = serde_json::from_str(&body).map_err(|_| human("invalid json request"))?; + + let logins = request + .owners + .or(request.users) + .ok_or_else(|| human("invalid json request"))?; + + let mut msgs = Vec::new(); + + for login in &logins { + if add { + if owners.iter().any(|owner| owner.login() == *login) { + return Err(human(&format_args!("`{}` is already an owner", login))); + } + let msg = krate.owner_add(req.app(), &conn, user, login)?; + msgs.push(msg); + } else { + // Removing the team that gives you rights is prevented because + // team members only have Rights::Publish + if owners.len() == 1 { + return Err(human("cannot remove the sole owner of a crate")); + } + krate.owner_remove(req.app(), &conn, user, login)?; + } + } + + let comma_sep_msg = msgs.join(","); + + #[derive(Serialize)] + struct R { + ok: bool, + msg: String, + } + Ok(req.json(&R { + ok: true, + msg: comma_sep_msg, + })) +} diff --git a/src/lib.rs b/src/lib.rs index 957a87d250f..7fa0681639d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -142,9 +142,9 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder { // Routes used by `cargo` api_router.put("/crates/new", C(krate::publish::publish)); - api_router.get("/crates/:crate_id/owners", C(krate::owners)); - api_router.put("/crates/:crate_id/owners", C(krate::add_owners)); - api_router.delete("/crates/:crate_id/owners", C(krate::remove_owners)); + api_router.get("/crates/:crate_id/owners", C(krate::owners::owners)); + api_router.put("/crates/:crate_id/owners", C(krate::owners::add_owners)); + api_router.delete("/crates/:crate_id/owners", C(krate::owners::remove_owners)); api_router.delete("/crates/:crate_id/:version/yank", C(version::yank)); api_router.put("/crates/:crate_id/:version/unyank", C(version::unyank)); api_router.get("/crates/:crate_id/:version/download", C(krate::download)); @@ -171,8 +171,8 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder { api_router.put("/crates/:crate_id/follow", C(krate::follow)); api_router.delete("/crates/:crate_id/follow", C(krate::unfollow)); api_router.get("/crates/:crate_id/following", C(krate::following)); - api_router.get("/crates/:crate_id/owner_team", C(krate::owner_team)); - api_router.get("/crates/:crate_id/owner_user", C(krate::owner_user)); + api_router.get("/crates/:crate_id/owner_team", C(krate::owners::owner_team)); + api_router.get("/crates/:crate_id/owner_user", C(krate::owners::owner_user)); api_router.get( "/crates/:crate_id/reverse_dependencies", C(krate::reverse_dependencies), From 6f5ab34ae87b73786d14c24128c4f90c1c631f6d Mon Sep 17 00:00:00 2001 From: Justin Geibel Date: Wed, 11 Oct 2017 19:13:44 -0400 Subject: [PATCH 04/10] Move follow routes to krate::follow::* --- src/krate/follow.rs | 68 +++++++++++++++++++++++++++++++++++++++++++++ src/krate/mod.rs | 55 +----------------------------------- src/lib.rs | 6 ++-- 3 files changed, 72 insertions(+), 57 deletions(-) create mode 100644 src/krate/follow.rs diff --git a/src/krate/follow.rs b/src/krate/follow.rs new file mode 100644 index 00000000000..5151d38e3cd --- /dev/null +++ b/src/krate/follow.rs @@ -0,0 +1,68 @@ +//! Endpoints for managing a per user list of followed crates + +use conduit::{Request, Response}; +use conduit_router::RequestParams; +use diesel::associations::Identifiable; +use diesel::pg::upsert::*; +use diesel::prelude::*; +use diesel; + +use db::RequestTransaction; +use schema::*; +use user::RequestUser; +use util::{CargoResult, RequestUtils}; + +use super::{Crate, Follow}; + +fn follow_target(req: &mut Request) -> CargoResult { + let user = req.user()?; + let conn = req.db_conn()?; + let crate_name = &req.params()["crate_id"]; + let crate_id = Crate::by_name(crate_name).select(crates::id).first(&*conn)?; + Ok(Follow { + user_id: user.id, + crate_id: crate_id, + }) +} + +/// Handles the `PUT /crates/:crate_id/follow` route. +pub fn follow(req: &mut Request) -> CargoResult { + let follow = follow_target(req)?; + let conn = req.db_conn()?; + diesel::insert(&follow.on_conflict_do_nothing()) + .into(follows::table) + .execute(&*conn)?; + #[derive(Serialize)] + struct R { + ok: bool, + } + Ok(req.json(&R { ok: true })) +} + +/// Handles the `DELETE /crates/:crate_id/follow` route. +pub fn unfollow(req: &mut Request) -> CargoResult { + let follow = follow_target(req)?; + let conn = req.db_conn()?; + diesel::delete(&follow).execute(&*conn)?; + #[derive(Serialize)] + struct R { + ok: bool, + } + Ok(req.json(&R { ok: true })) +} + +/// Handles the `GET /crates/:crate_id/following` route. +pub fn following(req: &mut Request) -> CargoResult { + use diesel::expression::dsl::exists; + + let follow = follow_target(req)?; + let conn = req.db_conn()?; + let following = diesel::select(exists(follows::table.find(follow.id()))).get_result(&*conn)?; + #[derive(Serialize)] + struct R { + following: bool, + } + Ok(req.json(&R { + following: following, + })) +} diff --git a/src/krate/mod.rs b/src/krate/mod.rs index 4d75bdc3cd5..8f094b33569 100644 --- a/src/krate/mod.rs +++ b/src/krate/mod.rs @@ -25,7 +25,6 @@ use keyword::{CrateKeyword, EncodableKeyword}; use owner::{CrateOwner, Owner, OwnerKind}; use crate_owner_invitation::NewCrateOwnerInvitation; use schema::*; -use user::RequestUser; use util::{human, CargoResult, RequestUtils}; use version::EncodableVersion; use {Badge, Category, Keyword, Replica, User, Version}; @@ -33,6 +32,7 @@ use {Badge, Category, Keyword, Replica, User, Version}; pub mod search; pub mod publish; pub mod owners; +pub mod follow; /// Hosts in this blacklist are known to not be hosting documentation, /// and are possibly of malicious intent e.g. ad tracking networks, etc. @@ -840,59 +840,6 @@ pub struct Follow { crate_id: i32, } -fn follow_target(req: &mut Request) -> CargoResult { - let user = req.user()?; - let conn = req.db_conn()?; - let crate_name = &req.params()["crate_id"]; - let crate_id = Crate::by_name(crate_name).select(crates::id).first(&*conn)?; - Ok(Follow { - user_id: user.id, - crate_id: crate_id, - }) -} - -/// Handles the `PUT /crates/:crate_id/follow` route. -pub fn follow(req: &mut Request) -> CargoResult { - let follow = follow_target(req)?; - let conn = req.db_conn()?; - diesel::insert(&follow.on_conflict_do_nothing()) - .into(follows::table) - .execute(&*conn)?; - #[derive(Serialize)] - struct R { - ok: bool, - } - Ok(req.json(&R { ok: true })) -} - -/// Handles the `DELETE /crates/:crate_id/follow` route. -pub fn unfollow(req: &mut Request) -> CargoResult { - let follow = follow_target(req)?; - let conn = req.db_conn()?; - diesel::delete(&follow).execute(&*conn)?; - #[derive(Serialize)] - struct R { - ok: bool, - } - Ok(req.json(&R { ok: true })) -} - -/// Handles the `GET /crates/:crate_id/following` route. -pub fn following(req: &mut Request) -> CargoResult { - use diesel::expression::dsl::exists; - - let follow = follow_target(req)?; - let conn = req.db_conn()?; - let following = diesel::select(exists(follows::table.find(follow.id()))).get_result(&*conn)?; - #[derive(Serialize)] - struct R { - following: bool, - } - Ok(req.json(&R { - following: following, - })) -} - /// Handles the `GET /crates/:crate_id/versions` route. // FIXME: Not sure why this is necessary since /crates/:crate_id returns // this information already, but ember is definitely requesting it diff --git a/src/lib.rs b/src/lib.rs index 7fa0681639d..32c74a80b5c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -168,9 +168,9 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder { api_router.get("/crates/:crate_id/:version/authors", C(version::authors)); api_router.get("/crates/:crate_id/downloads", C(krate::downloads)); api_router.get("/crates/:crate_id/versions", C(krate::versions)); - api_router.put("/crates/:crate_id/follow", C(krate::follow)); - api_router.delete("/crates/:crate_id/follow", C(krate::unfollow)); - api_router.get("/crates/:crate_id/following", C(krate::following)); + api_router.put("/crates/:crate_id/follow", C(krate::follow::follow)); + api_router.delete("/crates/:crate_id/follow", C(krate::follow::unfollow)); + api_router.get("/crates/:crate_id/following", C(krate::follow::following)); api_router.get("/crates/:crate_id/owner_team", C(krate::owners::owner_team)); api_router.get("/crates/:crate_id/owner_user", C(krate::owners::owner_user)); api_router.get( From 6199bf9ba7bc9c72ff495fc8a68250ea439a5542 Mon Sep 17 00:00:00 2001 From: Justin Geibel Date: Wed, 11 Oct 2017 19:30:58 -0400 Subject: [PATCH 05/10] Move download related functionality to {krate,version}::downloads::* --- src/krate/downloads.rs | 68 +++++++++++++++++++++++++ src/krate/mod.rs | 107 +-------------------------------------- src/lib.rs | 12 +++-- src/version/downloads.rs | 87 +++++++++++++++++++++++++++++++ src/version/mod.rs | 32 ++---------- 5 files changed, 169 insertions(+), 137 deletions(-) create mode 100644 src/krate/downloads.rs create mode 100644 src/version/downloads.rs diff --git a/src/krate/downloads.rs b/src/krate/downloads.rs new file mode 100644 index 00000000000..44974bb93b3 --- /dev/null +++ b/src/krate/downloads.rs @@ -0,0 +1,68 @@ +use std::cmp; + +use conduit::{Request, Response}; +use conduit_router::RequestParams; +use diesel::prelude::*; + +use db::RequestTransaction; +use download::{EncodableVersionDownload, VersionDownload}; +use schema::*; +use util::{CargoResult, RequestUtils}; +use Version; + +use super::{to_char, Crate}; + +/// Handles the `GET /crates/:crate_id/downloads` route. +pub fn downloads(req: &mut Request) -> CargoResult { + use diesel::expression::dsl::*; + use diesel::types::BigInt; + + let crate_name = &req.params()["crate_id"]; + let conn = req.db_conn()?; + let krate = Crate::by_name(crate_name).first::(&*conn)?; + + let mut versions = Version::belonging_to(&krate).load::(&*conn)?; + versions.sort_by(|a, b| b.num.cmp(&a.num)); + let (latest_five, rest) = versions.split_at(cmp::min(5, versions.len())); + + let downloads = VersionDownload::belonging_to(latest_five) + .filter(version_downloads::date.gt(date(now - 90.days()))) + .order(version_downloads::date.asc()) + .load(&*conn)? + .into_iter() + .map(VersionDownload::encodable) + .collect::>(); + + let sum_downloads = sql::("SUM(version_downloads.downloads)"); + let extra = VersionDownload::belonging_to(rest) + .select(( + to_char(version_downloads::date, "YYYY-MM-DD"), + sum_downloads, + )) + .filter(version_downloads::date.gt(date(now - 90.days()))) + .group_by(version_downloads::date) + .order(version_downloads::date.asc()) + .load::(&*conn)?; + + #[derive(Serialize, Queryable)] + struct ExtraDownload { + date: String, + downloads: i64, + } + #[derive(Serialize)] + struct R { + version_downloads: Vec, + meta: Meta, + } + #[derive(Serialize)] + struct Meta { + extra_downloads: Vec, + } + let meta = Meta { + extra_downloads: extra, + }; + Ok(req.json(&R { + version_downloads: downloads, + meta: meta, + })) +} diff --git a/src/krate/mod.rs b/src/krate/mod.rs index 8f094b33569..c657e39f6cf 100644 --- a/src/krate/mod.rs +++ b/src/krate/mod.rs @@ -1,5 +1,4 @@ use std::ascii::AsciiExt; -use std::cmp; use chrono::{NaiveDate, NaiveDateTime}; use conduit::{Request, Response}; @@ -20,19 +19,19 @@ use badge::EncodableBadge; use category::{CrateCategory, EncodableCategory}; use db::RequestTransaction; use dependency::{EncodableDependency, ReverseDependency}; -use download::{EncodableVersionDownload, VersionDownload}; use keyword::{CrateKeyword, EncodableKeyword}; use owner::{CrateOwner, Owner, OwnerKind}; use crate_owner_invitation::NewCrateOwnerInvitation; use schema::*; use util::{human, CargoResult, RequestUtils}; use version::EncodableVersion; -use {Badge, Category, Keyword, Replica, User, Version}; +use {Badge, Category, Keyword, User, Version}; pub mod search; pub mod publish; pub mod owners; pub mod follow; +pub mod downloads; /// Hosts in this blacklist are known to not be hosting documentation, /// and are possibly of malicious intent e.g. ad tracking networks, etc. @@ -707,39 +706,6 @@ pub fn show(req: &mut Request) -> CargoResult { ) } -/// Handles the `GET /crates/:crate_id/:version/download` route. -/// This returns a URL to the location where the crate is stored. -pub fn download(req: &mut Request) -> CargoResult { - let crate_name = &req.params()["crate_id"]; - let version = &req.params()["version"]; - - // If we are a mirror, ignore failure to update download counts. - // API-only mirrors won't have any crates in their database, and - // incrementing the download count will look up the crate in the - // database. Mirrors just want to pass along a redirect URL. - if req.app().config.mirror == Replica::ReadOnlyMirror { - let _ = increment_download_counts(req, crate_name, version); - } else { - increment_download_counts(req, crate_name, version)?; - } - - let redirect_url = req.app() - .config - .uploader - .crate_location(crate_name, version) - .ok_or_else(|| human("crate files not found"))?; - - if req.wants_json() { - #[derive(Serialize)] - struct R { - url: String, - } - Ok(req.json(&R { url: redirect_url })) - } else { - Ok(req.redirect(redirect_url)) - } -} - /// Handles the `GET /crates/:crate_id/:version/readme` route. pub fn readme(req: &mut Request) -> CargoResult { let crate_name = &req.params()["crate_id"]; @@ -762,75 +728,6 @@ pub fn readme(req: &mut Request) -> CargoResult { } } -fn increment_download_counts(req: &Request, crate_name: &str, version: &str) -> CargoResult<()> { - use self::versions::dsl::*; - - let conn = req.db_conn()?; - let version_id = versions - .select(id) - .filter(crate_id.eq_any(Crate::by_name(crate_name).select(crates::id))) - .filter(num.eq(version)) - .first(&*conn)?; - - VersionDownload::create_or_increment(version_id, &conn)?; - Ok(()) -} - -/// Handles the `GET /crates/:crate_id/downloads` route. -pub fn downloads(req: &mut Request) -> CargoResult { - use diesel::expression::dsl::*; - use diesel::types::BigInt; - - let crate_name = &req.params()["crate_id"]; - let conn = req.db_conn()?; - let krate = Crate::by_name(crate_name).first::(&*conn)?; - - let mut versions = Version::belonging_to(&krate).load::(&*conn)?; - versions.sort_by(|a, b| b.num.cmp(&a.num)); - let (latest_five, rest) = versions.split_at(cmp::min(5, versions.len())); - - let downloads = VersionDownload::belonging_to(latest_five) - .filter(version_downloads::date.gt(date(now - 90.days()))) - .order(version_downloads::date.asc()) - .load(&*conn)? - .into_iter() - .map(VersionDownload::encodable) - .collect::>(); - - let sum_downloads = sql::("SUM(version_downloads.downloads)"); - let extra = VersionDownload::belonging_to(rest) - .select(( - to_char(version_downloads::date, "YYYY-MM-DD"), - sum_downloads, - )) - .filter(version_downloads::date.gt(date(now - 90.days()))) - .group_by(version_downloads::date) - .order(version_downloads::date.asc()) - .load::(&*conn)?; - - #[derive(Serialize, Queryable)] - struct ExtraDownload { - date: String, - downloads: i64, - } - #[derive(Serialize)] - struct R { - version_downloads: Vec, - meta: Meta, - } - #[derive(Serialize)] - struct Meta { - extra_downloads: Vec, - } - let meta = Meta { - extra_downloads: extra, - }; - Ok(req.json(&R { - version_downloads: downloads, - meta: meta, - })) -} - #[derive(Insertable, Queryable, Identifiable, Associations, Clone, Copy, Debug)] #[belongs_to(User)] #[primary_key(user_id, crate_id)] diff --git a/src/lib.rs b/src/lib.rs index 32c74a80b5c..313dcac1d36 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -147,7 +147,10 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder { api_router.delete("/crates/:crate_id/owners", C(krate::owners::remove_owners)); api_router.delete("/crates/:crate_id/:version/yank", C(version::yank)); api_router.put("/crates/:crate_id/:version/unyank", C(version::unyank)); - api_router.get("/crates/:crate_id/:version/download", C(krate::download)); + api_router.get( + "/crates/:crate_id/:version/download", + C(version::downloads::download), + ); // Routes that appear to be unused api_router.get("/versions", C(version::index)); @@ -163,10 +166,13 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder { ); api_router.get( "/crates/:crate_id/:version/downloads", - C(version::downloads), + C(version::downloads::downloads), ); api_router.get("/crates/:crate_id/:version/authors", C(version::authors)); - api_router.get("/crates/:crate_id/downloads", C(krate::downloads)); + api_router.get( + "/crates/:crate_id/downloads", + C(krate::downloads::downloads), + ); api_router.get("/crates/:crate_id/versions", C(krate::versions)); api_router.put("/crates/:crate_id/follow", C(krate::follow::follow)); api_router.delete("/crates/:crate_id/follow", C(krate::follow::unfollow)); diff --git a/src/version/downloads.rs b/src/version/downloads.rs new file mode 100644 index 00000000000..059ab4f81ad --- /dev/null +++ b/src/version/downloads.rs @@ -0,0 +1,87 @@ +use chrono::{Duration, NaiveDate, Utc}; +use conduit::{Request, Response}; +use conduit_router::RequestParams; +use diesel::prelude::*; + +use app::RequestApp; +use db::RequestTransaction; +use download::{EncodableVersionDownload, VersionDownload}; +use schema::*; +use util::{human, CargoResult, RequestUtils}; +use {Crate, Replica}; + +use super::version_and_crate; + +/// Handles the `GET /crates/:crate_id/:version/download` route. +/// This returns a URL to the location where the crate is stored. +pub fn download(req: &mut Request) -> CargoResult { + let crate_name = &req.params()["crate_id"]; + let version = &req.params()["version"]; + + // If we are a mirror, ignore failure to update download counts. + // API-only mirrors won't have any crates in their database, and + // incrementing the download count will look up the crate in the + // database. Mirrors just want to pass along a redirect URL. + if req.app().config.mirror == Replica::ReadOnlyMirror { + let _ = increment_download_counts(req, crate_name, version); + } else { + increment_download_counts(req, crate_name, version)?; + } + + let redirect_url = req.app() + .config + .uploader + .crate_location(crate_name, version) + .ok_or_else(|| human("crate files not found"))?; + + if req.wants_json() { + #[derive(Serialize)] + struct R { + url: String, + } + Ok(req.json(&R { url: redirect_url })) + } else { + Ok(req.redirect(redirect_url)) + } +} + +fn increment_download_counts(req: &Request, crate_name: &str, version: &str) -> CargoResult<()> { + use self::versions::dsl::*; + + let conn = req.db_conn()?; + let version_id = versions + .select(id) + .filter(crate_id.eq_any(Crate::by_name(crate_name).select(crates::id))) + .filter(num.eq(version)) + .first(&*conn)?; + + VersionDownload::create_or_increment(version_id, &conn)?; + Ok(()) +} + +/// Handles the `GET /crates/:crate_id/:version/downloads` route. +pub fn downloads(req: &mut Request) -> CargoResult { + let (version, _) = version_and_crate(req)?; + let conn = req.db_conn()?; + let cutoff_end_date = req.query() + .get("before_date") + .and_then(|d| NaiveDate::parse_from_str(d, "%F").ok()) + .unwrap_or_else(|| Utc::today().naive_utc()); + let cutoff_start_date = cutoff_end_date - Duration::days(89); + + let downloads = VersionDownload::belonging_to(&version) + .filter(version_downloads::date.between(cutoff_start_date..cutoff_end_date)) + .order(version_downloads::date) + .load(&*conn)? + .into_iter() + .map(VersionDownload::encodable) + .collect(); + + #[derive(Serialize)] + struct R { + version_downloads: Vec, + } + Ok(req.json(&R { + version_downloads: downloads, + })) +} diff --git a/src/version/mod.rs b/src/version/mod.rs index 2afd9ce8f35..51866e9c8c1 100644 --- a/src/version/mod.rs +++ b/src/version/mod.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use chrono::{Duration, NaiveDate, NaiveDateTime, Utc}; +use chrono::NaiveDateTime; use conduit::{Request, Response}; use conduit_router::RequestParams; use diesel; @@ -14,7 +14,6 @@ use Crate; use app::RequestApp; use db::RequestTransaction; use dependency::{Dependency, EncodableDependency}; -use download::{EncodableVersionDownload, VersionDownload}; use git; use owner::{rights, Rights}; use schema::*; @@ -23,6 +22,8 @@ use util::errors::CargoError; use util::{human, CargoResult, RequestUtils}; use license_exprs; +pub mod downloads; + // Queryable has a custom implementation below #[derive(Clone, Identifiable, Associations, Debug)] #[belongs_to(Crate)] @@ -352,33 +353,6 @@ pub fn dependencies(req: &mut Request) -> CargoResult { Ok(req.json(&R { dependencies: deps })) } -/// Handles the `GET /crates/:crate_id/:version/downloads` route. -pub fn downloads(req: &mut Request) -> CargoResult { - let (version, _) = version_and_crate(req)?; - let conn = req.db_conn()?; - let cutoff_end_date = req.query() - .get("before_date") - .and_then(|d| NaiveDate::parse_from_str(d, "%F").ok()) - .unwrap_or_else(|| Utc::today().naive_utc()); - let cutoff_start_date = cutoff_end_date - Duration::days(89); - - let downloads = VersionDownload::belonging_to(&version) - .filter(version_downloads::date.between(cutoff_start_date..cutoff_end_date)) - .order(version_downloads::date) - .load(&*conn)? - .into_iter() - .map(VersionDownload::encodable) - .collect(); - - #[derive(Serialize)] - struct R { - version_downloads: Vec, - } - Ok(req.json(&R { - version_downloads: downloads, - })) -} - /// Handles the `GET /crates/:crate_id/:version/authors` route. pub fn authors(req: &mut Request) -> CargoResult { let (version, _) = version_and_crate(req)?; From c156edbfd4331be4bb17e18aa125ca4ffc8ee28d Mon Sep 17 00:00:00 2001 From: Justin Geibel Date: Wed, 11 Oct 2017 19:41:22 -0400 Subject: [PATCH 06/10] Move remaining krate endpoints to krate::metadata::* --- src/krate/metadata.rs | 251 ++++++++++++++++++++++++++++++++++++++++++ src/krate/mod.rs | 249 +---------------------------------------- src/lib.rs | 13 ++- 3 files changed, 263 insertions(+), 250 deletions(-) create mode 100644 src/krate/metadata.rs diff --git a/src/krate/metadata.rs b/src/krate/metadata.rs new file mode 100644 index 00000000000..a0ff8c7f515 --- /dev/null +++ b/src/krate/metadata.rs @@ -0,0 +1,251 @@ +use conduit::{Request, Response}; +use conduit_router::RequestParams; +use diesel::prelude::*; + +use app::RequestApp; +use category::{CrateCategory, EncodableCategory}; +use db::RequestTransaction; +use dependency::EncodableDependency; +use keyword::{CrateKeyword, EncodableKeyword}; +use schema::*; +use util::{human, CargoResult, RequestUtils}; +use version::EncodableVersion; +use {Category, Keyword, Version}; + +use super::{Crate, CrateDownload, EncodableCrate, ALL_COLUMNS}; + +/// Handles the `GET /summary` route. +pub fn summary(req: &mut Request) -> CargoResult { + use diesel::expression::{date, now, sql, DayAndMonthIntervalDsl}; + use diesel::types::{BigInt, Nullable}; + use schema::crates::dsl::*; + + let conn = req.db_conn()?; + let num_crates = crates.count().get_result(&*conn)?; + let num_downloads = metadata::table + .select(metadata::total_downloads) + .get_result(&*conn)?; + + let encode_crates = |krates: Vec| -> CargoResult> { + Version::belonging_to(&krates) + .filter(versions::yanked.eq(false)) + .load::(&*conn)? + .grouped_by(&krates) + .into_iter() + .map(|versions| Version::max(versions.into_iter().map(|v| v.num))) + .zip(krates) + .map(|(max_version, krate)| { + Ok(krate.minimal_encodable(max_version, None, false, None)) + }) + .collect() + }; + + let new_crates = crates + .order(created_at.desc()) + .select(ALL_COLUMNS) + .limit(10) + .load(&*conn)?; + let just_updated = crates + .filter(updated_at.ne(created_at)) + .order(updated_at.desc()) + .select(ALL_COLUMNS) + .limit(10) + .load(&*conn)?; + let most_downloaded = crates + .order(downloads.desc()) + .select(ALL_COLUMNS) + .limit(10) + .load(&*conn)?; + + let recent_downloads = sql::>("SUM(crate_downloads.downloads)"); + let most_recently_downloaded = crates + .left_join( + crate_downloads::table.on( + id.eq(crate_downloads::crate_id) + .and(crate_downloads::date.gt(date(now - 90.days()))), + ), + ) + .group_by(id) + .order(recent_downloads.desc().nulls_last()) + .limit(10) + .select(ALL_COLUMNS) + .load::(&*conn)?; + + let popular_keywords = keywords::table + .order(keywords::crates_cnt.desc()) + .limit(10) + .load(&*conn)? + .into_iter() + .map(Keyword::encodable) + .collect(); + + let popular_categories = Category::toplevel(&conn, "crates", 10, 0)? + .into_iter() + .map(Category::encodable) + .collect(); + + #[derive(Serialize)] + struct R { + num_downloads: i64, + num_crates: i64, + new_crates: Vec, + most_downloaded: Vec, + most_recently_downloaded: Vec, + just_updated: Vec, + popular_keywords: Vec, + popular_categories: Vec, + } + Ok(req.json(&R { + num_downloads: num_downloads, + num_crates: num_crates, + new_crates: encode_crates(new_crates)?, + most_downloaded: encode_crates(most_downloaded)?, + most_recently_downloaded: encode_crates(most_recently_downloaded)?, + just_updated: encode_crates(just_updated)?, + popular_keywords: popular_keywords, + popular_categories: popular_categories, + })) +} + +/// Handles the `GET /crates/:crate_id` route. +pub fn show(req: &mut Request) -> CargoResult { + use diesel::expression::dsl::*; + + let name = &req.params()["crate_id"]; + let conn = req.db_conn()?; + let krate = Crate::by_name(name).first::(&*conn)?; + + let mut versions = Version::belonging_to(&krate).load::(&*conn)?; + versions.sort_by(|a, b| b.num.cmp(&a.num)); + let ids = versions.iter().map(|v| v.id).collect(); + + let kws = CrateKeyword::belonging_to(&krate) + .inner_join(keywords::table) + .select(keywords::all_columns) + .load(&*conn)?; + let cats = CrateCategory::belonging_to(&krate) + .inner_join(categories::table) + .select(categories::all_columns) + .load(&*conn)?; + let recent_downloads = CrateDownload::belonging_to(&krate) + .filter(crate_downloads::date.gt(date(now - 90.days()))) + .select(sum(crate_downloads::downloads)) + .get_result(&*conn)?; + + let badges = badges::table + .filter(badges::crate_id.eq(krate.id)) + .load(&*conn)?; + let max_version = krate.max_version(&conn)?; + + #[derive(Serialize)] + struct R { + #[serde(rename = "crate")] krate: EncodableCrate, + versions: Vec, + keywords: Vec, + categories: Vec, + } + Ok( + req.json(&R { + krate: krate.clone().encodable( + max_version, + Some(ids), + Some(&kws), + Some(&cats), + Some(badges), + false, + recent_downloads, + ), + versions: versions + .into_iter() + .map(|v| v.encodable(&krate.name)) + .collect(), + keywords: kws.into_iter().map(|k| k.encodable()).collect(), + categories: cats.into_iter().map(|k| k.encodable()).collect(), + }), + ) +} + +/// Handles the `GET /crates/:crate_id/:version/readme` route. +pub fn readme(req: &mut Request) -> CargoResult { + let crate_name = &req.params()["crate_id"]; + let version = &req.params()["version"]; + + let redirect_url = req.app() + .config + .uploader + .readme_location(crate_name, version) + .ok_or_else(|| human("crate readme not found"))?; + + if req.wants_json() { + #[derive(Serialize)] + struct R { + url: String, + } + Ok(req.json(&R { url: redirect_url })) + } else { + Ok(req.redirect(redirect_url)) + } +} + +/// Handles the `GET /crates/:crate_id/versions` route. +// FIXME: Not sure why this is necessary since /crates/:crate_id returns +// this information already, but ember is definitely requesting it +pub fn versions(req: &mut Request) -> CargoResult { + let crate_name = &req.params()["crate_id"]; + let conn = req.db_conn()?; + let krate = Crate::by_name(crate_name).first::(&*conn)?; + let mut versions = Version::belonging_to(&krate).load::(&*conn)?; + versions.sort_by(|a, b| b.num.cmp(&a.num)); + let versions = versions + .into_iter() + .map(|v| v.encodable(crate_name)) + .collect(); + + #[derive(Serialize)] + struct R { + versions: Vec, + } + Ok(req.json(&R { versions: versions })) +} + +/// Handles the `GET /crates/:crate_id/reverse_dependencies` route. +pub fn reverse_dependencies(req: &mut Request) -> CargoResult { + use diesel::expression::dsl::any; + + let name = &req.params()["crate_id"]; + let conn = req.db_conn()?; + let krate = Crate::by_name(name).first::(&*conn)?; + let (offset, limit) = req.pagination(10, 100)?; + let (rev_deps, total) = krate.reverse_dependencies(&*conn, offset, limit)?; + let rev_deps: Vec<_> = rev_deps + .into_iter() + .map(|dep| dep.encodable(&krate.name)) + .collect(); + + let version_ids: Vec = rev_deps.iter().map(|dep| dep.version_id).collect(); + + let versions = versions::table + .filter(versions::id.eq(any(version_ids))) + .inner_join(crates::table) + .select((versions::all_columns, crates::name)) + .load::<(Version, String)>(&*conn)? + .into_iter() + .map(|(version, krate_name)| version.encodable(&krate_name)) + .collect(); + + #[derive(Serialize)] + struct R { + dependencies: Vec, + versions: Vec, + meta: Meta, + } + #[derive(Serialize)] + struct Meta { + total: i64, + } + Ok(req.json(&R { + dependencies: rev_deps, + versions, + meta: Meta { total: total }, + })) +} diff --git a/src/krate/mod.rs b/src/krate/mod.rs index c657e39f6cf..6b71914f3a8 100644 --- a/src/krate/mod.rs +++ b/src/krate/mod.rs @@ -1,8 +1,6 @@ use std::ascii::AsciiExt; use chrono::{NaiveDate, NaiveDateTime}; -use conduit::{Request, Response}; -use conduit_router::RequestParams; use diesel::associations::Identifiable; use diesel::expression::helper_types::Eq; use diesel::helper_types::Select; @@ -14,17 +12,13 @@ use license_exprs; use semver; use url::Url; -use app::{App, RequestApp}; +use app::App; use badge::EncodableBadge; -use category::{CrateCategory, EncodableCategory}; -use db::RequestTransaction; -use dependency::{EncodableDependency, ReverseDependency}; -use keyword::{CrateKeyword, EncodableKeyword}; +use dependency::ReverseDependency; use owner::{CrateOwner, Owner, OwnerKind}; use crate_owner_invitation::NewCrateOwnerInvitation; use schema::*; -use util::{human, CargoResult, RequestUtils}; -use version::EncodableVersion; +use util::{human, CargoResult}; use {Badge, Category, Keyword, User, Version}; pub mod search; @@ -32,6 +26,7 @@ pub mod publish; pub mod owners; pub mod follow; pub mod downloads; +pub mod metadata; /// Hosts in this blacklist are known to not be hosting documentation, /// and are possibly of malicious intent e.g. ad tracking networks, etc. @@ -555,179 +550,6 @@ impl Crate { } } -/// Handles the `GET /summary` route. -pub fn summary(req: &mut Request) -> CargoResult { - use diesel::expression::{date, now, sql, DayAndMonthIntervalDsl}; - use diesel::types::{BigInt, Nullable}; - use schema::crates::dsl::*; - - let conn = req.db_conn()?; - let num_crates = crates.count().get_result(&*conn)?; - let num_downloads = metadata::table - .select(metadata::total_downloads) - .get_result(&*conn)?; - - let encode_crates = |krates: Vec| -> CargoResult> { - Version::belonging_to(&krates) - .filter(versions::yanked.eq(false)) - .load::(&*conn)? - .grouped_by(&krates) - .into_iter() - .map(|versions| Version::max(versions.into_iter().map(|v| v.num))) - .zip(krates) - .map(|(max_version, krate)| { - Ok(krate.minimal_encodable(max_version, None, false, None)) - }) - .collect() - }; - - let new_crates = crates - .order(created_at.desc()) - .select(ALL_COLUMNS) - .limit(10) - .load(&*conn)?; - let just_updated = crates - .filter(updated_at.ne(created_at)) - .order(updated_at.desc()) - .select(ALL_COLUMNS) - .limit(10) - .load(&*conn)?; - let most_downloaded = crates - .order(downloads.desc()) - .select(ALL_COLUMNS) - .limit(10) - .load(&*conn)?; - - let recent_downloads = sql::>("SUM(crate_downloads.downloads)"); - let most_recently_downloaded = crates - .left_join( - crate_downloads::table.on( - id.eq(crate_downloads::crate_id) - .and(crate_downloads::date.gt(date(now - 90.days()))), - ), - ) - .group_by(id) - .order(recent_downloads.desc().nulls_last()) - .limit(10) - .select(ALL_COLUMNS) - .load::(&*conn)?; - - let popular_keywords = keywords::table - .order(keywords::crates_cnt.desc()) - .limit(10) - .load(&*conn)? - .into_iter() - .map(Keyword::encodable) - .collect(); - - let popular_categories = Category::toplevel(&conn, "crates", 10, 0)? - .into_iter() - .map(Category::encodable) - .collect(); - - #[derive(Serialize)] - struct R { - num_downloads: i64, - num_crates: i64, - new_crates: Vec, - most_downloaded: Vec, - most_recently_downloaded: Vec, - just_updated: Vec, - popular_keywords: Vec, - popular_categories: Vec, - } - Ok(req.json(&R { - num_downloads: num_downloads, - num_crates: num_crates, - new_crates: encode_crates(new_crates)?, - most_downloaded: encode_crates(most_downloaded)?, - most_recently_downloaded: encode_crates(most_recently_downloaded)?, - just_updated: encode_crates(just_updated)?, - popular_keywords: popular_keywords, - popular_categories: popular_categories, - })) -} - -/// Handles the `GET /crates/:crate_id` route. -pub fn show(req: &mut Request) -> CargoResult { - use diesel::expression::dsl::*; - - let name = &req.params()["crate_id"]; - let conn = req.db_conn()?; - let krate = Crate::by_name(name).first::(&*conn)?; - - let mut versions = Version::belonging_to(&krate).load::(&*conn)?; - versions.sort_by(|a, b| b.num.cmp(&a.num)); - let ids = versions.iter().map(|v| v.id).collect(); - - let kws = CrateKeyword::belonging_to(&krate) - .inner_join(keywords::table) - .select(keywords::all_columns) - .load(&*conn)?; - let cats = CrateCategory::belonging_to(&krate) - .inner_join(categories::table) - .select(categories::all_columns) - .load(&*conn)?; - let recent_downloads = CrateDownload::belonging_to(&krate) - .filter(crate_downloads::date.gt(date(now - 90.days()))) - .select(sum(crate_downloads::downloads)) - .get_result(&*conn)?; - - let badges = badges::table - .filter(badges::crate_id.eq(krate.id)) - .load(&*conn)?; - let max_version = krate.max_version(&conn)?; - - #[derive(Serialize)] - struct R { - #[serde(rename = "crate")] krate: EncodableCrate, - versions: Vec, - keywords: Vec, - categories: Vec, - } - Ok( - req.json(&R { - krate: krate.clone().encodable( - max_version, - Some(ids), - Some(&kws), - Some(&cats), - Some(badges), - false, - recent_downloads, - ), - versions: versions - .into_iter() - .map(|v| v.encodable(&krate.name)) - .collect(), - keywords: kws.into_iter().map(|k| k.encodable()).collect(), - categories: cats.into_iter().map(|k| k.encodable()).collect(), - }), - ) -} - -/// Handles the `GET /crates/:crate_id/:version/readme` route. -pub fn readme(req: &mut Request) -> CargoResult { - let crate_name = &req.params()["crate_id"]; - let version = &req.params()["version"]; - - let redirect_url = req.app() - .config - .uploader - .readme_location(crate_name, version) - .ok_or_else(|| human("crate readme not found"))?; - - if req.wants_json() { - #[derive(Serialize)] - struct R { - url: String, - } - Ok(req.json(&R { url: redirect_url })) - } else { - Ok(req.redirect(redirect_url)) - } -} - #[derive(Insertable, Queryable, Identifiable, Associations, Clone, Copy, Debug)] #[belongs_to(User)] #[primary_key(user_id, crate_id)] @@ -737,69 +559,6 @@ pub struct Follow { crate_id: i32, } -/// Handles the `GET /crates/:crate_id/versions` route. -// FIXME: Not sure why this is necessary since /crates/:crate_id returns -// this information already, but ember is definitely requesting it -pub fn versions(req: &mut Request) -> CargoResult { - let crate_name = &req.params()["crate_id"]; - let conn = req.db_conn()?; - let krate = Crate::by_name(crate_name).first::(&*conn)?; - let mut versions = Version::belonging_to(&krate).load::(&*conn)?; - versions.sort_by(|a, b| b.num.cmp(&a.num)); - let versions = versions - .into_iter() - .map(|v| v.encodable(crate_name)) - .collect(); - - #[derive(Serialize)] - struct R { - versions: Vec, - } - Ok(req.json(&R { versions: versions })) -} - -/// Handles the `GET /crates/:crate_id/reverse_dependencies` route. -pub fn reverse_dependencies(req: &mut Request) -> CargoResult { - use diesel::expression::dsl::any; - - let name = &req.params()["crate_id"]; - let conn = req.db_conn()?; - let krate = Crate::by_name(name).first::(&*conn)?; - let (offset, limit) = req.pagination(10, 100)?; - let (rev_deps, total) = krate.reverse_dependencies(&*conn, offset, limit)?; - let rev_deps: Vec<_> = rev_deps - .into_iter() - .map(|dep| dep.encodable(&krate.name)) - .collect(); - - let version_ids: Vec = rev_deps.iter().map(|dep| dep.version_id).collect(); - - let versions = versions::table - .filter(versions::id.eq(any(version_ids))) - .inner_join(crates::table) - .select((versions::all_columns, crates::name)) - .load::<(Version, String)>(&*conn)? - .into_iter() - .map(|(version, krate_name)| version.encodable(&krate_name)) - .collect(); - - #[derive(Serialize)] - struct R { - dependencies: Vec, - versions: Vec, - meta: Meta, - } - #[derive(Serialize)] - struct Meta { - total: i64, - } - Ok(req.json(&R { - dependencies: rev_deps, - versions, - meta: Meta { total: total }, - })) -} - use diesel::types::{Date, Text}; sql_function!(canon_crate_name, canon_crate_name_t, (x: Text) -> Text); sql_function!(to_char, to_char_t, (a: Date, b: Text) -> Text); diff --git a/src/lib.rs b/src/lib.rs index 313dcac1d36..144b3e819f0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -157,9 +157,12 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder { api_router.get("/versions/:version_id", C(version::show)); // Routes used by the frontend - api_router.get("/crates/:crate_id", C(krate::show)); + api_router.get("/crates/:crate_id", C(krate::metadata::show)); api_router.get("/crates/:crate_id/:version", C(version::show)); - api_router.get("/crates/:crate_id/:version/readme", C(krate::readme)); + api_router.get( + "/crates/:crate_id/:version/readme", + C(krate::metadata::readme), + ); api_router.get( "/crates/:crate_id/:version/dependencies", C(version::dependencies), @@ -173,7 +176,7 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder { "/crates/:crate_id/downloads", C(krate::downloads::downloads), ); - api_router.get("/crates/:crate_id/versions", C(krate::versions)); + api_router.get("/crates/:crate_id/versions", C(krate::metadata::versions)); api_router.put("/crates/:crate_id/follow", C(krate::follow::follow)); api_router.delete("/crates/:crate_id/follow", C(krate::follow::unfollow)); api_router.get("/crates/:crate_id/following", C(krate::follow::following)); @@ -181,7 +184,7 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder { api_router.get("/crates/:crate_id/owner_user", C(krate::owners::owner_user)); api_router.get( "/crates/:crate_id/reverse_dependencies", - C(krate::reverse_dependencies), + C(krate::metadata::reverse_dependencies), ); api_router.get("/keywords", C(keyword::index)); api_router.get("/keywords/:keyword_id", C(keyword::show)); @@ -205,7 +208,7 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder { "/me/crate_owner_invitations/:crate_id", C(crate_owner_invitation::handle_invite), ); - api_router.get("/summary", C(krate::summary)); + api_router.get("/summary", C(krate::metadata::summary)); api_router.put("/confirm/:email_token", C(user::confirm_user_email)); api_router.put("/users/:user_id/resend", C(user::regenerate_token_and_send)); let api_router = Arc::new(R404(api_router)); From 1364c27d2f3f7022a6a3b20fc438f995d3ce36ee Mon Sep 17 00:00:00 2001 From: Justin Geibel Date: Wed, 11 Oct 2017 19:48:49 -0400 Subject: [PATCH 07/10] Move several version routes to version::deprecated::* --- src/lib.rs | 6 +-- src/version/deprecated.rs | 80 +++++++++++++++++++++++++++++++++++++++ src/version/mod.rs | 59 +---------------------------- 3 files changed, 84 insertions(+), 61 deletions(-) create mode 100644 src/version/deprecated.rs diff --git a/src/lib.rs b/src/lib.rs index 144b3e819f0..699a2ba21d4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -153,12 +153,12 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder { ); // Routes that appear to be unused - api_router.get("/versions", C(version::index)); - api_router.get("/versions/:version_id", C(version::show)); + api_router.get("/versions", C(version::deprecated::index)); + api_router.get("/versions/:version_id", C(version::deprecated::show)); // Routes used by the frontend api_router.get("/crates/:crate_id", C(krate::metadata::show)); - api_router.get("/crates/:crate_id/:version", C(version::show)); + api_router.get("/crates/:crate_id/:version", C(version::deprecated::show)); api_router.get( "/crates/:crate_id/:version/readme", C(krate::metadata::readme), diff --git a/src/version/deprecated.rs b/src/version/deprecated.rs new file mode 100644 index 00000000000..d25a2b48258 --- /dev/null +++ b/src/version/deprecated.rs @@ -0,0 +1,80 @@ +//! Deprecated api endpoints +//! +//! There are no known uses of these endpoints. There is currently no plan for +//! removing these endpoints. At a minimum, logs should be reviewed over a +//! period of time to ensure there are no external users of an endpoint before +//! it is removed. + +use conduit_router::RequestParams; +use conduit::{Request, Response}; +use diesel::prelude::*; +use url; + +use db::RequestTransaction; +use schema::*; +use util::{CargoResult, RequestUtils}; + +use super::{version_and_crate, EncodableVersion, Version}; + +/// Handles the `GET /versions` route. +pub fn index(req: &mut Request) -> CargoResult { + use diesel::expression::dsl::any; + let conn = req.db_conn()?; + + // Extract all ids requested. + let query = url::form_urlencoded::parse(req.query_string().unwrap_or("").as_bytes()); + let ids = query + .filter_map(|(ref a, ref b)| if *a == "ids[]" { + b.parse().ok() + } else { + None + }) + .collect::>(); + + let versions = versions::table + .inner_join(crates::table) + .select((versions::all_columns, crates::name)) + .filter(versions::id.eq(any(ids))) + .load::<(Version, String)>(&*conn)? + .into_iter() + .map(|(version, crate_name)| version.encodable(&crate_name)) + .collect(); + + #[derive(Serialize)] + struct R { + versions: Vec, + } + Ok(req.json(&R { versions: versions })) +} + +/// Handles the `GET /versions/:version_id` and +/// `GET /crates/:crate_id/:version` routes. +/// +/// The frontend doesn't appear to hit either of these endpoints. Instead the +/// version information appears to be returned by `krate::show`. +/// +/// FIXME: These two routes have very different semantics and should be split into +/// a separate function for each endpoint. +pub fn show(req: &mut Request) -> CargoResult { + let (version, krate) = match req.params().find("crate_id") { + Some(..) => version_and_crate(req)?, + None => { + let id = &req.params()["version_id"]; + let id = id.parse().unwrap_or(0); + let conn = req.db_conn()?; + versions::table + .find(id) + .inner_join(crates::table) + .select((versions::all_columns, ::krate::ALL_COLUMNS)) + .first(&*conn)? + } + }; + + #[derive(Serialize)] + struct R { + version: EncodableVersion, + } + Ok(req.json(&R { + version: version.encodable(&krate.name), + })) +} diff --git a/src/version/mod.rs b/src/version/mod.rs index 51866e9c8c1..4964c0cc259 100644 --- a/src/version/mod.rs +++ b/src/version/mod.rs @@ -8,7 +8,6 @@ use diesel::pg::Pg; use diesel::prelude::*; use semver; use serde_json; -use url; use Crate; use app::RequestApp; @@ -22,6 +21,7 @@ use util::errors::CargoError; use util::{human, CargoResult, RequestUtils}; use license_exprs; +pub mod deprecated; pub mod downloads; // Queryable has a custom implementation below @@ -259,63 +259,6 @@ impl Queryable for Version { } } -/// Handles the `GET /versions` route. -// FIXME: where/how is this used? -pub fn index(req: &mut Request) -> CargoResult { - use diesel::expression::dsl::any; - let conn = req.db_conn()?; - - // Extract all ids requested. - let query = url::form_urlencoded::parse(req.query_string().unwrap_or("").as_bytes()); - let ids = query - .filter_map(|(ref a, ref b)| if *a == "ids[]" { - b.parse().ok() - } else { - None - }) - .collect::>(); - - let versions = versions::table - .inner_join(crates::table) - .select((versions::all_columns, crates::name)) - .filter(versions::id.eq(any(ids))) - .load::<(Version, String)>(&*conn)? - .into_iter() - .map(|(version, crate_name)| version.encodable(&crate_name)) - .collect(); - - #[derive(Serialize)] - struct R { - versions: Vec, - } - Ok(req.json(&R { versions: versions })) -} - -/// Handles the `GET /versions/:version_id` route. -pub fn show(req: &mut Request) -> CargoResult { - let (version, krate) = match req.params().find("crate_id") { - Some(..) => version_and_crate(req)?, - None => { - let id = &req.params()["version_id"]; - let id = id.parse().unwrap_or(0); - let conn = req.db_conn()?; - versions::table - .find(id) - .inner_join(crates::table) - .select((versions::all_columns, ::krate::ALL_COLUMNS)) - .first(&*conn)? - } - }; - - #[derive(Serialize)] - struct R { - version: EncodableVersion, - } - Ok(req.json(&R { - version: version.encodable(&krate.name), - })) -} - fn version_and_crate(req: &mut Request) -> CargoResult<(Version, Crate)> { let crate_name = &req.params()["crate_id"]; let semver = &req.params()["version"]; From f65b8ad4337860d0ebbef0a0260632b8db2ebced Mon Sep 17 00:00:00 2001 From: Justin Geibel Date: Wed, 11 Oct 2017 19:54:21 -0400 Subject: [PATCH 08/10] Move yank and unyank to version::yank::* --- src/lib.rs | 7 ++++-- src/version/mod.rs | 51 +-------------------------------------- src/version/yank.rs | 59 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 52 deletions(-) create mode 100644 src/version/yank.rs diff --git a/src/lib.rs b/src/lib.rs index 699a2ba21d4..41410d8326f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -145,8 +145,11 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder { api_router.get("/crates/:crate_id/owners", C(krate::owners::owners)); api_router.put("/crates/:crate_id/owners", C(krate::owners::add_owners)); api_router.delete("/crates/:crate_id/owners", C(krate::owners::remove_owners)); - api_router.delete("/crates/:crate_id/:version/yank", C(version::yank)); - api_router.put("/crates/:crate_id/:version/unyank", C(version::unyank)); + api_router.delete("/crates/:crate_id/:version/yank", C(version::yank::yank)); + api_router.put( + "/crates/:crate_id/:version/unyank", + C(version::yank::unyank), + ); api_router.get( "/crates/:crate_id/:version/download", C(version::downloads::download), diff --git a/src/version/mod.rs b/src/version/mod.rs index 4964c0cc259..9d595de52d9 100644 --- a/src/version/mod.rs +++ b/src/version/mod.rs @@ -10,19 +10,15 @@ use semver; use serde_json; use Crate; -use app::RequestApp; use db::RequestTransaction; use dependency::{Dependency, EncodableDependency}; -use git; -use owner::{rights, Rights}; use schema::*; -use user::RequestUser; -use util::errors::CargoError; use util::{human, CargoResult, RequestUtils}; use license_exprs; pub mod deprecated; pub mod downloads; +pub mod yank; // Queryable has a custom implementation below #[derive(Clone, Identifiable, Associations, Debug)] @@ -323,48 +319,3 @@ pub fn authors(req: &mut Request) -> CargoResult { meta: Meta { names: names }, })) } - -/// Handles the `DELETE /crates/:crate_id/:version/yank` route. -/// This does not delete a crate version, it makes the crate -/// version accessible only to crates that already have a -/// `Cargo.lock` containing this version. -/// -/// Notes: -/// Crate deletion is not implemented to avoid breaking builds, -/// and the goal of yanking a crate is to prevent crates -/// beginning to depend on the yanked crate version. -pub fn yank(req: &mut Request) -> CargoResult { - modify_yank(req, true) -} - -/// Handles the `PUT /crates/:crate_id/:version/unyank` route. -pub fn unyank(req: &mut Request) -> CargoResult { - modify_yank(req, false) -} - -/// Changes `yanked` flag on a crate version record -fn modify_yank(req: &mut Request, yanked: bool) -> CargoResult { - let (version, krate) = version_and_crate(req)?; - let user = req.user()?; - let conn = req.db_conn()?; - let owners = krate.owners(&conn)?; - if rights(req.app(), &owners, user)? < Rights::Publish { - return Err(human("must already be an owner to yank or unyank")); - } - - if version.yanked != yanked { - conn.transaction::<_, Box, _>(|| { - diesel::update(&version) - .set(versions::yanked.eq(yanked)) - .execute(&*conn)?; - git::yank(&**req.app(), &krate.name, &version.num, yanked)?; - Ok(()) - })?; - } - - #[derive(Serialize)] - struct R { - ok: bool, - } - Ok(req.json(&R { ok: true })) -} diff --git a/src/version/yank.rs b/src/version/yank.rs new file mode 100644 index 00000000000..f221b1500b1 --- /dev/null +++ b/src/version/yank.rs @@ -0,0 +1,59 @@ +use conduit::{Request, Response}; +use diesel; +use diesel::prelude::*; + +use app::RequestApp; +use db::RequestTransaction; +use git; +use owner::{rights, Rights}; +use schema::*; +use user::RequestUser; +use util::errors::CargoError; +use util::{human, CargoResult, RequestUtils}; + +use super::version_and_crate; + +/// Handles the `DELETE /crates/:crate_id/:version/yank` route. +/// This does not delete a crate version, it makes the crate +/// version accessible only to crates that already have a +/// `Cargo.lock` containing this version. +/// +/// Notes: +/// Crate deletion is not implemented to avoid breaking builds, +/// and the goal of yanking a crate is to prevent crates +/// beginning to depend on the yanked crate version. +pub fn yank(req: &mut Request) -> CargoResult { + modify_yank(req, true) +} + +/// Handles the `PUT /crates/:crate_id/:version/unyank` route. +pub fn unyank(req: &mut Request) -> CargoResult { + modify_yank(req, false) +} + +/// Changes `yanked` flag on a crate version record +fn modify_yank(req: &mut Request, yanked: bool) -> CargoResult { + let (version, krate) = version_and_crate(req)?; + let user = req.user()?; + let conn = req.db_conn()?; + let owners = krate.owners(&conn)?; + if rights(req.app(), &owners, user)? < Rights::Publish { + return Err(human("must already be an owner to yank or unyank")); + } + + if version.yanked != yanked { + conn.transaction::<_, Box, _>(|| { + diesel::update(&version) + .set(versions::yanked.eq(yanked)) + .execute(&*conn)?; + git::yank(&**req.app(), &krate.name, &version.num, yanked)?; + Ok(()) + })?; + } + + #[derive(Serialize)] + struct R { + ok: bool, + } + Ok(req.json(&R { ok: true })) +} From 8affbac0f1523d88fc3be38077ae3e008f8909e4 Mon Sep 17 00:00:00 2001 From: Justin Geibel Date: Wed, 11 Oct 2017 20:02:15 -0400 Subject: [PATCH 09/10] Move remaining version endpoints to version::* --- src/lib.rs | 7 +++-- src/version/metadata.rs | 65 +++++++++++++++++++++++++++++++++++++++++ src/version/mod.rs | 51 +++----------------------------- 3 files changed, 74 insertions(+), 49 deletions(-) create mode 100644 src/version/metadata.rs diff --git a/src/lib.rs b/src/lib.rs index 41410d8326f..d20e8d88fdc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -168,13 +168,16 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder { ); api_router.get( "/crates/:crate_id/:version/dependencies", - C(version::dependencies), + C(version::metadata::dependencies), ); api_router.get( "/crates/:crate_id/:version/downloads", C(version::downloads::downloads), ); - api_router.get("/crates/:crate_id/:version/authors", C(version::authors)); + api_router.get( + "/crates/:crate_id/:version/authors", + C(version::metadata::authors), + ); api_router.get( "/crates/:crate_id/downloads", C(krate::downloads::downloads), diff --git a/src/version/metadata.rs b/src/version/metadata.rs new file mode 100644 index 00000000000..cc0dd90ad57 --- /dev/null +++ b/src/version/metadata.rs @@ -0,0 +1,65 @@ +//! Endpoints that expose metadata about crate versions +//! +//! These endpoints provide data that could be obtained direclty from the +//! index or cached metadata which was extracted (client side) from the +//! `Cargo.toml` file. + +use conduit::{Request, Response}; + +use diesel::prelude::*; +use db::RequestTransaction; +use dependency::EncodableDependency; +use schema::*; +use util::{CargoResult, RequestUtils}; + +use super::version_and_crate; + +/// Handles the `GET /crates/:crate_id/:version/dependencies` route. +/// +/// This information can be obtained direclty from the index. +/// +/// In addition to returning cached data from the index, this returns +/// fields for `id`, `version_id`, and `downloads` (which appears to always +/// be 0) +pub fn dependencies(req: &mut Request) -> CargoResult { + let (version, _) = version_and_crate(req)?; + let conn = req.db_conn()?; + let deps = version.dependencies(&*conn)?; + let deps = deps.into_iter() + .map(|(dep, crate_name)| dep.encodable(&crate_name, None)) + .collect(); + + #[derive(Serialize)] + struct R { + dependencies: Vec, + } + Ok(req.json(&R { dependencies: deps })) +} + +/// Handles the `GET /crates/:crate_id/:version/authors` route. +pub fn authors(req: &mut Request) -> CargoResult { + let (version, _) = version_and_crate(req)?; + let conn = req.db_conn()?; + let names = version_authors::table + .filter(version_authors::version_id.eq(version.id)) + .select(version_authors::name) + .order(version_authors::name) + .load(&*conn)?; + + // It was imagined that we wold associate authors with users. + // This was never implemented. This complicated return struct + // is all that is left, hear for backwards compatibility. + #[derive(Serialize)] + struct R { + users: Vec<::user::EncodablePublicUser>, + meta: Meta, + } + #[derive(Serialize)] + struct Meta { + names: Vec, + } + Ok(req.json(&R { + users: vec![], + meta: Meta { names: names }, + })) +} diff --git a/src/version/mod.rs b/src/version/mod.rs index 9d595de52d9..2449db90604 100644 --- a/src/version/mod.rs +++ b/src/version/mod.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use chrono::NaiveDateTime; -use conduit::{Request, Response}; +use conduit::Request; use conduit_router::RequestParams; use diesel; use diesel::pg::Pg; @@ -11,13 +11,14 @@ use serde_json; use Crate; use db::RequestTransaction; -use dependency::{Dependency, EncodableDependency}; +use dependency::Dependency; use schema::*; -use util::{human, CargoResult, RequestUtils}; +use util::{human, CargoResult}; use license_exprs; pub mod deprecated; pub mod downloads; +pub mod metadata; pub mod yank; // Queryable has a custom implementation below @@ -275,47 +276,3 @@ fn version_and_crate(req: &mut Request) -> CargoResult<(Version, Crate)> { })?; Ok((version, krate)) } - -/// Handles the `GET /crates/:crate_id/:version/dependencies` route. -pub fn dependencies(req: &mut Request) -> CargoResult { - let (version, _) = version_and_crate(req)?; - let conn = req.db_conn()?; - let deps = version.dependencies(&*conn)?; - let deps = deps.into_iter() - .map(|(dep, crate_name)| dep.encodable(&crate_name, None)) - .collect(); - - #[derive(Serialize)] - struct R { - dependencies: Vec, - } - Ok(req.json(&R { dependencies: deps })) -} - -/// Handles the `GET /crates/:crate_id/:version/authors` route. -pub fn authors(req: &mut Request) -> CargoResult { - let (version, _) = version_and_crate(req)?; - let conn = req.db_conn()?; - let names = version_authors::table - .filter(version_authors::version_id.eq(version.id)) - .select(version_authors::name) - .order(version_authors::name) - .load(&*conn)?; - - // It was imagined that we wold associate authors with users. - // This was never implemented. This complicated return struct - // is all that is left, hear for backwards compatibility. - #[derive(Serialize)] - struct R { - users: Vec<::user::EncodablePublicUser>, - meta: Meta, - } - #[derive(Serialize)] - struct Meta { - names: Vec, - } - Ok(req.json(&R { - users: vec![], - meta: Meta { names: names }, - })) -} From 18cd661a14bb249c2b293b63ea569efb6c2ccd97 Mon Sep 17 00:00:00 2001 From: Justin Geibel Date: Wed, 11 Oct 2017 20:13:41 -0400 Subject: [PATCH 10/10] Add documentation for the new krate and version submodules --- src/krate/downloads.rs | 5 +++++ src/krate/metadata.rs | 6 ++++++ src/krate/publish.rs | 2 ++ src/krate/search.rs | 2 ++ src/version/downloads.rs | 4 ++++ src/version/yank.rs | 2 ++ 6 files changed, 21 insertions(+) diff --git a/src/krate/downloads.rs b/src/krate/downloads.rs index 44974bb93b3..96e9329e271 100644 --- a/src/krate/downloads.rs +++ b/src/krate/downloads.rs @@ -1,3 +1,8 @@ +//! Endpoint for exposing crate download counts +//! +//! The enpoints for download a crate and exposing version specific +//! download counts are located in `krate::downloads`. + use std::cmp; use conduit::{Request, Response}; diff --git a/src/krate/metadata.rs b/src/krate/metadata.rs index a0ff8c7f515..23dd6d278b4 100644 --- a/src/krate/metadata.rs +++ b/src/krate/metadata.rs @@ -1,3 +1,9 @@ +//! Endpoints that expose metadata about a crate +//! +//! These endpoints provide data that could be obtained direclty from the +//! index or cached metadata which was extracted (client side) from the +//! `Cargo.toml` file. + use conduit::{Request, Response}; use conduit_router::RequestParams; use diesel::prelude::*; diff --git a/src/krate/publish.rs b/src/krate/publish.rs index d600bc46c2a..a436e60d287 100644 --- a/src/krate/publish.rs +++ b/src/krate/publish.rs @@ -1,3 +1,5 @@ +//! Functionality related to publishing a new crate or version of a crate. + use std::collections::HashMap; use std::sync::Arc; diff --git a/src/krate/search.rs b/src/krate/search.rs index 3498ad01c57..f95ce337a44 100644 --- a/src/krate/search.rs +++ b/src/krate/search.rs @@ -1,3 +1,5 @@ +//! Endpoint for searching and discovery functionality + use conduit::{Request, Response}; use diesel::prelude::*; use diesel_full_text_search::*; diff --git a/src/version/downloads.rs b/src/version/downloads.rs index 059ab4f81ad..b4117b23b9d 100644 --- a/src/version/downloads.rs +++ b/src/version/downloads.rs @@ -1,3 +1,7 @@ +//! Functionality for downloading crates and maintaining download counts +//! +//! Crate level functionality is located in `krate::downloads`. + use chrono::{Duration, NaiveDate, Utc}; use conduit::{Request, Response}; use conduit_router::RequestParams; diff --git a/src/version/yank.rs b/src/version/yank.rs index f221b1500b1..919069dd435 100644 --- a/src/version/yank.rs +++ b/src/version/yank.rs @@ -1,3 +1,5 @@ +//! Endpoints for yanking and unyanking specific versions of crates + use conduit::{Request, Response}; use diesel; use diesel::prelude::*;