Add doc coverage information

Author: GuillaumeGomez

src/db/add_package.rs

@@ -130,6 +130,26 @@ pub(crate) fn add_package_into_database(
     Ok(release_id)
 }
 
+pub(crate) fn add_doc_coverage(
+    conn: &mut Client,
+    release_id: i32,
+    total_items: i32,
+    documented_items: i32,
+) -> Result<i32> {
+    debug!("Adding doc coverage into database");
+    let rows = conn.query(
+        "INSERT INTO doc_coverage (release_id, total_items, documented_items)
+            VALUES ($1, $2, $3)
+            ON CONFLICT (release_id) DO UPDATE
+                SET
+                    total_items = $2,
+                    documented_items = $3
+            RETURNING release_id",
+        &[&release_id, &total_items, &documented_items],
+    )?;
+    Ok(rows[0].get(0))
+}
+
 /// Adds a build into database
 pub(crate) fn add_build_into_database(
     conn: &mut Client,

src/db/delete.rs

@@ -54,6 +54,7 @@ const METADATA: &[(&str, &str)] = &[
     ("keyword_rels", "rid"),
     ("builds", "rid"),
     ("compression_rels", "release"),
+    ("doc_coverage", "release_id"),
 ];
 
 fn delete_version_from_database(conn: &mut Client, name: &str, version: &str) -> Result<(), Error> {
@@ -65,6 +66,12 @@ fn delete_version_from_database(conn: &mut Client, name: &str, version: &str) ->
             &[&crate_id, &version],
         )?;
     }
+    transaction.execute(
+        "DELETE FROM doc_coverage WHERE release_id = (
+            SELECT id FROM releases WHERE crate_id = $1 AND version = $2
+        )",
+        &[&crate_id, &version],
+    )?;
     transaction.execute(
         "DELETE FROM releases WHERE crate_id = $1 AND version = $2",
         &[&crate_id, &version],

src/db/migrate.rs

@@ -381,6 +381,23 @@ pub fn migrate(version: Option<Version>, conn: &mut Client) -> CratesfyiResult<(
             -- Nope, this is a pure database fix, no going back.
             "
         ),
+        migration!(
+            context,
+            // version
+            16,
+            // description
+            "Create new table for doc coverage",
+            // upgrade query
+            "
+            CREATE TABLE doc_coverage (
+                release_id INT NOT NULL REFERENCES releases(id),
+                total_items INT,
+                documented_items INT
+            );
+            ",
+            // downgrade query
+            "DROP TABLE doc_coverage;"
+        ),
     ];
 
     for migration in migrations {

src/db/mod.rs

@@ -1,7 +1,9 @@
 //! Database operations
 
 pub use self::add_package::update_crate_data_in_database;
-pub(crate) use self::add_package::{add_build_into_database, add_package_into_database};
+pub(crate) use self::add_package::{
+    add_build_into_database, add_doc_coverage, add_package_into_database,
+};
 pub use self::delete::{delete_crate, delete_version};
 pub use self::file::add_path_into_database;
 pub use self::migrate::migrate;

src/docbuilder/rustwide_builder.rs

@@ -3,7 +3,8 @@ use super::Metadata;
 use crate::db::blacklist::is_blacklisted;
 use crate::db::file::add_path_into_database;
 use crate::db::{
-    add_build_into_database, add_package_into_database, update_crate_data_in_database, Pool,
+    add_build_into_database, add_doc_coverage, add_package_into_database,
+    update_crate_data_in_database, Pool,
 };
 use crate::docbuilder::{crates::crates_from_path, Limits};
 use crate::error::Result;
@@ -428,6 +429,12 @@ impl RustwideBuilder {
                     algs,
                 )?;
 
+                if let (Some(total), Some(documented)) =
+                    (res.result.total_items, res.result.documented_items)
+                {
+                    add_doc_coverage(&mut conn, release_id, total, documented)?;
+                }
+
                 add_build_into_database(&mut conn, release_id, &res.result)?;
 
                 // Some crates.io crate data is mutable, so we proactively update it during a release
@@ -469,6 +476,106 @@ impl RustwideBuilder {
         Ok(())
     }
 
+    fn get_coverage(
+        &self,
+        target: &str,
+        build: &Build,
+        metadata: &Metadata,
+        limits: &Limits,
+    ) -> Option<(i32, i32)> {
+        let rustdoc_flags: Vec<String> = vec![
+            "-Z".to_string(),
+            "unstable-options".to_string(),
+            "--static-root-path".to_string(),
+            "/".to_string(),
+            "--cap-lints".to_string(),
+            "warn".to_string(),
+            "--output-format".to_string(),
+            "json".to_string(),
+            "--show-coverage".to_string(),
+        ];
+
+        let mut cargo_args = vec!["doc", "--lib", "--no-deps"];
+        if target != HOST_TARGET {
+            // If the explicit target is not a tier one target, we need to install it.
+            if !TARGETS.contains(&target) {
+                // This is a no-op if the target is already installed.
+                self.toolchain.add_target(&self.workspace, target).ok()?;
+            }
+            cargo_args.push("--target");
+            cargo_args.push(target);
+        };
+
+        let tmp_jobs;
+        if let Some(cpu_limit) = self.cpu_limit {
+            tmp_jobs = format!("-j{}", cpu_limit);
+            cargo_args.push(&tmp_jobs);
+        }
+
+        let tmp;
+        if let Some(features) = &metadata.features {
+            cargo_args.push("--features");
+            tmp = features.join(" ");
+            cargo_args.push(&tmp);
+        }
+        if metadata.all_features {
+            cargo_args.push("--all-features");
+        }
+        if metadata.no_default_features {
+            cargo_args.push("--no-default-features");
+        }
+
+        let mut storage = LogStorage::new(LevelFilter::Info);
+        storage.set_max_size(limits.max_log_size());
+
+        let mut json = String::new();
+        if build
+            .cargo()
+            .timeout(Some(limits.timeout()))
+            .no_output_timeout(None)
+            .env(
+                "RUSTFLAGS",
+                metadata
+                    .rustc_args
+                    .as_ref()
+                    .map(|args| args.join(" "))
+                    .unwrap_or_default(),
+            )
+            .env("RUSTDOCFLAGS", rustdoc_flags.join(" "))
+            // For docs.rs detection from build script:
+            // https://github.com/rust-lang/docs.rs/issues/147
+            .env("DOCS_RS", "1")
+            .args(&cargo_args)
+            .log_output(false)
+            .process_lines(&mut |line, _| {
+                if line.starts_with('{') && line.ends_with('}') {
+                    json = line.to_owned();
+                }
+            })
+            .run()
+            .is_ok()
+        {
+            match serde_json::from_str(&json).expect("conversion failed...") {
+                Value::Object(m) => {
+                    let mut total = 0;
+                    let mut documented = 0;
+                    for entry in m.values() {
+                        if let Some(Value::Number(n)) = entry.get("total") {
+                            total += n.as_i64().unwrap_or(0) as i32;
+                        }
+                        if let Some(Value::Number(n)) = entry.get("with_docs") {
+                            documented += n.as_i64().unwrap_or(0) as i32;
+                        }
+                    }
+                    Some((total, documented))
+                }
+                _ => None,
+            }
+        } else {
+            None
+        }
+    }
+
     fn execute_build(
         &self,
         target: &str,
@@ -556,6 +663,14 @@ impl RustwideBuilder {
                 .run()
                 .is_ok()
         });
+        let mut total_items = None;
+        let mut documented_items = None;
+        if successful {
+            if let Some((total, documented)) = self.get_coverage(target, build, metadata, limits) {
+                total_items = Some(total);
+                documented_items = Some(documented);
+            }
+        }
         // If we're passed a default_target which requires a cross-compile,
         // cargo will put the output in `target/<target>/doc`.
         // However, if this is the default build, we don't want it there,
@@ -575,6 +690,8 @@ impl RustwideBuilder {
                 rustc_version: self.rustc_version.clone(),
                 docsrs_version: format!("docsrs {}", crate::BUILD_VERSION),
                 successful,
+                total_items,
+                documented_items,
             },
             cargo_metadata,
             target: target.to_string(),
@@ -631,4 +748,7 @@ pub(crate) struct BuildResult {
     pub(crate) docsrs_version: String,
     pub(crate) build_log: String,
     pub(crate) successful: bool,
+    // The two next fields are used for the doc coverage.
+    pub(crate) total_items: Option<i32>,
+    pub(crate) documented_items: Option<i32>,
 }

src/test/fakes.rs

@@ -59,6 +59,8 @@ impl<'a> FakeRelease<'a> {
                 docsrs_version: "docs.rs 1.0.0 (000000000 1970-01-01)".into(),
                 build_log: "It works!".into(),
                 successful: true,
+                total_items: None,
+                documented_items: None,
             },
             source_files: Vec::new(),
             rustdoc_files: Vec::new(),

src/web/crate_details.rs

@@ -44,6 +44,7 @@ pub struct CrateDetails {
     pub(crate) doc_targets: Vec<String>,
     license: Option<String>,
     documentation_url: Option<String>,
+    doc_coverage: Option<f32>,
 }
 
 fn optional_markdown<S>(markdown: &Option<String>, serializer: S) -> Result<S::Ok, S::Error>
@@ -97,9 +98,12 @@ impl CrateDetails {
                 releases.doc_targets,
                 releases.license,
                 releases.documentation_url,
-                releases.default_target
+                releases.default_target,
+                doc_coverage.total_items,
+                doc_coverage.documented_items
             FROM releases
             INNER JOIN crates ON releases.crate_id = crates.id
+            LEFT JOIN doc_coverage ON doc_coverage.release_id = releases.id
             WHERE crates.name = $1 AND releases.version = $2;";
 
         let rows = conn.query(query, &[&name, &version]).unwrap();
@@ -180,6 +184,16 @@ impl CrateDetails {
             doc_targets,
             license: krate.get("license"),
             documentation_url: krate.get("documentation_url"),
+            doc_coverage: {
+                let total: Option<i32> = krate.get("total_items");
+                let documented: Option<i32> = krate.get("documented_items");
+                match (total, documented) {
+                    (Some(total), Some(documented)) => {
+                        Some(documented as f32 * 100. / total as f32)
+                    }
+                    _ => None,
+                }
+            },
         };
 
         if let Some(repository_url) = crate_details.repository_url.clone() {

templates/crate/details.html

@@ -16,6 +16,10 @@
             <div class="pure-u-1 pure-u-sm-7-24 pure-u-md-5-24">
                 <div class="pure-menu package-menu">
                     <ul class="pure-menu-list">
+                        {%- if details.doc_coverage -%}
+                            <li class="pure-menu-heading">Documentation<br>coverage</li>
+                            <li class="pure-menu-item" style="text-align:center;">{{ details.doc_coverage | round(precision=2) }} %</li>
+                        {%- endif -%}
                         {# List the release author's names and a link to their docs.rs profile #}
                         <li class="pure-menu-heading">Authors</li>
                         {%- for author in details.authors -%}