Merge pull request #344 from QuietMisdreavus/build-queue-priority

add "priority" to the build queue, and decouple builds from reading new crates

Author: QuietMisdreavus

src/bin/cratesfyi.rs

@@ -12,9 +12,9 @@ use std::path::PathBuf;
 
 use clap::{Arg, App, SubCommand};
 use cratesfyi::{DocBuilder, DocBuilderOptions, db};
-use cratesfyi::utils::build_doc;
+use cratesfyi::utils::{build_doc, add_crate_to_queue};
 use cratesfyi::start_web_server;
-use cratesfyi::db::add_path_into_database;
+use cratesfyi::db::{add_path_into_database, connect_db};
 
 
 pub fn main() {
@@ -130,6 +130,23 @@ pub fn main() {
                                                               chart")
             .subcommand(SubCommand::with_name("update-search-index"))
             .about("Updates search index"))
+        .subcommand(SubCommand::with_name("queue")
+            .about("Interactions with the build queue")
+            .subcommand(SubCommand::with_name("add")
+                .about("Add a crate to the build queue")
+                .arg(Arg::with_name("CRATE_NAME")
+                    .index(1)
+                    .required(true)
+                    .help("Name of crate to build"))
+                .arg(Arg::with_name("CRATE_VERSION")
+                    .index(2)
+                    .required(true)
+                    .help("Version of crate to build"))
+                .arg(Arg::with_name("BUILD_PRIORITY")
+                    .short("p")
+                    .long("priority")
+                    .help("Priority of build (default: 5) (new crate builds get priority 0)")
+                    .takes_value(true))))
         .get_matches();
 
 
@@ -227,6 +244,17 @@ pub fn main() {
         start_web_server(Some(matches.value_of("SOCKET_ADDR").unwrap_or("0.0.0.0:3000")));
     } else if let Some(_) = matches.subcommand_matches("daemon") {
         cratesfyi::utils::start_daemon();
+    } else if let Some(matches) = matches.subcommand_matches("queue") {
+        if let Some(matches) = matches.subcommand_matches("add") {
+            let priority = matches.value_of("BUILD_PRIORITY").unwrap_or("5");
+            let priority: i32 = priority.parse().expect("--priority was not a number");
+            let conn = connect_db().expect("Could not connect to database");
+
+            add_crate_to_queue(&conn,
+                               matches.value_of("CRATE_NAME").unwrap(),
+                               matches.value_of("CRATE_VERSION").unwrap(),
+                               priority).expect("Could not add crate to queue");
+        }
     } else {
         println!("{}", matches.usage());
     }

src/db/migrate.rs

@@ -169,6 +169,16 @@ pub fn migrate(version: Option<Version>) -> CratesfyiResult<()> {
             "DROP TABLE authors, author_rels, keyword_rels, keywords, owner_rels,
                         owners, releases, crates, builds, queue, files, config;"
         ),
+        migration!(
+            // version
+            2,
+            // description
+            "Added priority column to build queue",
+            // upgrade query
+            "ALTER TABLE queue ADD COLUMN priority INT DEFAULT 0;",
+            // downgrade query
+            "ALTER TABLE queue DROP COLUMN priority;"
+        ),
     ];
 
     for migration in migrations {

src/docbuilder/mod.rs

@@ -110,6 +110,11 @@ impl DocBuilder {
         Ok(())
     }
 
+    /// Checks for the lock file and returns whether it currently exists.
+    pub fn is_locked(&self) -> bool {
+        self.lock_path().exists()
+    }
+
     /// Returns a reference of options
     pub fn options(&self) -> &DocBuilderOptions {
         &self.options

src/docbuilder/queue.rs

@@ -5,67 +5,74 @@ use super::DocBuilder;
 use db::connect_db;
 use error::Result;
 use crates_index_diff::{ChangeKind, Index};
+use utils::add_crate_to_queue;
 
 
 impl DocBuilder {
     /// Updates crates.io-index repository and adds new crates into build queue.
     /// Returns size of queue
-    pub fn get_new_crates(&mut self) -> Result<i64> {
+    pub fn get_new_crates(&mut self) -> Result<usize> {
         let conn = try!(connect_db());
         let index = try!(Index::from_path_or_cloned(&self.options.crates_io_index_path));
         let mut changes = try!(index.fetch_changes());
+        let mut add_count: usize = 0;
 
         // I belive this will fix ordering of queue if we get more than one crate from changes
         changes.reverse();
 
         for krate in changes.iter().filter(|k| k.kind != ChangeKind::Yanked) {
-            conn.execute("INSERT INTO queue (name, version) VALUES ($1, $2)",
-                         &[&krate.name, &krate.version])
-                .ok();
+            add_crate_to_queue(&conn, &krate.name, &krate.version, 0).ok();
             debug!("{}-{} added into build queue", krate.name, krate.version);
+            add_count += 1;
         }
 
-        let queue_count = conn.query("SELECT COUNT(*) FROM queue WHERE attempt < 5", &[])
+        Ok(add_count)
+    }
+
+    pub fn get_queue_count(&self) -> Result<i64> {
+        let conn = try!(connect_db());
+        Ok(conn.query("SELECT COUNT(*) FROM queue WHERE attempt < 5", &[])
             .unwrap()
             .get(0)
-            .get(0);
-
-        Ok(queue_count)
+            .get(0))
     }
 
-
-    /// Builds packages from queue
-    pub fn build_packages_queue(&mut self) -> Result<usize> {
+    /// Builds the top package from the queue. Returns whether the queue was empty.
+    pub fn build_next_queue_package(&mut self) -> Result<bool> {
         let conn = try!(connect_db());
-        let mut build_count = 0;
 
-        for row in &try!(conn.query("SELECT id, name, version
+        let query = try!(conn.query("SELECT id, name, version
                                      FROM queue
                                      WHERE attempt < 5
-                                     ORDER BY id ASC",
-                                    &[])) {
-            let id: i32 = row.get(0);
-            let name: String = row.get(1);
-            let version: String = row.get(2);
-
-            match self.build_package(&name[..], &version[..]) {
-                Ok(_) => {
-                    build_count += 1;
-                    let _ = conn.execute("DELETE FROM queue WHERE id = $1", &[&id]);
-                }
-                Err(e) => {
-                    // Increase attempt count
-                    let _ = conn.execute("UPDATE queue SET attempt = attempt + 1 WHERE id = $1",
-                                         &[&id]);
-                    error!("Failed to build package {}-{} from queue: {}",
-                           name,
-                           version,
-                           e)
-                }
+                                     ORDER BY priority ASC, attempt ASC, id ASC
+                                     LIMIT 1",
+                                    &[]));
+
+        if query.is_empty() {
+            // nothing in the queue; bail
+            return Ok(false);
+        }
+
+        let id: i32 = query.get(0).get(0);
+        let name: String = query.get(0).get(1);
+        let version: String = query.get(0).get(2);
+
+        match self.build_package(&name[..], &version[..]) {
+            Ok(_) => {
+                let _ = conn.execute("DELETE FROM queue WHERE id = $1", &[&id]);
+            }
+            Err(e) => {
+                // Increase attempt count
+                let _ = conn.execute("UPDATE queue SET attempt = attempt + 1 WHERE id = $1",
+                                     &[&id]);
+                error!("Failed to build package {}-{} from queue: {}",
+                       name,
+                       version,
+                       e)
             }
         }
 
-        Ok(build_count)
+        Ok(true)
     }
 }
 
@@ -87,14 +94,4 @@ mod test {
         }
         assert!(res.is_ok());
     }
-
-
-    #[test]
-    #[ignore]
-    fn test_build_packages_queue() {
-        let _ = env_logger::try_init();
-        let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix"));
-        let mut docbuilder = DocBuilder::new(options);
-        assert!(docbuilder.build_packages_queue().is_ok());
-    }
 }