Skip to content

Commit d8b8d33

Browse files
GuillaumeGomezGuillaumeGomez
committed
* Add Pool field into RepositoryStatsUpdater
* Return a default icon * Put back cron initialization into daemon
1 parent 25702f1 commit d8b8d33

File tree

6 files changed

+52
-62
lines changed

6 files changed

+52
-62
lines changed

‎src/bin/cratesfyi.rs

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -423,15 +423,11 @@ impl DatabaseSubcommand {
423423
}
424424

425425
Self::UpdateRepositoryFields => {
426-
let config = ctx.config()?;
427-
let updater = RepositoryStatsUpdater::new(&config);
428-
updater.update_all_crates(&ctx.pool()?)?;
426+
ctx.repository_stats_updater()?.update_all_crates()?;
429427
}
430428

431429
Self::BackfillRepositoryStats => {
432-
let config = ctx.config()?;
433-
let updater = RepositoryStatsUpdater::new(&config);
434-
updater.backfill_repositories(&ctx)?;
430+
ctx.repository_stats_updater()?.backfill_repositories()?;
435431
}
436432

437433
Self::UpdateCrateRegistryFields { name } => {
@@ -626,7 +622,8 @@ impl Context for BinContext {
626622
.repository_stats_updater
627623
.get_or_try_init::<_, Error>(|| {
628624
let config = self.config()?;
629-
Ok(Arc::new(RepositoryStatsUpdater::new(&config)))
625+
let pool = self.pool()?;
626+
Ok(Arc::new(RepositoryStatsUpdater::new(&config, pool)))
630627
})?
631628
.clone())
632629
}

‎src/docbuilder/rustwide_builder.rs

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -367,7 +367,7 @@ impl RustwideBuilder {
367367
};
368368

369369
let cargo_metadata = res.cargo_metadata.root();
370-
let repository = self.get_repo(&mut conn, cargo_metadata)?;
370+
let repository = self.get_repo(cargo_metadata)?;
371371

372372
let release_id = add_package_into_database(
373373
&mut conn,
@@ -654,9 +654,8 @@ impl RustwideBuilder {
654654
}
655655
}
656656

657-
fn get_repo(&self, conn: &mut Client, metadata: &MetadataPackage) -> Result<Option<i32>> {
658-
self.repository_stats_updater
659-
.load_repository(conn, metadata)
657+
fn get_repo(&self, metadata: &MetadataPackage) -> Result<Option<i32>> {
658+
self.repository_stats_updater.load_repository(metadata)
660659
}
661660
}
662661

‎src/repositories/gitlab.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ pub struct GitLab {
4444
}
4545

4646
impl GitLab {
47-
pub fn new(host: &'static str, access_token: &Option<String>) -> Result<Option<Self>> {
47+
pub fn new(host: &'static str, access_token: &Option<String>) -> Result<Self> {
4848
let mut headers = HeaderMap::new();
4949
headers.insert(USER_AGENT, HeaderValue::from_static(APP_USER_AGENT));
5050
headers.insert(ACCEPT, HeaderValue::from_static("application/json"));
@@ -62,7 +62,7 @@ impl GitLab {
6262
}
6363

6464
let client = HttpClient::builder().default_headers(headers).build()?;
65-
Ok(Some(GitLab { client, host }))
65+
Ok(GitLab { client, host })
6666
}
6767
}
6868

‎src/repositories/updater.rs

Lines changed: 22 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,14 @@
11
use crate::error::Result;
22
use crate::repositories::{GitHub, GitLab};
3-
use crate::utils::{daemon::cron, MetadataPackage};
4-
use crate::{db::Pool, Config, Context};
3+
use crate::utils::MetadataPackage;
4+
use crate::{db::Pool, Config};
55
use chrono::{DateTime, Utc};
66
use log::{debug, info, trace, warn};
77
use once_cell::sync::Lazy;
88
use postgres::Client;
99
use regex::Regex;
1010
use std::collections::{HashMap, HashSet};
1111
use std::fmt;
12-
use std::sync::Arc;
13-
use std::time::Duration;
1412

1513
pub trait RepositoryForge {
1614
/// Result used both as the `host` column in the DB and to match repository URLs during
@@ -52,6 +50,7 @@ pub struct FetchRepositoriesResult {
5250

5351
pub struct RepositoryStatsUpdater {
5452
updaters: Vec<Box<dyn RepositoryForge + Send + Sync>>,
53+
pool: Pool,
5554
}
5655

5756
impl fmt::Debug for RepositoryStatsUpdater {
@@ -65,33 +64,30 @@ impl fmt::Debug for RepositoryStatsUpdater {
6564
}
6665

6766
impl RepositoryStatsUpdater {
68-
pub fn new(config: &Config) -> Self {
69-
let mut updaters: Vec<Box<dyn RepositoryForge + Send + Sync>> = Vec::with_capacity(3);
67+
pub fn new(config: &Config, pool: Pool) -> Self {
68+
let mut updaters: Vec<Box<dyn RepositoryForge + Send + Sync>> = Vec::new();
7069
if let Ok(Some(updater)) = GitHub::new(&config) {
7170
updaters.push(Box::new(updater));
7271
}
73-
if let Ok(Some(updater)) = GitLab::new("gitlab.com", &config.gitlab_accesstoken) {
72+
if let Ok(updater) = GitLab::new("gitlab.com", &config.gitlab_accesstoken) {
7473
updaters.push(Box::new(updater));
7574
}
76-
if let Ok(Some(updater)) = GitLab::new("gitlab.freedesktop.org", &None) {
75+
if let Ok(updater) = GitLab::new("gitlab.freedesktop.org", &None) {
7776
updaters.push(Box::new(updater));
7877
}
79-
Self { updaters }
78+
Self { updaters, pool }
8079
}
8180

82-
pub(crate) fn load_repository(
83-
&self,
84-
conn: &mut Client,
85-
metadata: &MetadataPackage,
86-
) -> Result<Option<i32>> {
81+
pub(crate) fn load_repository(&self, metadata: &MetadataPackage) -> Result<Option<i32>> {
8782
let url = match &metadata.repository {
8883
Some(url) => url,
8984
None => {
9085
debug!("did not collect stats as no repository URL was present");
9186
return Ok(None);
9287
}
9388
};
94-
self.load_repository_inner(conn, url)
89+
let mut conn = self.pool.get()?;
90+
self.load_repository_inner(&mut conn, url)
9591
}
9692

9793
fn load_repository_inner(&self, conn: &mut Client, url: &str) -> Result<Option<i32>> {
@@ -109,7 +105,7 @@ impl RepositoryStatsUpdater {
109105
}
110106
for updater in self.updaters.iter().filter(|u| u.host() == name.host) {
111107
let res = match updater.fetch_repository(&name) {
112-
Ok(Some(repo)) => Self::store_repository(conn, updater.host(), repo),
108+
Ok(Some(repo)) => self.store_repository(conn, updater.host(), repo),
113109
Err(err) => {
114110
warn!("failed to collect `{}` stats: {}", updater.host(), err);
115111
return Ok(None);
@@ -128,19 +124,8 @@ impl RepositoryStatsUpdater {
128124
Ok(None)
129125
}
130126

131-
pub fn start_crons(config: Arc<Config>, pool: Pool) -> Result<()> {
132-
let instance = Self::new(&config);
133-
cron(
134-
"repositories stats updater",
135-
Duration::from_secs(60 * 60),
136-
move || instance.update_all_crates(&pool),
137-
)?;
138-
139-
Ok(())
140-
}
141-
142-
pub fn update_all_crates(&self, pool: &Pool) -> Result<()> {
143-
let mut conn = pool.get()?;
127+
pub fn update_all_crates(&self) -> Result<()> {
128+
let mut conn = self.pool.get()?;
144129
for updater in &self.updaters {
145130
info!("started updating `{}` repositories stats", updater.host());
146131

@@ -165,20 +150,19 @@ impl RepositoryStatsUpdater {
165150
for chunk in needs_update.chunks(updater.chunk_size()) {
166151
let res = updater.fetch_repositories(chunk)?;
167152
for node in res.missing {
168-
Self::delete_repository(&mut conn, &node, updater.host())?;
153+
self.delete_repository(&mut conn, &node, updater.host())?;
169154
}
170155
for (_, repo) in res.present {
171-
Self::store_repository(&mut conn, updater.host(), repo)?;
156+
self.store_repository(&mut conn, updater.host(), repo)?;
172157
}
173158
}
174159
info!("finished updating `{}` repositories stats", updater.host());
175160
}
176161
Ok(())
177162
}
178163

179-
pub fn backfill_repositories(&self, ctx: &dyn Context) -> Result<()> {
180-
let pool = ctx.pool()?;
181-
let mut conn = pool.get()?;
164+
pub fn backfill_repositories(&self) -> Result<()> {
165+
let mut conn = self.pool.get()?;
182166
for updater in &self.updaters {
183167
info!(
184168
"started backfilling `{}` repositories stats",
@@ -234,10 +218,11 @@ impl RepositoryStatsUpdater {
234218
return updater.icon();
235219
}
236220
}
237-
""
221+
// The default icon in case it doesn't match any of the "known" ones.
222+
"code-branch"
238223
}
239224

240-
fn store_repository(conn: &mut Client, host: &str, repo: Repository) -> Result<i32> {
225+
fn store_repository(&self, conn: &mut Client, host: &str, repo: Repository) -> Result<i32> {
241226
trace!(
242227
"storing {} repository stats for {}",
243228
host,
@@ -271,7 +256,7 @@ impl RepositoryStatsUpdater {
271256
Ok(data.get(0))
272257
}
273258

274-
fn delete_repository(conn: &mut Client, host_id: &str, host: &str) -> Result<()> {
259+
fn delete_repository(&self, conn: &mut Client, host_id: &str, host: &str) -> Result<()> {
275260
trace!(
276261
"removing repository stats for host ID `{}` and host `{}`",
277262
host_id,

‎src/test/mod.rs

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,12 @@ impl TestEnvironment {
217217

218218
pub(crate) fn repository_stats_updater(&self) -> Arc<RepositoryStatsUpdater> {
219219
self.repository_stats_updater
220-
.get_or_init(|| Arc::new(RepositoryStatsUpdater::new(&self.config())))
220+
.get_or_init(|| {
221+
Arc::new(RepositoryStatsUpdater::new(
222+
&self.config(),
223+
self.pool().expect("failed to get the pool"),
224+
))
225+
})
221226
.clone()
222227
}
223228

@@ -277,7 +282,9 @@ impl TestDatabase {
277282
// A random schema name is generated and used for the current connection. This allows each
278283
// test to create a fresh instance of the database to run within.
279284
let schema = format!("docs_rs_test_schema_{}", rand::random::<u64>());
280-
let repository_stats_updater = RepositoryStatsUpdater::new(&config);
285+
286+
let pool = Pool::new_with_schema(&config, metrics, &schema)?;
287+
let repository_stats_updater = RepositoryStatsUpdater::new(config, pool.clone());
281288

282289
let mut conn = Connection::connect(&config.database_url, postgres::NoTls)?;
283290
conn.batch_execute(&format!(
@@ -313,7 +320,7 @@ impl TestDatabase {
313320
conn.batch_execute(&query)?;
314321

315322
Ok(TestDatabase {
316-
pool: Pool::new_with_schema(config, metrics, &schema)?,
323+
pool,
317324
schema,
318325
repository_stats_updater,
319326
})

‎src/utils/daemon.rs

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,7 @@
22
//!
33
//! This daemon will start web server, track new packages and build them
44
5-
use crate::{
6-
repositories::RepositoryStatsUpdater,
7-
utils::queue_builder,
8-
Context, DocBuilder, RustwideBuilder,
9-
};
5+
use crate::{utils::queue_builder, Context, DocBuilder, RustwideBuilder};
106
use failure::Error;
117
use log::{debug, error, info};
128
use std::thread;
@@ -67,18 +63,24 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res
6763
// build new crates every minute
6864
let pool = context.pool()?;
6965
let build_queue = context.build_queue()?;
70-
let cloned_config = config.clone();
7166
let rustwide_builder = RustwideBuilder::init(context)?;
7267
thread::Builder::new()
7368
.name("build queue reader".to_string())
7469
.spawn(move || {
75-
let doc_builder =
76-
DocBuilder::new(cloned_config.clone(), pool.clone(), build_queue.clone());
70+
let doc_builder = DocBuilder::new(config.clone(), pool.clone(), build_queue.clone());
7771
queue_builder(doc_builder, rustwide_builder, build_queue).unwrap();
7872
})
7973
.unwrap();
8074

81-
RepositoryStatsUpdater::start_crons(config, context.pool()?)?;
75+
let updater = context.repository_stats_updater()?;
76+
cron(
77+
"repositories stats updater",
78+
Duration::from_secs(60 * 60),
79+
move || {
80+
updater.update_all_crates()?;
81+
Ok(())
82+
},
83+
)?;
8284

8385
// Never returns; `server` blocks indefinitely when dropped
8486
// NOTE: if a failure occurred earlier in `start_daemon`, the server will _not_ be joined -

0 commit comments

Comments
 (0)