Skip to content

Commit d5a3d57

Browse files
GuillaumeGomezGuillaumeGomez
committed
Improve code
1 parent 6b6c774 commit d5a3d57

File tree

5 files changed

+57
-63
lines changed

5 files changed

+57
-63
lines changed

‎README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -190,10 +190,10 @@ docker-compose run -- database add-directory <DIRECTORY> [PREFIX]
190190
# Updates github stats for crates.
191191
# You need to set CRATESFYI_GITHUB_USERNAME, CRATESFYI_GITHUB_ACCESSTOKEN
192192
# environment variables in order to run this command.
193-
# Even though it's not mandatory, setting CRATESFYI_GITLAB_ACCESSTOKEN
194-
# environment variable might be a good idea too.
193+
# Set CRATESFYI_GITLAB_ACCESSTOKEN to raise the rate limit,
194+
# or leave it blank to fetch repositories at a slower rate.
195195
# You can set this environment variables in ~/.cratesfyi.env file.
196-
cargo run -- database update-repositories-fields
196+
cargo run -- database update-repository-fields
197197
```
198198

199199
If you want to explore or edit database manually, you can connect to the database

‎src/bin/cratesfyi.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -372,10 +372,10 @@ enum DatabaseSubcommand {
372372
},
373373

374374
/// Updates Github/Gitlab stats for crates.
375-
UpdateRepositoriesFields,
375+
UpdateRepositoryFields,
376376

377377
/// Backfill GitHub/Gitlab stats for crates.
378-
BackfillRepositoriesStats,
378+
BackfillRepositoryStats,
379379

380380
/// Updates info for a crate from the registry's API
381381
UpdateCrateRegistryFields {
@@ -424,11 +424,11 @@ impl DatabaseSubcommand {
424424
.context("Failed to run database migrations")?;
425425
}
426426

427-
Self::UpdateRepositoriesFields => {
427+
Self::UpdateRepositoryFields => {
428428
RepositoryStatsUpdater::update_all_crates(&ctx)?;
429429
}
430430

431-
Self::BackfillRepositoriesStats => {
431+
Self::BackfillRepositoryStats => {
432432
RepositoryStatsUpdater::backfill_repositories(&ctx)?;
433433
}
434434

‎src/test/fakes.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -370,14 +370,14 @@ impl FakeGithubStats {
370370
.get(0);
371371
let host_id = base64::encode(format!("FAKE ID {}", existing_count));
372372

373-
let data = conn.query(
373+
let data = conn.query_one(
374374
"INSERT INTO repositories (host, host_id, name, description, last_commit, stars, forks, issues, updated_at)
375375
VALUES ('github', $1, $2, 'Fake description!', NOW(), $3, $4, $5, NOW())
376376
RETURNING id;",
377377
&[&host_id, &self.repo, &self.stars, &self.forks, &self.issues],
378378
)?;
379379

380-
Ok(data[0].get(0))
380+
Ok(data.get(0))
381381
}
382382
}
383383

‎src/utils/github_updater.rs

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -193,17 +193,13 @@ impl Updater for GithubUpdater {
193193
.unwrap()
194194
});
195195

196-
match RE.captures(url) {
197-
Some(cap) => {
198-
let owner = cap.name("owner").expect("missing group 'owner'").as_str();
199-
let repo = cap.name("repo").expect("missing group 'repo'").as_str();
200-
Some(RepositoryName {
201-
owner,
202-
repo: repo.strip_suffix(".git").unwrap_or(repo),
203-
})
204-
}
205-
None => None,
206-
}
196+
let cap = RE.captures(url)?;
197+
let owner = cap.name("owner").expect("missing group 'owner'").as_str();
198+
let repo = cap.name("repo").expect("missing group 'repo'").as_str();
199+
Some(RepositoryName {
200+
owner,
201+
repo: repo.strip_suffix(".git").unwrap_or(repo),
202+
})
207203
}
208204

209205
fn name() -> &'static str {
@@ -273,7 +269,7 @@ impl GithubUpdater {
273269
"storing GitHub repository stats for {}",
274270
repo.name_with_owner
275271
);
276-
let rows = conn.query(
272+
let data = conn.query_one(
277273
"INSERT INTO repositories (
278274
host, host_id, name, description, last_commit, stars, forks, issues, updated_at
279275
) VALUES ('github', $1, $2, $3, $4, $5, $6, $7, NOW())
@@ -297,7 +293,7 @@ impl GithubUpdater {
297293
&(repo.issues.total_count as i32),
298294
],
299295
)?;
300-
Ok(rows[0].get(0))
296+
Ok(data.get(0))
301297
}
302298
}
303299

‎src/utils/gitlab_updater.rs

Lines changed: 39 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -81,32 +81,34 @@ impl Updater for GitlabUpdater {
8181
info!("started backfilling Gitlab repository stats");
8282

8383
let mut conn = self.pool.get()?;
84-
let needs_backfilling = conn.query(
85-
"SELECT releases.id, crates.name, releases.version, releases.repository_url
86-
FROM releases
87-
INNER JOIN crates ON (crates.id = releases.crate_id)
88-
WHERE repository IS NULL AND repository_url LIKE '%gitlab.%/%';",
89-
&[],
90-
)?;
84+
for host in ALLOWED_HOSTS {
85+
let needs_backfilling = conn.query(
86+
"SELECT releases.id, crates.name, releases.version, releases.repository_url
87+
FROM releases
88+
INNER JOIN crates ON (crates.id = releases.crate_id)
89+
WHERE repository IS NULL AND repository_url = $1;",
90+
&[host],
91+
)?;
9192

92-
let mut missing_urls = HashSet::new();
93-
for row in &needs_backfilling {
94-
let id: i32 = row.get("id");
95-
let name: String = row.get("name");
96-
let version: String = row.get("version");
97-
let url: String = row.get("repository_url");
98-
99-
if missing_urls.contains(&url) {
100-
eprintln!("{} {} points to a known missing repo", name, version);
101-
} else if let Some(node_id) = self.load_repository(&mut conn, &url)? {
102-
conn.execute(
103-
"UPDATE releases SET repository = $1 WHERE id = $2;",
104-
&[&node_id, &id],
105-
)?;
106-
info!("backfilled Gitlab repository for {} {}", name, version);
107-
} else {
108-
eprintln!("{} {} does not point to a Gitlab repository", name, version);
109-
missing_urls.insert(url);
93+
let mut missing_urls = HashSet::new();
94+
for row in &needs_backfilling {
95+
let id: i32 = row.get("id");
96+
let name: String = row.get("name");
97+
let version: String = row.get("version");
98+
let url: String = row.get("repository_url");
99+
100+
if missing_urls.contains(&url) {
101+
eprintln!("{} {} points to a known missing repo", name, version);
102+
} else if let Some(node_id) = self.load_repository(&mut conn, &url)? {
103+
conn.execute(
104+
"UPDATE releases SET repository = $1 WHERE id = $2;",
105+
&[&node_id, &id],
106+
)?;
107+
info!("backfilled Gitlab repository for {} {}", name, version);
108+
} else {
109+
eprintln!("{} {} does not point to a Gitlab repository", name, version);
110+
missing_urls.insert(url);
111+
}
110112
}
111113
}
112114

@@ -203,21 +205,17 @@ impl Updater for GitlabUpdater {
203205
Regex::new(r"https?://(?P<host>.+)/(?P<owner>[\w\._-]+)/(?P<repo>[\w\._-]+)").unwrap()
204206
});
205207

206-
match RE.captures(url) {
207-
Some(cap) => {
208-
let host = cap.name("host").expect("missing group 'host'").as_str();
209-
if !ALLOWED_HOSTS.iter().any(|s| *s == host) {
210-
return None;
211-
}
212-
let owner = cap.name("owner").expect("missing group 'owner'").as_str();
213-
let repo = cap.name("repo").expect("missing group 'repo'").as_str();
214-
Some(RepositoryName {
215-
owner,
216-
repo: repo.strip_suffix(".git").unwrap_or(repo),
217-
})
218-
}
219-
None => None,
208+
let cap = RE.captures(url)?;
209+
let host = cap.name("host").expect("missing group 'host'").as_str();
210+
if !ALLOWED_HOSTS.iter().any(|s| *s == host) {
211+
return None;
220212
}
213+
let owner = cap.name("owner").expect("missing group 'owner'").as_str();
214+
let repo = cap.name("repo").expect("missing group 'repo'").as_str();
215+
Some(RepositoryName {
216+
owner,
217+
repo: repo.strip_suffix(".git").unwrap_or(repo),
218+
})
221219
}
222220

223221
fn name() -> &'static str {
@@ -279,7 +277,7 @@ impl GitlabUpdater {
279277
"storing Gitlab repository stats for {}",
280278
repo.name_with_namespace
281279
);
282-
let rows = conn.query(
280+
let data = conn.query_one(
283281
"INSERT INTO repositories (
284282
host, host_id, name, description, last_commit, stars, forks, issues, updated_at
285283
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())
@@ -304,7 +302,7 @@ impl GitlabUpdater {
304302
&(repo.open_issues_count as i32),
305303
],
306304
)?;
307-
Ok(rows[0].get(0))
305+
Ok(data.get(0))
308306
}
309307
}
310308

0 commit comments

Comments
 (0)