Skip to content

Commit bd90856

Browse files
pietroalbinipietroalbini
committed
db: add the cratesfyi database delete-crate command
1 parent 21030d8 commit bd90856

File tree

4 files changed

+132
-6
lines changed

4 files changed

+132
-6
lines changed

‎src/bin/cratesfyi.rs

+8-1
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,11 @@ pub fn main() {
111111
.subcommand(SubCommand::with_name("update-release-activity"))
112112
.about("Updates montly release activity \
113113
chart")
114-
.subcommand(SubCommand::with_name("update-search-index"))
114+
.subcommand(SubCommand::with_name("update-search-index")
115115
.about("Updates search index"))
116+
.subcommand(SubCommand::with_name("delete-crate")
117+
.about("Removes a whole crate from the database")
118+
.arg(Arg::with_name("CRATE_NAME").help("Name of the crate to delete"))))
116119
.subcommand(SubCommand::with_name("queue")
117120
.about("Interactions with the build queue")
118121
.subcommand(SubCommand::with_name("add")
@@ -225,6 +228,10 @@ pub fn main() {
225228
count, total
226229
);
227230
}
231+
} else if let Some(matches) = matches.subcommand_matches("delete-crate") {
232+
let name = matches.value_of("CRATE_NAME").expect("missing crate name");
233+
let conn = db::connect_db().expect("failed to connect to the database");
234+
db::delete_crate(&conn, &name).expect("failed to delete the crate");
228235
}
229236
} else if let Some(matches) = matches.subcommand_matches("start-web-server") {
230237
start_web_server(Some(matches.value_of("SOCKET_ADDR").unwrap_or("0.0.0.0:3000")));

‎src/db/delete_crate.rs

+115
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
use super::file::{s3_client, S3_BUCKET_NAME};
2+
use failure::Error;
3+
use postgres::Connection;
4+
use rusoto_s3::{DeleteObjectsRequest, ListObjectsV2Request, ObjectIdentifier, S3Client, S3};
5+
6+
/// List of directories in docs.rs's underlying storage (either the database or S3) containing a
7+
/// subdirectory named after the crate. Those subdirectories will be deleted.
8+
static STORAGE_PATHS_TO_DELETE: &[&str] = &["rustdoc", "sources"];
9+
10+
#[derive(Debug, Fail)]
11+
enum CrateDeletionError {
12+
#[fail(display = "crate is missing: {}", _0)]
13+
MissingCrate(String),
14+
}
15+
16+
pub fn delete_crate(conn: &Connection, name: &str) -> Result<(), Error> {
17+
let crate_id_res = conn.query("SELECT id FROM crates WHERE name = $1", &[&name])?;
18+
let crate_id = if crate_id_res.is_empty() {
19+
return Err(CrateDeletionError::MissingCrate(name.into()).into());
20+
} else {
21+
crate_id_res.get(0).get("id")
22+
};
23+
24+
delete_from_database(conn, name, crate_id)?;
25+
if let Some(s3) = s3_client() {
26+
delete_from_s3(&s3, name)?;
27+
}
28+
29+
Ok(())
30+
}
31+
32+
fn delete_from_database(conn: &Connection, name: &str, crate_id: i32) -> Result<(), Error> {
33+
let transaction = conn.transaction()?;
34+
35+
transaction.execute(
36+
"DELETE FROM sandbox_overrides WHERE crate_name = $1",
37+
&[&name],
38+
)?;
39+
transaction.execute(
40+
"DELETE FROM author_rels WHERE rid IN (SELECT id FROM releases WHERE crate_id = $1);",
41+
&[&crate_id],
42+
)?;
43+
transaction.execute(
44+
"DELETE FROM owner_rels WHERE cid IN (SELECT id FROM releases WHERE crate_id = $1);",
45+
&[&crate_id],
46+
)?;
47+
transaction.execute(
48+
"DELETE FROM keyword_rels WHERE rid IN (SELECT id FROM releases WHERE crate_id = $1);",
49+
&[&crate_id],
50+
)?;
51+
transaction.execute(
52+
"DELETE FROM builds WHERE rid IN (SELECT id FROM releases WHERE crate_id = $1);",
53+
&[&crate_id],
54+
)?;
55+
transaction.execute("DELETE FROM releases WHERE crate_id = $1;", &[&crate_id])?;
56+
transaction.execute("DELETE FROM crates WHERE id = $1;", &[&crate_id])?;
57+
58+
for prefix in STORAGE_PATHS_TO_DELETE {
59+
transaction.execute(
60+
"DELETE FROM files WHERE path LIKE $1;",
61+
&[&format!("{}/{}/%", prefix, name)],
62+
)?;
63+
}
64+
65+
// Transactions automatically rollback when not committing, so if any of the previous queries
66+
// fail the whole transaction will be aborted.
67+
transaction.commit()?;
68+
Ok(())
69+
}
70+
71+
fn delete_from_s3(s3: &S3Client, name: &str) -> Result<(), Error> {
72+
for prefix in STORAGE_PATHS_TO_DELETE {
73+
delete_prefix_from_s3(s3, &format!("{}/{}/", prefix, name))?;
74+
}
75+
Ok(())
76+
}
77+
78+
fn delete_prefix_from_s3(s3: &S3Client, name: &str) -> Result<(), Error> {
79+
let mut continuation_token = None;
80+
loop {
81+
let list = s3
82+
.list_objects_v2(ListObjectsV2Request {
83+
bucket: S3_BUCKET_NAME.into(),
84+
prefix: Some(name.into()),
85+
continuation_token,
86+
..ListObjectsV2Request::default()
87+
})
88+
.sync()?;
89+
90+
let to_delete = list
91+
.contents
92+
.unwrap_or_else(Vec::new)
93+
.into_iter()
94+
.filter_map(|o| o.key)
95+
.map(|key| ObjectIdentifier {
96+
key,
97+
version_id: None,
98+
})
99+
.collect::<Vec<_>>();
100+
s3.delete_objects(DeleteObjectsRequest {
101+
bucket: S3_BUCKET_NAME.into(),
102+
delete: rusoto_s3::Delete {
103+
objects: to_delete,
104+
quiet: None,
105+
},
106+
..DeleteObjectsRequest::default()
107+
})
108+
.sync()?;
109+
110+
continuation_token = list.continuation_token;
111+
if continuation_token.is_none() {
112+
return Ok(());
113+
}
114+
}
115+
}

‎src/db/file.rs

+7-5
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ use rusoto_credential::DefaultCredentialsProvider;
2020

2121
const MAX_CONCURRENT_UPLOADS: usize = 1000;
2222

23+
pub(super) static S3_BUCKET_NAME: &str = "rust-docs-rs";
24+
2325

2426
fn get_file_list_from_dir<P: AsRef<Path>>(path: P,
2527
files: &mut Vec<PathBuf>)
@@ -69,7 +71,7 @@ pub struct Blob {
6971
pub fn get_path(conn: &Connection, path: &str) -> Option<Blob> {
7072
if let Some(client) = s3_client() {
7173
let res = client.get_object(GetObjectRequest {
72-
bucket: "rust-docs-rs".into(),
74+
bucket: S3_BUCKET_NAME.into(),
7375
key: path.into(),
7476
..Default::default()
7577
}).sync();
@@ -116,7 +118,7 @@ pub fn get_path(conn: &Connection, path: &str) -> Option<Blob> {
116118
}
117119
}
118120

119-
fn s3_client() -> Option<S3Client> {
121+
pub(super) fn s3_client() -> Option<S3Client> {
120122
// If AWS keys aren't configured, then presume we should use the DB exclusively
121123
// for file storage.
122124
if std::env::var_os("AWS_ACCESS_KEY_ID").is_none() && std::env::var_os("FORCE_S3").is_none() {
@@ -134,7 +136,7 @@ fn s3_client() -> Option<S3Client> {
134136
creds,
135137
std::env::var("S3_ENDPOINT").ok().map(|e| Region::Custom {
136138
name: std::env::var("S3_REGION")
137-
.unwrap_or_else(|| "us-west-1".to_owned()),
139+
.unwrap_or_else(|_| "us-west-1".to_owned()),
138140
endpoint: e,
139141
}).unwrap_or(Region::UsWest1),
140142
))
@@ -203,7 +205,7 @@ pub fn add_path_into_database<P: AsRef<Path>>(conn: &Connection,
203205

204206
if let Some(client) = &client {
205207
futures.push(client.put_object(PutObjectRequest {
206-
bucket: "rust-docs-rs".into(),
208+
bucket: S3_BUCKET_NAME.into(),
207209
key: bucket_path.clone(),
208210
body: Some(content.clone().into()),
209211
content_type: Some(mime.clone()),
@@ -295,7 +297,7 @@ pub fn move_to_s3(conn: &Connection, n: usize) -> Result<usize> {
295297
let content: Vec<u8> = row.get(2);
296298
let path_1 = path.clone();
297299
futures.push(client.put_object(PutObjectRequest {
298-
bucket: "rust-docs-rs".into(),
300+
bucket: S3_BUCKET_NAME.into(),
299301
key: path.clone(),
300302
body: Some(content.into()),
301303
content_type: Some(mime),

‎src/db/mod.rs

+2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ pub(crate) use self::add_package::add_package_into_database;
44
pub(crate) use self::add_package::add_build_into_database;
55
pub use self::file::add_path_into_database;
66
pub use self::migrate::migrate;
7+
pub use self::delete_crate::delete_crate;
78

89
use postgres::{Connection, TlsMode};
910
use postgres::error::Error;
@@ -14,6 +15,7 @@ use r2d2_postgres;
1415
mod add_package;
1516
pub mod file;
1617
mod migrate;
18+
mod delete_crate;
1719

1820

1921
/// Connects to database

0 commit comments

Comments
 (0)