Skip to content

use github fast path to check for changes before doing the git pull #47

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,6 @@ edition = "2018"
readme = "changelog.md"
include = ["src/**/*", "LICENSE.md", "README.md", "CHANGELOG.md"]

[lib]
test = false

[[test]]
name = "baseline"
path = "tests/baseline.rs"
Expand Down Expand Up @@ -46,6 +43,7 @@ bstr = "1.0.1"
thiserror = "1.0.32"
ahash = "0.8.0"
hashbrown = { version = "0.14.0", features = ["raw"] }
reqwest = { version = "0.12", features = ["blocking"] }

[dev-dependencies]
gix-testtools = "0.15.0"
Expand Down
113 changes: 113 additions & 0 deletions src/index/diff/github.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
use reqwest::{StatusCode, Url};

#[derive(Debug)]
pub(crate) enum FastPath {
UpToDate,
NeedsFetch,
Indeterminate,
}

/// extract username & repository from a fetch URL, only if it's on Github.
fn user_and_repo_from_url_if_github(fetch_url: &gix::Url) -> Option<(String, String)> {
let url = Url::parse(&fetch_url.to_string()).ok()?;
if !(url.host_str() == Some("github.com")) {
return None;
}

// This expects GitHub urls in the form `github.com/user/repo` and nothing
// else
let mut pieces = url.path_segments()?;
let username = pieces.next()?;
let repository = pieces.next()?;
let repository = repository.strip_suffix(".git").unwrap_or(repository);
if pieces.next().is_some() {
return None;
}
Some((username.to_string(), repository.to_string()))
}

/// use github fast-path to check if the repository has any changes
/// since the last seen reference.
///
/// To save server side resources on github side, we can use an API
/// to check if there are any changes in the repository before we
/// actually run `git fetch`.
///
/// On non-github fetch URLs we don't do anything and always run the fetch.
///
/// Code gotten and adapted from
/// https://github.com/rust-lang/cargo/blob/edd36eba5e0d6e0cfcb84bd0cc651ba8bf5e7f83/src/cargo/sources/git/utils.rs#L1396
///
/// GitHub documentation:
/// https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#get-a-commit
/// specifically using `application/vnd.github.sha`
pub(crate) fn has_changes(
fetch_url: &gix::Url,
last_seen_reference: &gix::ObjectId,
branch_name: &str,
) -> Result<FastPath, reqwest::Error> {
let (username, repository) = match user_and_repo_from_url_if_github(fetch_url) {
Some(url) => url,
None => return Ok(FastPath::Indeterminate),
};

let url = format!(
"https://api.github.com/repos/{}/{}/commits/{}",
username, repository, branch_name,
);

let client = reqwest::blocking::Client::builder()
.user_agent("crates-index-diff")
.build()?;
let response = client
.get(&url)
.header("Accept", "application/vnd.github.sha")
.header("If-None-Match", format!("\"{}\"", last_seen_reference))
.send()?;

let status = response.status();
if status == StatusCode::NOT_MODIFIED {
Ok(FastPath::UpToDate)
} else if status.is_success() {
Ok(FastPath::NeedsFetch)
} else {
// Usually response_code == 404 if the repository does not exist, and
// response_code == 422 if exists but GitHub is unable to resolve the
// requested rev.
Ok(FastPath::Indeterminate)
}
}

#[cfg(test)]
mod tests {
use super::*;
use std::convert::TryFrom;

#[test]
fn test_github_http_url() {
let (user, repo) = user_and_repo_from_url_if_github(
&gix::Url::try_from("https://github.com/some_user/some_repo.git").unwrap(),
)
.unwrap();
assert_eq!(user, "some_user");
assert_eq!(repo, "some_repo");
}

#[test]
fn test_github_ssh_url() {
let (user, repo) = user_and_repo_from_url_if_github(
&gix::Url::try_from("ssh://[email protected]/some_user/some_repo.git").unwrap(),
)
.unwrap();
assert_eq!(user, "some_user");
assert_eq!(repo, "some_repo");
}

#[test]
fn test_non_github_url() {
assert!(user_and_repo_from_url_if_github(
&gix::Url::try_from("https://not_github.com/some_user/some_repo.git").unwrap(),
)
.is_none());
}
}
60 changes: 36 additions & 24 deletions src/index/diff/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ use gix::prelude::ObjectIdExt;
use std::sync::atomic::AtomicBool;

mod delegate;
mod github;

use delegate::Delegate;

Expand Down Expand Up @@ -66,6 +67,8 @@ pub enum Error {
name: String,
mappings: Vec<gix::remote::fetch::Mapping>,
},
#[error("Error when fetching GitHub fastpath.")]
GithubFetch(#[from] reqwest::Error),
}

/// Find changes without modifying the underling repository
Expand Down Expand Up @@ -175,30 +178,39 @@ impl Index {
.replace_refspecs(Some(spec.as_str()), gix::remote::Direction::Fetch)
.expect("valid statically known refspec");
}
let res: gix::remote::fetch::Outcome = remote
.connect(gix::remote::Direction::Fetch)?
.prepare_fetch(&mut progress, Default::default())?
.receive(&mut progress, should_interrupt)?;
let branch_name = format!("refs/heads/{}", self.branch_name);
let local_tracking = res
.ref_map
.mappings
.iter()
.find_map(|m| match &m.remote {
gix::remote::fetch::Source::Ref(r) => (r.unpack().0 == branch_name)
.then_some(m.local.as_ref())
.flatten(),
_ => None,
})
.ok_or_else(|| Error::NoMatchingBranch {
name: branch_name,
mappings: res.ref_map.mappings.clone(),
})?;
self.repo
.find_reference(local_tracking)
.expect("local tracking branch exists if we see it here")
.id()
.detach()

let (url, _) = remote.sanitized_url_and_version(gix::remote::Direction::Fetch)?;
if matches!(
github::has_changes(&url, &from, self.branch_name)?,
github::FastPath::UpToDate
) {
from.clone()
} else {
let res: gix::remote::fetch::Outcome = remote
.connect(gix::remote::Direction::Fetch)?
.prepare_fetch(&mut progress, Default::default())?
.receive(&mut progress, should_interrupt)?;
let branch_name = format!("refs/heads/{}", self.branch_name);
let local_tracking = res
.ref_map
.mappings
.iter()
.find_map(|m| match &m.remote {
gix::remote::fetch::Source::Ref(r) => (r.unpack().0 == branch_name)
.then_some(m.local.as_ref())
.flatten(),
_ => None,
})
.ok_or_else(|| Error::NoMatchingBranch {
name: branch_name,
mappings: res.ref_map.mappings.clone(),
})?;
self.repo
.find_reference(local_tracking)
.expect("local tracking branch exists if we see it here")
.id()
.detach()
}
};

Ok((
Expand Down
Loading