diff --git a/Cargo.lock b/Cargo.lock index 7c02092d225..d1d5583bf9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -393,7 +393,7 @@ dependencies = [ [[package]] name = "cargo-smart-release" -version = "0.19.0" +version = "0.20.0" dependencies = [ "anyhow", "bitflags 2.1.0", @@ -1230,7 +1230,7 @@ dependencies = [ [[package]] name = "gitoxide" -version = "0.25.0" +version = "0.26.0" dependencies = [ "anyhow", "clap 4.2.4", @@ -1250,7 +1250,7 @@ dependencies = [ [[package]] name = "gitoxide-core" -version = "0.27.0" +version = "0.28.0" dependencies = [ "anyhow", "async-io", @@ -1279,7 +1279,7 @@ dependencies = [ [[package]] name = "gix" -version = "0.44.1" +version = "0.45.0" dependencies = [ "anyhow", "async-std", @@ -1445,7 +1445,7 @@ dependencies = [ [[package]] name = "gix-commitgraph" -version = "0.14.0" +version = "0.15.0" dependencies = [ "bstr", "document-features", @@ -1860,6 +1860,22 @@ dependencies = [ "thiserror", ] +[[package]] +name = "gix-negotiate" +version = "0.1.0" +dependencies = [ + "bitflags 2.1.0", + "gix-commitgraph", + "gix-hash 0.11.1", + "gix-object 0.29.2", + "gix-odb", + "gix-ref 0.29.1", + "gix-revision", + "gix-testtools", + "smallvec", + "thiserror", +] + [[package]] name = "gix-note" version = "0.0.0" @@ -2148,7 +2164,7 @@ dependencies = [ [[package]] name = "gix-refspec" -version = "0.10.1" +version = "0.11.0" dependencies = [ "bstr", "gix-hash 0.11.1", @@ -2161,7 +2177,7 @@ dependencies = [ [[package]] name = "gix-revision" -version = "0.13.0" +version = "0.14.0" dependencies = [ "bstr", "document-features", diff --git a/Cargo.toml b/Cargo.toml index 1e9d503f878..4eb6b807879 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ repository = "https://github.com/Byron/gitoxide" authors = ["Sebastian Thiel "] edition = "2021" license = "MIT OR Apache-2.0" -version = "0.25.0" +version = "0.26.0" default-run = "gix" include = ["src/**/*", "LICENSE-*", "README.md", "CHANGELOG.md"] resolver = "2" @@ -151,9 +151,9 @@ gitoxide-core-async-client = ["gitoxide-core/async-client", "futures-lite"] [dependencies] anyhow = "1.0.42" -gitoxide-core = { version = "^0.27.0", path = "gitoxide-core" } +gitoxide-core = { version = "^0.28.0", path = "gitoxide-core" } gix-features = { version = "^0.29.0", path = "gix-features" } -gix = { version = "^0.44.1", path = "gix", default-features = false } +gix = { version = "^0.45.0", path = "gix", default-features = false } time = "0.3.19" clap = { version = "4.1.1", features = ["derive", "cargo"] } @@ -221,6 +221,7 @@ members = [ "gix-packetline", "gix-mailmap", "gix-note", + "gix-negotiate", "gix-fetchhead", "gix-prompt", "gix-filter", diff --git a/README.md b/README.md index f81b045132b..d44f6e0fade 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,7 @@ is usable to some extent. * [gix-hashtable](https://github.com/Byron/gitoxide/blob/main/crate-status.md#gix-hashtable) * [gix-worktree](https://github.com/Byron/gitoxide/blob/main/crate-status.md#gix-worktree) * [gix-bitmap](https://github.com/Byron/gitoxide/blob/main/crate-status.md#gix-bitmap) + * [gix-negotiate](https://github.com/Byron/gitoxide/blob/main/crate-status.md#gix-negotiate) * `gitoxide-core` * **very early** _(possibly without any documentation and many rough edges)_ * [gix-date](https://github.com/Byron/gitoxide/blob/main/crate-status.md#gix-date) diff --git a/cargo-smart-release/Cargo.toml b/cargo-smart-release/Cargo.toml index d5e69460a15..f5954abbe34 100644 --- a/cargo-smart-release/Cargo.toml +++ b/cargo-smart-release/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cargo-smart-release" -version = "0.19.0" +version = "0.20.0" authors = ["Sebastian Thiel "] repository = "https://github.com/Byron/gitoxide" description = "Cargo subcommand for fearlessly releasing crates in workspaces." @@ -25,7 +25,7 @@ cache-efficiency-debug = ["gix/cache-efficiency-debug"] vendored-openssl = ["crates-index/vendored-openssl"] [dependencies] -gix = { version = "^0.44.1", path = "../gix", default-features = false, features = ["max-performance-safe"] } +gix = { version = "^0.45.0", path = "../gix", default-features = false, features = ["max-performance-safe"] } anyhow = "1.0.42" clap = { version = "4.1.0", features = ["derive", "cargo"] } env_logger = { version = "0.10.0", default-features = false, features = ["humantime", "auto-color"] } diff --git a/crate-status.md b/crate-status.md index 09e34418bf9..b0975224ef5 100644 --- a/crate-status.md +++ b/crate-status.md @@ -386,6 +386,12 @@ A mechanism to associate metadata with any object, and keep revisions of it usin * [ ] CRUD for git notes +### gix-negotiate +* **algorithms** + - [ ] `noop` + - [ ] `consecutive` + - [ ] `skipping` + ### gix-fetchhead * [ ] parse `FETCH_HEAD` information back entirely * [ ] write typical fetch-head lines diff --git a/gitoxide-core/Cargo.toml b/gitoxide-core/Cargo.toml index f642a829173..092eb7e0096 100644 --- a/gitoxide-core/Cargo.toml +++ b/gitoxide-core/Cargo.toml @@ -2,7 +2,7 @@ name = "gitoxide-core" description = "The library implementing all capabilities of the gitoxide CLI" repository = "https://github.com/Byron/gitoxide" -version = "0.27.0" +version = "0.28.0" authors = ["Sebastian Thiel "] license = "MIT/Apache-2.0" edition = "2021" @@ -38,7 +38,7 @@ serde = ["gix/serde", "serde_json", "dep:serde", "bytesize/serde"] [dependencies] # deselect everything else (like "performance") as this should be controllable by the parent application. -gix = { version = "^0.44.1", path = "../gix", default-features = false } +gix = { version = "^0.45.0", path = "../gix", default-features = false } gix-pack-for-configuration-only = { package = "gix-pack", version = "^0.35.0", path = "../gix-pack", default-features = false, features = ["pack-cache-lru-dynamic", "pack-cache-lru-static"] } gix-transport-configuration-only = { package = "gix-transport", version = "^0.31.0", path = "../gix-transport", default-features = false } serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"] } diff --git a/gix-commitgraph/CHANGELOG.md b/gix-commitgraph/CHANGELOG.md index cb2928b9114..336649dc477 100644 --- a/gix-commitgraph/CHANGELOG.md +++ b/gix-commitgraph/CHANGELOG.md @@ -5,6 +5,40 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.15.0 (2023-05-19) + +### New Features (BREAKING) + + - `describe` usees commitgraph. + With it it can leverage the commitgraph data structure would would be more prominent + on server-side applications, presumably. + +### Refactor (BREAKING) + + - make API more consistent with other `gix-*` crates. + For that, we remove duplicate import paths for types. + We also improve lifetimes around parent iteration, and make the type explicit. + +### Commit Statistics + + + + - 2 commits contributed to the release over the course of 1 calendar day. + - 21 days passed between releases. + - 2 commits were understood as [conventional](https://www.conventionalcommits.org). + - 0 issues like '(#ID)' were seen in commit messages + +### Commit Details + + + +
view details + + * **Uncategorized** + - `describe` usees commitgraph. ([`ed258da`](https://github.com/Byron/gitoxide/commit/ed258da9015d2d68734aeac485dd009760fc4da4)) + - Make API more consistent with other `gix-*` crates. ([`967f3b9`](https://github.com/Byron/gitoxide/commit/967f3b954e9fb4fc7757f8920998173caf0491ab)) +
+ ## 0.14.0 (2023-04-27) ### New Features (BREAKING) @@ -21,8 +55,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - - 9 commits contributed to the release over the course of 56 calendar days. - - 60 days passed between releases. + - 10 commits contributed to the release over the course of 57 calendar days. + - 61 days passed between releases. - 1 commit was understood as [conventional](https://www.conventionalcommits.org). - 1 unique issue was worked on: [#814](https://github.com/Byron/gitoxide/issues/814) @@ -35,6 +69,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * **[#814](https://github.com/Byron/gitoxide/issues/814)** - Rename `serde1` cargo feature to `serde` and use the weak-deps cargo capability. ([`b83ee36`](https://github.com/Byron/gitoxide/commit/b83ee366a3c65c717beb587ad809268f1c54b8ad)) * **Uncategorized** + - Release gix-commitgraph v0.14.0, gitoxide-core v0.26.0, gitoxide v0.24.0 ([`9f2317f`](https://github.com/Byron/gitoxide/commit/9f2317f2514872001168d2be6e33e2ee2872420e)) - Release gix-hash v0.11.1, gix-path v0.7.4, gix-glob v0.6.0, gix-attributes v0.11.0, gix-config-value v0.11.0, gix-fs v0.1.1, gix-tempfile v5.0.3, gix-utils v0.1.1, gix-lock v5.0.1, gix-object v0.29.1, gix-ref v0.28.0, gix-sec v0.7.0, gix-config v0.21.0, gix-prompt v0.4.0, gix-url v0.17.0, gix-credentials v0.13.0, gix-diff v0.29.0, gix-discover v0.17.0, gix-hashtable v0.2.0, gix-ignore v0.1.0, gix-bitmap v0.2.3, gix-traverse v0.25.0, gix-index v0.16.0, gix-mailmap v0.12.0, gix-pack v0.34.0, gix-odb v0.44.0, gix-packetline v0.16.0, gix-transport v0.30.0, gix-protocol v0.31.0, gix-revision v0.13.0, gix-refspec v0.10.0, gix-worktree v0.16.0, gix v0.44.0, safety bump 7 crates ([`91134a1`](https://github.com/Byron/gitoxide/commit/91134a11c8ba0e942f692488ec9bce9fa1086324)) - Release gix-utils v0.1.0, gix-hash v0.11.0, gix-date v0.5.0, gix-features v0.29.0, gix-actor v0.20.0, gix-object v0.29.0, gix-archive v0.1.0, gix-fs v0.1.0, safety bump 25 crates ([`8dbd0a6`](https://github.com/Byron/gitoxide/commit/8dbd0a60557a85acfa231800a058cbac0271a8cf)) - Merge branch 'main' into dev ([`cdef398`](https://github.com/Byron/gitoxide/commit/cdef398c4a3bd01baf0be2c27a3f77a400172b0d)) diff --git a/gix-commitgraph/Cargo.toml b/gix-commitgraph/Cargo.toml index 3d589b0258b..232dd20c9f7 100644 --- a/gix-commitgraph/Cargo.toml +++ b/gix-commitgraph/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gix-commitgraph" -version = "0.14.0" +version = "0.15.0" repository = "https://github.com/Byron/gitoxide" documentation = "https://git-scm.com/docs/commit-graph#:~:text=The%20commit-graph%20file%20is%20a%20supplemental%20data%20structure,or%20in%20the%20info%20directory%20of%20an%20alternate." license = "MIT/Apache-2.0" diff --git a/gix-commitgraph/src/lib.rs b/gix-commitgraph/src/lib.rs index 2d0cfbb6281..231c11c6fb0 100644 --- a/gix-commitgraph/src/lib.rs +++ b/gix-commitgraph/src/lib.rs @@ -15,6 +15,8 @@ #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] #![deny(missing_docs, rust_2018_idioms, unsafe_code)] +use std::path::Path; + /// A single commit-graph file. /// /// All operations on a `File` are local to that graph file. Since a commit graph can span multiple @@ -41,6 +43,11 @@ pub struct Graph { files: Vec, } +/// Instantiate a commit graph from an `.git/objects/info` directory, or one of the various commit-graph files. +pub fn at(path: impl AsRef) -> Result { + Graph::at(path) +} + mod access; pub mod file; /// diff --git a/gix-commitgraph/tests/access/mod.rs b/gix-commitgraph/tests/access/mod.rs index c5b11128f3d..9dc41c5d348 100644 --- a/gix-commitgraph/tests/access/mod.rs +++ b/gix-commitgraph/tests/access/mod.rs @@ -30,7 +30,7 @@ fn octupus_merges() -> crate::Result { "four_parents", ], ); - let cg = Graph::from_info_dir(repo_dir.join(".git").join("objects").join("info"))?; + let cg = Graph::at(repo_dir.join(".git").join("objects").join("info"))?; check_common(&cg, &refs); assert_eq!(cg.commit_at(refs["root"].pos()).generation(), 1); @@ -48,7 +48,7 @@ fn octupus_merges() -> crate::Result { fn single_commit() -> crate::Result { let repo_dir = make_readonly_repo("single_commit.sh"); let refs = inspect_refs(&repo_dir, &["commit"]); - let cg = Graph::from_info_dir(repo_dir.join(".git").join("objects").join("info"))?; + let cg = gix_commitgraph::at(repo_dir.join(".git").join("objects").join("info"))?; check_common(&cg, &refs); assert_eq!(cg.commit_at(refs["commit"].pos()).generation(), 1); diff --git a/gix-negotiate/CHANGELOG.md b/gix-negotiate/CHANGELOG.md new file mode 100644 index 00000000000..bcb55f20acf --- /dev/null +++ b/gix-negotiate/CHANGELOG.md @@ -0,0 +1,29 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## v0.1.0 (2023-05-19) + +Initial release with a single function to calculate the window size for `HAVE` lines. + +### Commit Statistics + + + + - 1 commit contributed to the release. + - 0 commits were understood as [conventional](https://www.conventionalcommits.org). + - 0 issues like '(#ID)' were seen in commit messages + +### Commit Details + + + +
view details + + * **Uncategorized** + - Add new crate for implementing and testing git negotiation logic. ([`372ba09`](https://github.com/Byron/gitoxide/commit/372ba09bb00e3fab674f0251f697aab11c5559f8)) +
+ diff --git a/gix-negotiate/Cargo.toml b/gix-negotiate/Cargo.toml new file mode 100644 index 00000000000..fbcb266d3ab --- /dev/null +++ b/gix-negotiate/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "gix-negotiate" +version = "0.1.0" +repository = "https://github.com/Byron/gitoxide" +license = "MIT/Apache-2.0" +description = "A crate of the gitoxide project implementing negotiation algorithms" +authors = ["Sebastian Thiel "] +edition = "2021" +rust-version = "1.64" + +[lib] +doctest = false +test = false + +[dependencies] +gix-hash = { version = "^0.11.1", path = "../gix-hash" } +gix-object = { version = "^0.29.2", path = "../gix-object" } +gix-commitgraph = { version = "^0.15.0", path = "../gix-commitgraph" } +gix-revision = { version = "^0.14.0", path = "../gix-revision" } +thiserror = "1.0.40" +smallvec = "1.10.0" +bitflags = "2" + +[dev-dependencies] +gix-testtools = { path = "../tests/tools" } +gix-odb = { path = "../gix-odb" } +gix-ref = { path = "../gix-ref" } diff --git a/gix-negotiate/src/consecutive.rs b/gix-negotiate/src/consecutive.rs new file mode 100644 index 00000000000..a813bc207ea --- /dev/null +++ b/gix-negotiate/src/consecutive.rs @@ -0,0 +1,190 @@ +use crate::{Error, Negotiator}; +use gix_hash::ObjectId; +use gix_revision::graph::CommitterTimestamp; +use smallvec::SmallVec; +bitflags::bitflags! { + /// Whether something can be read or written. + #[derive(Debug, Default, Copy, Clone)] + pub struct Flags: u8 { + /// The revision is known to be in common with the remote. + const COMMON = 1 << 0; + /// The revision is common and was set by merit of a remote tracking ref (e.g. `refs/heads/origin/main`). + const COMMON_REF = 1 << 1; + /// The revision has entered the priority queue. + const SEEN = 1 << 2; + /// The revision was popped off our primary priority queue, used to avoid double-counting of `non_common_revs` + const POPPED = 1 << 3; + } +} + +pub(crate) struct Algorithm<'find> { + graph: gix_revision::Graph<'find, Flags>, + revs: gix_revision::PriorityQueue, + non_common_revs: usize, +} + +impl<'a> Algorithm<'a> { + pub fn new(graph: gix_revision::Graph<'a, Flags>) -> Self { + Self { + graph, + revs: gix_revision::PriorityQueue::new(), + non_common_revs: 0, + } + } + + /// Add `id` to our priority queue and *add* `flags` to it. + fn add_to_queue(&mut self, id: ObjectId, mark: Flags) -> Result<(), Error> { + let mut is_common = false; + if self.graph.get(&id).map_or(false, |flags| flags.intersects(mark)) { + return Ok(()); + } + let commit = self.graph.try_lookup_and_insert(id, |current| { + *current |= mark; + is_common = current.contains(Flags::COMMON); + })?; + if let Some(timestamp) = commit.map(|c| c.committer_timestamp()).transpose()? { + self.revs.insert(timestamp, id); + if !is_common { + self.non_common_revs += 1; + } + } + Ok(()) + } + + fn mark_common(&mut self, id: ObjectId, mode: Mark, ancestors: Ancestors) -> Result<(), Error> { + let mut is_common = false; + if let Some(commit) = self + .graph + .try_lookup_and_insert(id, |current| is_common = current.contains(Flags::COMMON))? + .filter(|_| !is_common) + { + let mut queue = + gix_revision::PriorityQueue::from_iter(Some((commit.committer_timestamp()?, (id, 0_usize)))); + if let Mark::ThisCommitAndAncestors = mode { + let current = self.graph.get_mut(&id).expect("just inserted"); + *current |= Flags::COMMON; + if current.contains(Flags::SEEN) && !current.contains(Flags::POPPED) { + self.non_common_revs -= 1; + } + } + let mut parents = SmallVec::new(); + while let Some((id, generation)) = queue.pop() { + if self.graph.get(&id).map_or(true, |d| !d.contains(Flags::SEEN)) { + self.add_to_queue(id, Flags::SEEN)?; + } else if matches!(ancestors, Ancestors::AllUnseen) || generation < 2 { + if let Some(commit) = self.graph.try_lookup_and_insert(id, |_| {})? { + collect_parents(commit.iter_parents(), &mut parents)?; + for parent_id in parents.drain(..) { + let mut prev_flags = Flags::default(); + if let Some(parent) = self + .graph + .try_lookup_and_insert(parent_id, |d| { + prev_flags = *d; + *d |= Flags::COMMON; + })? + .filter(|_| !prev_flags.contains(Flags::COMMON)) + { + if prev_flags.contains(Flags::SEEN) && !prev_flags.contains(Flags::POPPED) { + self.non_common_revs -= 1; + } + queue.insert(parent.committer_timestamp()?, (parent_id, generation + 1)) + } + } + } + } + } + } + Ok(()) + } +} + +pub(crate) fn collect_parents( + parents: gix_revision::graph::commit::Parents<'_>, + out: &mut SmallVec<[ObjectId; 2]>, +) -> Result<(), Error> { + out.clear(); + for parent in parents { + out.push(parent.map_err(|err| match err { + gix_revision::graph::commit::iter_parents::Error::DecodeCommit(err) => Error::DecodeCommit(err), + gix_revision::graph::commit::iter_parents::Error::DecodeCommitGraph(err) => Error::DecodeCommitInGraph(err), + })?); + } + Ok(()) +} + +impl<'a> Negotiator for Algorithm<'a> { + fn known_common(&mut self, id: ObjectId) -> Result<(), Error> { + if self.graph.get(&id).map_or(true, |d| !d.contains(Flags::SEEN)) { + self.add_to_queue(id, Flags::COMMON_REF | Flags::SEEN)?; + self.mark_common(id, Mark::AncestorsOnly, Ancestors::DirectUnseen)?; + } + Ok(()) + } + + fn add_tip(&mut self, id: ObjectId) -> Result<(), Error> { + self.add_to_queue(id, Flags::SEEN) + } + + fn next_have(&mut self) -> Option> { + let mut parents = SmallVec::new(); + loop { + let id = self.revs.pop().filter(|_| self.non_common_revs != 0)?; + let flags = self.graph.get_mut(&id).expect("it was added to the graph by now"); + *flags |= Flags::POPPED; + + if !flags.contains(Flags::COMMON) { + self.non_common_revs -= 1; + } + + let (res, mark) = if flags.contains(Flags::COMMON) { + (None, Flags::COMMON | Flags::SEEN) + } else if flags.contains(Flags::COMMON_REF) { + (Some(id), Flags::COMMON | Flags::SEEN) + } else { + (Some(id), Flags::SEEN) + }; + + let commit = match self.graph.try_lookup(&id) { + Ok(c) => c.expect("it was found before, must still be there"), + Err(err) => return Some(Err(err.into())), + }; + if let Err(err) = collect_parents(commit.iter_parents(), &mut parents) { + return Some(Err(err)); + } + for parent_id in parents.drain(..) { + if self.graph.get(&parent_id).map_or(true, |d| !d.contains(Flags::SEEN)) { + if let Err(err) = self.add_to_queue(parent_id, mark) { + return Some(Err(err)); + } + } + if mark.contains(Flags::COMMON) { + if let Err(err) = self.mark_common(parent_id, Mark::AncestorsOnly, Ancestors::AllUnseen) { + return Some(Err(err)); + } + } + } + + if let Some(id) = res { + return Some(Ok(id)); + } + } + } + + fn in_common_with_remote(&mut self, id: ObjectId) -> Result { + let known_to_be_common = self.graph.get(&id).map_or(false, |d| d.contains(Flags::COMMON)); + self.mark_common(id, Mark::ThisCommitAndAncestors, Ancestors::DirectUnseen)?; + Ok(known_to_be_common) + } +} + +enum Mark { + AncestorsOnly, + ThisCommitAndAncestors, +} + +enum Ancestors { + /// Traverse only the parents of a commit. + DirectUnseen, + /// Traverse all ancestors that weren't yet seen. + AllUnseen, +} diff --git a/gix-negotiate/src/lib.rs b/gix-negotiate/src/lib.rs new file mode 100644 index 00000000000..c207b6861ab --- /dev/null +++ b/gix-negotiate/src/lib.rs @@ -0,0 +1,104 @@ +//! An implementation of negotiation algorithms to help the server figure out what we have in common so it can optimize +//! the pack it sends to only contain what we don't have. +#![deny(rust_2018_idioms, missing_docs)] +#![forbid(unsafe_code)] + +mod consecutive; +mod noop; +mod skipping; + +/// The way the negotiation is performed. +#[derive(Default, Debug, Copy, Clone, Eq, PartialEq)] +pub enum Algorithm { + /// Do not send any information at all, which typically leads to complete packs to be sent. + Noop, + /// Walk over consecutive commits and check each one. This can be costly be assures packs are exactly the size they need to be. + #[default] + Consecutive, + /// Like `Consecutive`, but skips commits to converge faster, at the cost of receiving packs that are larger than they have to be. + Skipping, +} + +/// Calculate how many `HAVE` lines we may send in one round, with variation depending on whether the `transport_is_stateless` or not. +/// `window_size` is the previous (or initial) value of the window size. +pub fn window_size(transport_is_stateless: bool, window_size: impl Into>) -> usize { + let current_size = match window_size.into() { + None => return 16, + Some(cs) => cs, + }; + const PIPESAFE_FLUSH: usize = 32; + const LARGE_FLUSH: usize = 16384; + + if transport_is_stateless { + if current_size < LARGE_FLUSH { + current_size * 2 + } else { + current_size * 11 / 10 + } + } else if current_size < PIPESAFE_FLUSH { + current_size * 2 + } else { + current_size + PIPESAFE_FLUSH + } +} + +impl Algorithm { + /// Create an instance of a negotiator which implements this algorithm. + pub fn into_negotiator<'find, Find, E>( + self, + find: Find, + cache: impl Into>, + ) -> Box + where + Find: + for<'a> FnMut(&gix_hash::oid, &'a mut Vec) -> Result>, E> + 'find, + E: std::error::Error + Send + Sync + 'static, + { + match &self { + Algorithm::Noop => Box::new(noop::Noop) as Box, + Algorithm::Consecutive => { + let graph = gix_revision::Graph::<'_, consecutive::Flags>::new(find, cache); + Box::new(consecutive::Algorithm::new(graph)) + } + Algorithm::Skipping => { + let graph = gix_revision::Graph::<'_, skipping::Entry>::new(find, cache); + Box::new(skipping::Algorithm::new(graph)) + } + } + } +} + +/// A delegate to implement a negotiation algorithm. +pub trait Negotiator { + /// Mark `id` as common between the remote and us. + /// + /// These ids are typically the local tips of remote tracking branches. + fn known_common(&mut self, id: gix_hash::ObjectId) -> Result<(), Error>; + + /// Add `id` as starting point of a traversal across commits that aren't necessarily common between the remote and us. + /// + /// These tips are usually the commits of local references whose tips should lead to objects that we have in common with the remote. + fn add_tip(&mut self, id: gix_hash::ObjectId) -> Result<(), Error>; + + /// Produce the next id of an object that we want the server to know we have. It's an object we don't know we have in common or not. + /// + /// Returns `None` if we have exhausted all options, which might mean we have traversed the entire commit graph. + fn next_have(&mut self) -> Option>; + + /// Mark `id` as being common with the remote (as informed by the remote itself) and return `true` if we knew it was common already. + /// + /// We can assume to have already seen `id` as we were the one to inform the remote in a prior `have`. + fn in_common_with_remote(&mut self, id: gix_hash::ObjectId) -> Result; +} + +/// An error that happened during any of the methods on a [`Negotiator`]. +#[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] +pub enum Error { + #[error(transparent)] + DecodeCommit(#[from] gix_object::decode::Error), + #[error(transparent)] + DecodeCommitInGraph(#[from] gix_commitgraph::file::commit::Error), + #[error(transparent)] + LookupCommitInGraph(#[from] gix_revision::graph::lookup::Error), +} diff --git a/gix-negotiate/src/noop.rs b/gix-negotiate/src/noop.rs new file mode 100644 index 00000000000..92ad1018604 --- /dev/null +++ b/gix-negotiate/src/noop.rs @@ -0,0 +1,22 @@ +use crate::{Error, Negotiator}; +use gix_hash::ObjectId; + +pub(crate) struct Noop; + +impl Negotiator for Noop { + fn known_common(&mut self, _id: ObjectId) -> Result<(), Error> { + Ok(()) + } + + fn add_tip(&mut self, _id: ObjectId) -> Result<(), Error> { + Ok(()) + } + + fn next_have(&mut self) -> Option> { + None + } + + fn in_common_with_remote(&mut self, _id: ObjectId) -> Result { + Ok(false) + } +} diff --git a/gix-negotiate/src/skipping.rs b/gix-negotiate/src/skipping.rs new file mode 100644 index 00000000000..719e3cf9a5a --- /dev/null +++ b/gix-negotiate/src/skipping.rs @@ -0,0 +1,221 @@ +use crate::consecutive::collect_parents; +use crate::{Error, Negotiator}; +use gix_hash::ObjectId; +use gix_revision::graph::CommitterTimestamp; +use smallvec::SmallVec; +bitflags::bitflags! { + /// Whether something can be read or written. + #[derive(Debug, Default, Copy, Clone)] + pub struct Flags: u8 { + /// The revision is known to be in common with the remote. + const COMMON = 1 << 0; + /// The remote let us know it has the object. We still have to tell the server we have this object or one of its descendants. + /// We won't tell the server about its ancestors. + const ADVERTISED = 1 << 1; + /// The revision has at one point entered the priority queue (even though it might not be on it anymore). + const SEEN = 1 << 2; + /// The revision was popped off our priority queue. + const POPPED = 1 << 3; + } +} + +#[derive(Default, Copy, Clone)] +pub(crate) struct Entry { + flags: Flags, + /// Only used if commit is not COMMON + original_ttl: u16, + ttl: u16, +} + +pub(crate) struct Algorithm<'find> { + graph: gix_revision::Graph<'find, Entry>, + revs: gix_revision::PriorityQueue, + non_common_revs: usize, +} + +impl<'a> Algorithm<'a> { + pub fn new(graph: gix_revision::Graph<'a, Entry>) -> Self { + Self { + graph, + revs: gix_revision::PriorityQueue::new(), + non_common_revs: 0, + } + } + + /// Add `id` to our priority queue and *add* `flags` to it. + fn add_to_queue(&mut self, id: ObjectId, mark: Flags) -> Result<(), Error> { + let commit = self.graph.try_lookup_and_insert(id, |entry| { + entry.flags |= mark | Flags::SEEN; + })?; + if let Some(timestamp) = commit.map(|c| c.committer_timestamp()).transpose()? { + self.revs.insert(timestamp, id); + if !mark.contains(Flags::COMMON) { + self.non_common_revs += 1; + } + } + Ok(()) + } + + fn mark_common(&mut self, id: ObjectId) -> Result<(), Error> { + let mut is_common = false; + if let Some(commit) = self + .graph + .try_lookup_and_insert(id, |entry| { + is_common = entry.flags.contains(Flags::COMMON); + entry.flags |= Flags::COMMON; + })? + .filter(|_| !is_common) + { + let mut queue = gix_revision::PriorityQueue::from_iter(Some((commit.committer_timestamp()?, id))); + let mut parents = SmallVec::new(); + while let Some(id) = queue.pop() { + // This is a bit of a problem as there is no representation of the `parsed` based skip, which probably + // prevents this traversal from going on for too long. There is no equivalent here, but when artificially + // limiting the traversal depth the tests fail as they actually require the traversal to happen. + if self + .graph + .get(&id) + .map_or(false, |entry| entry.flags.contains(Flags::POPPED)) + { + self.non_common_revs = self.non_common_revs.saturating_sub(1); + } + if let Some(commit) = self.graph.try_lookup_and_insert(id, |entry| { + if !entry.flags.contains(Flags::POPPED) { + self.non_common_revs -= 1; + } + })? { + collect_parents(commit.iter_parents(), &mut parents)?; + for parent_id in parents.drain(..) { + let mut was_unseen_or_common = false; + if let Some(parent) = self + .graph + .try_lookup_and_insert(parent_id, |entry| { + was_unseen_or_common = + !entry.flags.contains(Flags::SEEN) || entry.flags.contains(Flags::COMMON); + entry.flags |= Flags::COMMON + })? + .filter(|_| !was_unseen_or_common) + { + queue.insert(parent.committer_timestamp()?, parent_id); + } + } + } + } + } + Ok(()) + } + + fn push_parent(&mut self, entry: Entry, parent_id: ObjectId) -> Result { + let mut was_seen = false; + if let Some(parent_entry) = self + .graph + .get(&parent_id) + .map(|entry| { + was_seen = entry.flags.contains(Flags::SEEN); + entry + }) + .filter(|_| was_seen) + { + if parent_entry.flags.contains(Flags::POPPED) { + return Ok(false); + } + } else { + self.add_to_queue(parent_id, Flags::default())?; + } + if entry.flags.intersects(Flags::COMMON | Flags::ADVERTISED) { + self.mark_common(parent_id)?; + } else { + let new_original_ttl = if entry.ttl > 0 { + entry.original_ttl + } else { + entry.original_ttl * 3 / 2 + 1 + }; + let new_ttl = if entry.ttl > 0 { entry.ttl - 1 } else { new_original_ttl }; + let parent_entry = self.graph.get_mut(&parent_id).expect("present or inserted"); + if parent_entry.original_ttl < new_original_ttl { + parent_entry.original_ttl = new_original_ttl; + parent_entry.ttl = new_ttl; + } + } + Ok(true) + } +} + +impl<'a> Negotiator for Algorithm<'a> { + fn known_common(&mut self, id: ObjectId) -> Result<(), Error> { + if self + .graph + .get(&id) + .map_or(false, |entry| entry.flags.contains(Flags::SEEN)) + { + return Ok(()); + } + self.add_to_queue(id, Flags::ADVERTISED) + } + + fn add_tip(&mut self, id: ObjectId) -> Result<(), Error> { + if self + .graph + .get(&id) + .map_or(false, |entry| entry.flags.contains(Flags::SEEN)) + { + return Ok(()); + } + self.add_to_queue(id, Flags::default()) + } + + fn next_have(&mut self) -> Option> { + let mut parents = SmallVec::new(); + loop { + let id = self.revs.pop().filter(|_| self.non_common_revs != 0)?; + let entry = self.graph.get_mut(&id).expect("it was added to the graph by now"); + entry.flags |= Flags::POPPED; + + if !entry.flags.contains(Flags::COMMON) { + self.non_common_revs -= 1; + } + let mut to_send = None; + if !entry.flags.contains(Flags::COMMON) && entry.ttl == 0 { + to_send = Some(id); + } + let entry = *entry; + + let commit = match self.graph.try_lookup(&id) { + Ok(c) => c.expect("it was found before, must still be there"), + Err(err) => return Some(Err(err.into())), + }; + if let Err(err) = collect_parents(commit.iter_parents(), &mut parents) { + return Some(Err(err)); + } + let mut parent_pushed = false; + for parent_id in parents.drain(..) { + parent_pushed |= match self.push_parent(entry, parent_id) { + Ok(r) => r, + Err(err) => return Some(Err(err)), + } + } + + if !entry.flags.contains(Flags::COMMON) && !parent_pushed { + to_send = Some(id); + } + + if let Some(to_send) = to_send { + return Some(Ok(to_send)); + } + } + } + + fn in_common_with_remote(&mut self, id: ObjectId) -> Result { + let mut was_seen = false; + let known_to_be_common = self.graph.get(&id).map_or(false, |entry| { + was_seen = entry.flags.contains(Flags::SEEN); + entry.flags.contains(Flags::COMMON) + }); + assert!( + was_seen, + "Cannot receive ACK for commit we didn't send a HAVE for: {id}" + ); + self.mark_common(id)?; + Ok(known_to_be_common) + } +} diff --git a/gix-negotiate/tests/baseline/mod.rs b/gix-negotiate/tests/baseline/mod.rs new file mode 100644 index 00000000000..fd3063aa235 --- /dev/null +++ b/gix-negotiate/tests/baseline/mod.rs @@ -0,0 +1,216 @@ +use gix_negotiate::Algorithm; +use gix_object::bstr; +use gix_object::bstr::ByteSlice; +use gix_odb::Find; +use gix_odb::FindExt; +use gix_ref::file::ReferenceExt; +use gix_ref::store::WriteReflog; +use std::cell::RefCell; + +#[test] +fn run() -> crate::Result { + let root = gix_testtools::scripted_fixture_read_only("make_repos.sh")?; + for case in [ + "no_parents", + "clock_skew", + "two_colliding_skips", + "multi_round", + "advertisement_as_filter", + ] { + let base = root.join(case); + + for (algo_name, algo) in [ + ("noop", Algorithm::Noop), + ("consecutive", Algorithm::Consecutive), + ("skipping", Algorithm::Skipping), + ] { + let obj_buf = RefCell::new(Vec::new()); + let buf = std::fs::read(base.join(format!("baseline.{algo_name}")))?; + let store = gix_odb::at(base.join("client").join(".git/objects"))?; + let refs = gix_ref::file::Store::at( + base.join("client").join(".git"), + WriteReflog::Disable, + gix_hash::Kind::Sha1, + ); + let lookup_names = |names: &[&str]| -> Vec { + names + .iter() + .filter_map(|name| { + refs.try_find(*name).expect("one tag per commit").map(|mut r| { + r.peel_to_id_in_place(&refs, |id, buf| { + store.try_find(id, buf).map(|d| d.map(|d| (d.kind, d.data))) + }) + .expect("works"); + r.target.into_id() + }) + }) + .collect() + }; + let message = |id| { + store + .find_commit(id, obj_buf.borrow_mut().as_mut()) + .expect("present") + .message + .trim() + .as_bstr() + .to_owned() + }; + + for use_cache in [false, true] { + let cache = use_cache + .then(|| gix_commitgraph::at(store.store_ref().path().join("info")).ok()) + .flatten(); + let mut negotiator = algo.into_negotiator( + |id, buf| { + store + .try_find(id, buf) + .map(|r| r.and_then(|d| d.try_into_commit_iter())) + }, + cache, + ); + eprintln!("ALGO {algo_name} CASE {case}"); + // // In --negotiate-only mode, which seems to be the only thing that's working after trying --dry-run, we unfortunately + // // don't get to see what happens if known-common commits are added as git itself doesn't do that in this mode + // // for some reason. + // for common in lookup_names(&["origin/main"]) { + // eprintln!("COMMON {name} {common}", name = message(common)); + // negotiator.known_common(common)?; + // } + for tip in lookup_names(&["HEAD"]).into_iter().chain( + refs.iter()? + .prefixed("refs/heads")? + .filter_map(Result::ok) + .map(|r| r.target.into_id()), + ) { + eprintln!("TIP {name} {tip}", name = message(tip)); + negotiator.add_tip(tip)?; + } + for (round, Round { mut haves, common }) in ParseRounds::new(buf.lines()).enumerate() { + if algo == Algorithm::Skipping { + if case == "clock_skew" { + // Here for some reason the prio-queue of git manages to not sort the parent of C2, which is in the future, to be + // ahead of old4 that is in the past. In the git version of this test, they say to expect exactly this sequence + // as well even though it's not actually happening (but that they can't see due to the way they are testing). + haves = lookup_names(&["c2", "c1", "old4", "old2", "old1"]); + } else if case == "two_colliding_skips" { + // The same thing, we actually get exactly the right order, whereas git for some reason doesn't. + // This is the order expected in the git tests. + haves = lookup_names(&["c5side", "c11", "c9", "c6", "c1"]); + } else if case == "multi_round" && round == 1 { + // Here, probably also because of priority queue quirks, `git` emits the commits out of order, with only one + // branch, b5 I think, being out of place. This list puts the expectation in the right order, which is ordered + // by commit date. + haves = lookup_names(&[ + "b8.c14", "b7.c14", "b6.c14", "b5.c14", "b4.c14", "b3.c14", "b2.c14", "b8.c9", "b7.c9", + "b6.c9", "b5.c9", "b4.c9", "b3.c9", "b2.c9", "b8.c1", "b7.c1", "b6.c1", "b5.c1", + "b4.c1", "b3.c1", "b2.c1", "b8.c0", "b7.c0", "b6.c0", "b5.c0", "b4.c0", "b3.c0", + "b2.c0", + ]); + } else if case == "advertisement_as_filter" { + haves = lookup_names(&["c2side", "c5", "origin/main"]) + .into_iter() + .chain(Some( + gix_hash::ObjectId::from_hex(b"f36cefa0be2ac180d360a54b1cc4214985cea60a").unwrap(), + )) + .collect(); + } + } + for have in haves { + let actual = negotiator.next_have().unwrap_or_else(|| { + panic!("{algo_name}:cache={use_cache}: one have per baseline: {have} missing or in wrong order", have = message(have)) + })?; + assert_eq!( + actual, + have, + "{algo_name}:cache={use_cache}: order and commit matches exactly, wanted {expected}, got {actual}, commits left: {:?}", + std::iter::from_fn(|| negotiator.next_have()).map(|id| message(id.unwrap())).collect::>(), + actual = message(actual), + expected = message(have) + ); + eprintln!("have {}", message(actual)); + } + for common_revision in common { + eprintln!("ACK {}", message(common_revision)); + negotiator.in_common_with_remote(common_revision)?; + } + } + assert!( + negotiator.next_have().is_none(), + "{algo_name}:cache={use_cache}: negotiator should be depleted after all recorded baseline rounds" + ); + } + } + } + Ok(()) +} + +struct ParseRounds<'a> { + lines: bstr::Lines<'a>, +} + +impl<'a> ParseRounds<'a> { + pub fn new(mut lines: bstr::Lines<'a>) -> Self { + parse::command(&mut lines, parse::Command::Incoming).expect("handshake"); + Self { lines } + } +} + +impl<'a> Iterator for ParseRounds<'a> { + type Item = Round; + + fn next(&mut self) -> Option { + let haves = parse::object_ids("have", parse::command(&mut self.lines, parse::Command::Outgoing)?); + let common = parse::object_ids("ACK", parse::command(&mut self.lines, parse::Command::Incoming)?); + if haves.is_empty() { + assert!(common.is_empty(), "cannot ack what's not there"); + return None; + } + Round { haves, common }.into() + } +} + +struct Round { + pub haves: Vec, + pub common: Vec, +} + +mod parse { + use gix_object::bstr; + use gix_object::bstr::{BStr, ByteSlice}; + + #[derive(Debug, Eq, PartialEq, Copy, Clone)] + pub enum Command { + Incoming, + Outgoing, + } + + pub fn object_ids(prefix: &str, lines: impl IntoIterator>) -> Vec { + lines + .into_iter() + .filter_map(|line| { + line.as_ref() + .strip_prefix(prefix.as_bytes()) + .map(|id| gix_hash::ObjectId::from_hex(id.trim()).expect("valid hash")) + }) + .collect() + } + + pub fn command<'a>(lines: &mut bstr::Lines<'a>, wanted: Command) -> Option> { + let mut out = Vec::new(); + for line in lines { + let pos = line.find(b"fetch").expect("fetch token"); + let line_mode = match &line[pos + 5..][..2] { + b"< " => Command::Incoming, + b"> " => Command::Outgoing, + invalid => unreachable!("invalid fetch token: {:?}", invalid.as_bstr()), + }; + assert_eq!(line_mode, wanted, "command with unexpected mode"); + let line = line[pos + 7..].as_bstr(); + if line == "0000" { + break; + } + out.push(line); + } + (!out.is_empty()).then_some(out) + } +} diff --git a/gix-negotiate/tests/fixtures/generated-archives/make_repos.tar.xz b/gix-negotiate/tests/fixtures/generated-archives/make_repos.tar.xz new file mode 100644 index 00000000000..47e834dd56e --- /dev/null +++ b/gix-negotiate/tests/fixtures/generated-archives/make_repos.tar.xz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcfe215bf6f1db2f757b9201c5c3682ddc4340a627832a03a70d5768fe192beb +size 92828 diff --git a/gix-negotiate/tests/fixtures/make_repos.sh b/gix-negotiate/tests/fixtures/make_repos.sh new file mode 100644 index 00000000000..b1b96527d89 --- /dev/null +++ b/gix-negotiate/tests/fixtures/make_repos.sh @@ -0,0 +1,146 @@ +#!/bin/bash +set -eu -o pipefail + +function tick () { + if test -z "${tick+set}" + then + tick=1112911993 + else + tick=$(($tick + 60)) + fi + GIT_COMMITTER_DATE="$tick -0700" + GIT_AUTHOR_DATE="$tick -0700" + export GIT_COMMITTER_DATE GIT_AUTHOR_DATE +} + +tick +function commit() { + local message=${1:?first argument is the commit message} + local file="$message.t" + echo "$1" > "$file" + git add -- "$file" + tick + git commit -m "$message" + git tag "$message" +} + +function negotiation_tips () { + local tips="" + for arg in "$@"; do + tips+=" --negotiation-tip=$arg" + done + echo "$tips" +} + +function trace_fetch_baseline () { + local remote="${1:?need remote url}"; shift + git -C client commit-graph write --no-progress --reachable + git -C client repack -adq + + for algo in noop consecutive skipping; do + GIT_TRACE_PACKET="$PWD/baseline.$algo" \ + git -C client -c fetch.negotiationAlgorithm="$algo" fetch --negotiate-only $(negotiation_tips "$@") \ + --upload-pack 'unset GIT_TRACE_PACKET; git-upload-pack' \ + "$remote" || : + done +} + + +(mkdir no_parents && cd no_parents + git init -q server && cd server + commit to_fetch + cd .. + + (git init -q client && cd client + for i in $(seq 7); do + commit c$i + done + ) + + trace_fetch_baseline file://$PWD/server main +) + +(mkdir two_colliding_skips && cd two_colliding_skips + git init -q server && cd server + commit to_fetch + cd .. + + (git init -q client && cd client + for i in $(seq 11); do + commit c$i + done + git checkout c5 + commit c5side + ) + + trace_fetch_baseline file://$PWD/server HEAD main +) + +(mkdir advertisement_as_filter && cd advertisement_as_filter + git init -q server && cd server + commit c1 + commit c2 + commit c3 + git tag -d c1 c2 c3 + cd .. + git clone server client && cd client + commit c4 + commit c5 + git checkout c4^^ + commit c2side + cd .. + (cd server + git checkout --orphan anotherbranch + commit to_fetch + ) + + trace_fetch_baseline origin HEAD main +) + + +(mkdir multi_round && cd multi_round + git init -q server && cd server + commit to_fetch + cd .. + + git init -q client && cd client + for i in $(seq 8); do + git checkout --orphan b$i && + commit b$i.c0 + done + + for j in $(seq 19); do + for i in $(seq 8); do + git checkout b$i && + commit b$i.c$j + done + done + cd .. + (cd server + git fetch --no-tags "$PWD/../client" b1:refs/heads/b1 + git checkout b1 + commit commit-on-b1 + ) + trace_fetch_baseline file://$PWD/server $(ls client/.git/refs/heads | sort) +) + +(mkdir clock_skew && cd clock_skew + git init -q server && cd server + commit to_fetch + cd .. + + (git init -q client && cd client + tick=2000000000 + commit c1 + commit c2 + + tick=1000000000 + git checkout c1 + commit old1 + commit old2 + commit old3 + commit old4 + ) + + trace_fetch_baseline file://$PWD/server HEAD main +) diff --git a/gix-negotiate/tests/negotiate.rs b/gix-negotiate/tests/negotiate.rs new file mode 100644 index 00000000000..4e6a42981f3 --- /dev/null +++ b/gix-negotiate/tests/negotiate.rs @@ -0,0 +1,37 @@ +use gix_testtools::Result; + +mod window_size { + use gix_negotiate::window_size; + + #[test] + fn initial_value_without_previous_window_size() { + assert_eq!(window_size(false, None), 16); + assert_eq!(window_size(true, None), 16); + } + + #[test] + fn transport_is_stateless() { + let mut ws = window_size(true, None); + for expected in [32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 18022, 19824] { + ws = window_size(true, ws); + assert_eq!(ws, expected); + } + } + + #[test] + fn transport_is_not_stateless() { + let mut ws = window_size(false, None); + for expected in [32, 64, 96] { + ws = window_size(false, ws); + assert_eq!(ws, expected); + } + + let mut ws = 4; + for expected in [8, 16, 32, 64, 96] { + ws = window_size(false, ws); + assert_eq!(ws, expected); + } + } +} + +mod baseline; diff --git a/gix-refspec/Cargo.toml b/gix-refspec/Cargo.toml index d2a295687b1..272cf8c66eb 100644 --- a/gix-refspec/Cargo.toml +++ b/gix-refspec/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gix-refspec" -version = "0.10.1" +version = "0.11.0" repository = "https://github.com/Byron/gitoxide" license = "MIT/Apache-2.0" description = "A WIP crate of the gitoxide project for parsing and representing refspecs" @@ -13,7 +13,7 @@ rust-version = "1.64" doctest = false [dependencies] -gix-revision = { version = "^0.13.0", path = "../gix-revision" } +gix-revision = { version = "^0.14.0", path = "../gix-revision" } gix-validate = { version = "^0.7.3", path = "../gix-validate" } gix-hash = { version = "^0.11.1", path = "../gix-hash" } diff --git a/gix-revision/CHANGELOG.md b/gix-revision/CHANGELOG.md index 8df1aff193c..e00fc24c2d9 100644 --- a/gix-revision/CHANGELOG.md +++ b/gix-revision/CHANGELOG.md @@ -5,6 +5,44 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.14.0 (2023-05-19) + +### New Features + + - A Graph for quick access to commits and for associating state with them. + This data structure should be used whenever stateful traversal is required, usually + by associating information with each commit to remember what was seen and what wasn't. + - A PriorityQueue that is useful for graph traversal. + +### New Features (BREAKING) + + - `describe` usees commitgraph. + With it it can leverage the commitgraph data structure would would be more prominent + on server-side applications, presumably. + +### Commit Statistics + + + + - 5 commits contributed to the release over the course of 7 calendar days. + - 22 days passed between releases. + - 3 commits were understood as [conventional](https://www.conventionalcommits.org). + - 0 issues like '(#ID)' were seen in commit messages + +### Commit Details + + + +
view details + + * **Uncategorized** + - `describe` usees commitgraph. ([`ed258da`](https://github.com/Byron/gitoxide/commit/ed258da9015d2d68734aeac485dd009760fc4da4)) + - A Graph for quick access to commits and for associating state with them. ([`59ce4c6`](https://github.com/Byron/gitoxide/commit/59ce4c606f8ccd9b6a16da2025e6746984d32fd6)) + - A PriorityQueue that is useful for graph traversal. ([`dde8c3a`](https://github.com/Byron/gitoxide/commit/dde8c3aca545ba20cd5752f02283b98647fd3970)) + - Make clear that we do handle shallow repos, refactor tests ([`fc423e4`](https://github.com/Byron/gitoxide/commit/fc423e470de50491a725d4802066d26c05bd2b2a)) + - Release gix-object v0.29.2 ([`4f879bf`](https://github.com/Byron/gitoxide/commit/4f879bf35653bdc8f9729d524c6e8e1fb3c6886b)) +
+ ## 0.13.0 (2023-04-26) ### New Features (BREAKING) @@ -21,7 +59,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - - 10 commits contributed to the release over the course of 14 calendar days. + - 11 commits contributed to the release over the course of 14 calendar days. - 25 days passed between releases. - 1 commit was understood as [conventional](https://www.conventionalcommits.org). - 1 unique issue was worked on: [#814](https://github.com/Byron/gitoxide/issues/814) @@ -41,6 +79,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * **[#814](https://github.com/Byron/gitoxide/issues/814)** - Rename `serde1` cargo feature to `serde` and use the weak-deps cargo capability. ([`b83ee36`](https://github.com/Byron/gitoxide/commit/b83ee366a3c65c717beb587ad809268f1c54b8ad)) * **Uncategorized** + - Release gix-index v0.16.0, gix-mailmap v0.12.0, gix-pack v0.34.0, gix-odb v0.44.0, gix-packetline v0.16.0, gix-transport v0.30.0, gix-protocol v0.31.0, gix-revision v0.13.0, gix-refspec v0.10.0, gix-worktree v0.16.0, gix v0.44.0 ([`d7173b2`](https://github.com/Byron/gitoxide/commit/d7173b2d2cb79685fdf7f618c31c576db24fa648)) - Release gix-index v0.16.0, gix-mailmap v0.12.0, gix-pack v0.34.0, gix-odb v0.44.0, gix-packetline v0.16.0, gix-transport v0.30.0, gix-protocol v0.31.0, gix-revision v0.13.0, gix-refspec v0.10.0, gix-worktree v0.16.0, gix v0.44.0 ([`e4df557`](https://github.com/Byron/gitoxide/commit/e4df5574c0813a0236319fa6e8b3b41bab179fc8)) - Release gix-hash v0.11.1, gix-path v0.7.4, gix-glob v0.6.0, gix-attributes v0.11.0, gix-config-value v0.11.0, gix-fs v0.1.1, gix-tempfile v5.0.3, gix-utils v0.1.1, gix-lock v5.0.1, gix-object v0.29.1, gix-ref v0.28.0, gix-sec v0.7.0, gix-config v0.21.0, gix-prompt v0.4.0, gix-url v0.17.0, gix-credentials v0.13.0, gix-diff v0.29.0, gix-discover v0.17.0, gix-hashtable v0.2.0, gix-ignore v0.1.0, gix-bitmap v0.2.3, gix-traverse v0.25.0, gix-index v0.16.0, gix-mailmap v0.12.0, gix-pack v0.34.0, gix-odb v0.44.0, gix-packetline v0.16.0, gix-transport v0.30.0, gix-protocol v0.31.0, gix-revision v0.13.0, gix-refspec v0.10.0, gix-worktree v0.16.0, gix v0.44.0, safety bump 7 crates ([`91134a1`](https://github.com/Byron/gitoxide/commit/91134a11c8ba0e942f692488ec9bce9fa1086324)) - Prepare changelogs prior to release ([`30a1a71`](https://github.com/Byron/gitoxide/commit/30a1a71f36f24faac0e0b362ffdfedea7f9cdbf1)) diff --git a/gix-revision/Cargo.toml b/gix-revision/Cargo.toml index 21516155a2c..f7fbd9beebf 100644 --- a/gix-revision/Cargo.toml +++ b/gix-revision/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gix-revision" -version = "0.13.0" +version = "0.14.0" repository = "https://github.com/Byron/gitoxide" license = "MIT/Apache-2.0" description = "A WIP crate of the gitoxide project dealing with finding names for revisions and parsing specifications" @@ -21,7 +21,7 @@ gix-hash = { version = "^0.11.1", path = "../gix-hash" } gix-object = { version = "^0.29.2", path = "../gix-object" } gix-date = { version = "^0.5.0", path = "../gix-date" } gix-hashtable = { version = "^0.2.0", path = "../gix-hashtable" } -gix-commitgraph = { version = "0.14.0", path = "../gix-commitgraph" } +gix-commitgraph = { version = "^0.15.0", path = "../gix-commitgraph" } bstr = { version = "1.3.0", default-features = false, features = ["std"]} thiserror = "1.0.26" diff --git a/gix-revision/src/graph.rs b/gix-revision/src/graph.rs index 9b02bc2868c..88f33db25e6 100644 --- a/gix-revision/src/graph.rs +++ b/gix-revision/src/graph.rs @@ -3,8 +3,38 @@ use gix_hash::oid; use smallvec::SmallVec; use std::ops::Index; +/// The time in seconds since unix epoch at which a commit was created. +pub type CommitterTimestamp = u64; + +impl<'find, T: std::fmt::Debug> std::fmt::Debug for Graph<'find, T> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Debug::fmt(&self.set, f) + } +} + +impl<'find, T: Default> Graph<'find, T> { + /// Lookup `id` without failing if the commit doesn't exist, and assure that `id` is inserted into our set. + /// If it wasn't, associate it with the default value. Assure `update_data(data)` gets run. + /// Return the commit when done. + /// Note that none of the data updates happen if there was no commit named `id`. + pub fn try_lookup_and_insert( + &mut self, + id: gix_hash::ObjectId, + update_data: impl FnOnce(&mut T), + ) -> Result>, lookup::Error> { + self.try_lookup_and_insert_default(id, || T::default(), update_data) + } +} + impl<'find, T> Graph<'find, T> { /// Create a new instance with `find` to retrieve commits and optionally `cache` to accelerate commit access. + /// + /// ### Performance + /// + /// `find` should be optimized to access the same object repeatedly, ideally with an object cache to keep the last couple of + /// most recently used commits. + /// Furthermore, **none-existing commits should not trigger the pack-db to be refreshed.** Otherwise, performance may be sub-optimal + /// in shallow repositories as running into non-existing commits will trigger a refresh of the `packs` directory. pub fn new(mut find: Find, cache: impl Into>) -> Self where Find: @@ -22,6 +52,33 @@ impl<'find, T> Graph<'find, T> { } } + /// Lookup `id` without failing if the commit doesn't exist, and assure that `id` is inserted into our set + /// with a `default` value assigned to it. + /// If it wasn't, associate it with the default value. Assure `update_data(data)` gets run. + /// Return the commit when done. + /// Note that none of the data updates happen if there was no commit named `id`. + pub fn try_lookup_and_insert_default( + &mut self, + id: gix_hash::ObjectId, + default: impl FnOnce() -> T, + update_data: impl FnOnce(&mut T), + ) -> Result>, lookup::Error> { + let res = try_lookup(&id, &mut self.find, self.cache.as_ref(), &mut self.buf)?; + Ok(res.map(|commit| { + match self.set.entry(id) { + gix_hashtable::hash_map::Entry::Vacant(entry) => { + let mut data = default(); + update_data(&mut data); + entry.insert(data); + } + gix_hashtable::hash_map::Entry::Occupied(mut entry) => { + update_data(entry.get_mut()); + } + }; + commit + })) + } + /// Returns true if `id` has data associated with it, meaning that we processed it already. pub fn contains(&self, id: &gix_hash::oid) -> bool { self.set.contains_key(id.as_ref()) @@ -66,7 +123,7 @@ impl<'find, T> Graph<'find, T> { pub fn insert_parents( &mut self, id: &gix_hash::oid, - mut new_parent_data: impl FnMut(gix_hash::ObjectId, u64) -> T, + mut new_parent_data: impl FnMut(gix_hash::ObjectId, CommitterTimestamp) -> T, mut update_existing: impl FnMut(gix_hash::ObjectId, &mut T), first_parent: bool, ) -> Result<(), insert_parents::Error> { @@ -135,10 +192,10 @@ impl<'a, 'find, T> Index<&'a gix_hash::oid> for Graph<'find, T> { pub mod lookup { /// The error returned by [`try_lookup()`][crate::Graph::try_lookup()]. #[derive(Debug, thiserror::Error)] - #[allow(missing_docs)] - pub enum Error { - #[error("There was an error looking up a commit")] - Find(#[from] Box), + #[error("There was an error looking up a commit")] + pub struct Error { + #[from] + source: Box, } /// @@ -186,7 +243,7 @@ pub struct Commit<'graph> { /// pub mod commit { use super::Commit; - use crate::graph::Either; + use crate::graph::{CommitterTimestamp, Either}; impl<'graph> Commit<'graph> { /// Return an iterator over the parents of this commit. @@ -202,13 +259,13 @@ pub mod commit { /// /// This is the single-most important date for determining recency of commits. /// Note that this can only fail if the commit is backed by the object database *and* parsing fails. - pub fn committer_timestamp(&self) -> Result { + pub fn committer_timestamp(&self) -> Result { Ok(match &self.backing { Either::Left(buf) => { gix_object::CommitRefIter::from_bytes(buf) .committer()? .time - .seconds_since_unix_epoch as u64 + .seconds_since_unix_epoch as CommitterTimestamp } Either::Right((cache, pos)) => cache.commit_at(*pos).committer_timestamp(), }) diff --git a/gix-revision/src/queue.rs b/gix-revision/src/queue.rs index 2fc88c34187..5d24e1638b5 100644 --- a/gix-revision/src/queue.rs +++ b/gix-revision/src/queue.rs @@ -7,6 +7,18 @@ pub(crate) struct Item { value: T, } +impl std::fmt::Debug for Item { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "({:?}: {:?})", self.key, self.value) + } +} + +impl std::fmt::Debug for PriorityQueue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Debug::fmt(&self.0, f) + } +} + impl PartialEq for Item { fn eq(&self, other: &Self) -> bool { Ord::cmp(self, other).is_eq() @@ -28,6 +40,10 @@ impl Ord for Item { } impl PriorityQueue { + /// Create a new instance. + pub fn new() -> Self { + PriorityQueue(Default::default()) + } /// Insert `value` so that it is ordered according to `key`. pub fn insert(&mut self, key: K, value: T) { self.0.push(Item { key, value }); diff --git a/gix/Cargo.toml b/gix/Cargo.toml index 37e11095a8d..81ca16f0a0b 100644 --- a/gix/Cargo.toml +++ b/gix/Cargo.toml @@ -3,7 +3,7 @@ name = "gix" repository = "https://github.com/Byron/gitoxide" description = "Interact with git repositories just like git would" license = "MIT/Apache-2.0" -version = "0.44.1" +version = "0.45.0" authors = ["Sebastian Thiel "] edition = "2021" include = ["src/**/*", "CHANGELOG.md"] @@ -122,7 +122,7 @@ gix-lock = { version = "^5.0.0", path = "../gix-lock" } gix-validate = { version = "^0.7.4", path = "../gix-validate" } gix-sec = { version = "^0.8.0", path = "../gix-sec" } gix-date = { version = "^0.5.0", path = "../gix-date" } -gix-refspec = { version = "^0.10.1", path = "../gix-refspec" } +gix-refspec = { version = "^0.11.0", path = "../gix-refspec" } gix-config = { version = "^0.22.0", path = "../gix-config" } gix-odb = { version = "^0.45.0", path = "../gix-odb" } @@ -130,7 +130,7 @@ gix-hash = { version = "^0.11.1", path = "../gix-hash" } gix-object = { version = "^0.29.2", path = "../gix-object" } gix-actor = { version = "^0.20.0", path = "../gix-actor" } gix-pack = { version = "^0.35.0", path = "../gix-pack", features = ["object-cache-dynamic"] } -gix-revision = { version = "^0.13.0", path = "../gix-revision" } +gix-revision = { version = "^0.14.0", path = "../gix-revision" } gix-path = { version = "^0.8.0", path = "../gix-path" } gix-url = { version = "^0.18.0", path = "../gix-url" } @@ -149,7 +149,7 @@ gix-prompt = { version = "^0.5.0", path = "../gix-prompt" } gix-index = { version = "^0.16.1", path = "../gix-index" } gix-worktree = { version = "^0.17.1", path = "../gix-worktree" } gix-hashtable = { version = "^0.2.0", path = "../gix-hashtable" } -gix-commitgraph = { version = "^0.14.0", path = "../gix-commitgraph" } +gix-commitgraph = { version = "^0.15.0", path = "../gix-commitgraph" } prodash = { version = "23.1", optional = true, default-features = false, features = ["progress-tree"] } once_cell = "1.14.0"