A baseline test for the noop negotiator

Author: Byron

Cargo.lock

@@ -17,3 +17,7 @@ gix-hash = { version = "^0.11.1", path = "../gix-hash" }
 gix-object = { version = "^0.29.2", path = "../gix-object" }
 gix-commitgraph = { version = "^0.15.0", path = "../gix-commitgraph" }
 gix-revision = { version = "^0.14.0", path = "../gix-revision" }
+
+[dev-dependencies]
+gix-testtools = { path = "../tests/tools" }
+gix-odb = { path = "../gix-odb" }

gix-negotiate/Cargo.toml

@@ -9,7 +9,7 @@ mod noop;
 /// The way the negotiation is performed.
 #[derive(Default, Debug, Copy, Clone, Eq, PartialEq)]
 pub enum Algorithm {
-    /// Do not send any information at all, likely at cost of larger-than-necessary packs.
+    /// Do not send any information at all, which typically leads to complete packs to be sent.
     Noop,
     /// Walk over consecutive commits and check each one. This can be costly be assures packs are exactly the size they need to be.
     #[default]

gix-negotiate/src/lib.rs

@@ -0,0 +1,130 @@
+use gix_negotiate::Algorithm;
+use gix_object::bstr;
+use gix_object::bstr::ByteSlice;
+use gix_odb::Find;
+
+#[test]
+fn run() -> crate::Result {
+    let root = gix_testtools::scripted_fixture_read_only("make_repos.sh")?;
+    for case in ["no_parents"] {
+        let base = root.join(case);
+
+        for (algo_name, algo) in [
+            ("noop", Algorithm::Noop),
+            // ("consecutive", Algorithm::Consecutive),
+            // ("skipping", Algorithm::Skipping),
+        ] {
+            let buf = std::fs::read(base.join(format!("baseline.{algo_name}")))?;
+            let tips = parse::object_ids("", std::fs::read(base.join("tips"))?.lines());
+            let store = gix_odb::at(base.join("client").join(".git/objects"))?;
+
+            for use_cache in [false, true] {
+                let cache = use_cache
+                    .then(|| gix_commitgraph::at(store.store_ref().path().join("info")).ok())
+                    .flatten();
+                let mut negotiator = algo.into_negotiator(
+                    |id, buf| {
+                        store
+                            .try_find(id, buf)
+                            .map(|r| r.and_then(|d| d.try_into_commit_iter()))
+                    },
+                    cache,
+                );
+                for tip in &tips {
+                    negotiator.add_tip(tip);
+                }
+                for Round { haves, common } in ParseRounds::new(buf.lines()) {
+                    for have in haves {
+                        let actual = negotiator.next_have().unwrap_or_else(|| {
+                            panic!(
+                                "{algo_name}: one have per baseline: {have} missing or in wrong order, left: {:?}",
+                                std::iter::from_fn(|| negotiator.next_have()).collect::<Vec<_>>()
+                            )
+                        });
+                        assert_eq!(actual, have, "{algo_name}: order and commit matches exactly");
+                    }
+                    for common_revision in common {
+                        negotiator.in_common_with_remote(&common_revision);
+                    }
+                }
+                assert_eq!(
+                    negotiator.next_have(),
+                    None,
+                    "{algo_name}: negotiator should be depleted after all recorded baseline rounds"
+                );
+            }
+        }
+    }
+    Ok(())
+}
+
+struct ParseRounds<'a> {
+    lines: bstr::Lines<'a>,
+}
+
+impl<'a> ParseRounds<'a> {
+    pub fn new(mut lines: bstr::Lines<'a>) -> Self {
+        parse::command(&mut lines, parse::Command::Incoming).expect("handshake");
+        Self { lines }
+    }
+}
+
+impl<'a> Iterator for ParseRounds<'a> {
+    type Item = Round;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let haves = parse::object_ids("have", parse::command(&mut self.lines, parse::Command::Outgoing)?);
+        let common = parse::object_ids("ACK", parse::command(&mut self.lines, parse::Command::Incoming)?);
+        if haves.is_empty() {
+            assert!(common.is_empty(), "cannot ack what's not there");
+            return None;
+        }
+        Round { haves, common }.into()
+    }
+}
+
+struct Round {
+    pub haves: Vec<gix_hash::ObjectId>,
+    pub common: Vec<gix_hash::ObjectId>,
+}
+
+mod parse {
+    use gix_object::bstr;
+    use gix_object::bstr::{BStr, ByteSlice};
+
+    #[derive(Debug, Eq, PartialEq, Copy, Clone)]
+    pub enum Command {
+        Incoming,
+        Outgoing,
+    }
+
+    pub fn object_ids(prefix: &str, lines: impl IntoIterator<Item = impl AsRef<[u8]>>) -> Vec<gix_hash::ObjectId> {
+        lines
+            .into_iter()
+            .filter_map(|line| {
+                line.as_ref()
+                    .strip_prefix(prefix.as_bytes())
+                    .map(|id| gix_hash::ObjectId::from_hex(id.trim()).expect("valid hash"))
+            })
+            .collect()
+    }
+
+    pub fn command<'a>(lines: &mut bstr::Lines<'a>, wanted: Command) -> Option<Vec<&'a BStr>> {
+        let mut out = Vec::new();
+        for line in lines {
+            let pos = line.find(b"fetch").expect("fetch token");
+            let line_mode = match &line[pos + 5..][..2] {
+                b"< " => Command::Incoming,
+                b"> " => Command::Outgoing,
+                invalid => unreachable!("invalid fetch token: {:?}", invalid.as_bstr()),
+            };
+            assert_eq!(line_mode, wanted, "command with unexpected mode");
+            let line = line[pos + 7..].as_bstr();
+            if line == "0000" {
+                break;
+            }
+            out.push(line);
+        }
+        (!out.is_empty()).then_some(out)
+    }
+}

gix-negotiate/tests/baseline/mod.rs

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:157b0dfa79e30e3b1f6808f4e6f9c62e3ec136b2366b16e81007644db49c6d9a
+size 87912

gix-negotiate/tests/fixtures/generated-archives/make_repos.tar.xz

@@ -0,0 +1,124 @@
+#!/bin/bash
+set -eu -o pipefail
+
+function tick () {
+  if test -z "${tick+set}"
+  then
+    tick=1112911993
+  else
+    tick=$(($tick + 60))
+  fi
+  GIT_COMMITTER_DATE="$tick -0700"
+  GIT_AUTHOR_DATE="$tick -0700"
+  export GIT_COMMITTER_DATE GIT_AUTHOR_DATE
+}
+
+tick
+function commit() {
+  local message=${1:?first argument is the commit message}
+  local file="$message.t"
+  echo "$1" > "$file"
+  git add -- "$file"
+  git commit -m "$message"
+  git tag "$message"
+  tick
+}
+
+function negotiation_tips () {
+  local tips=""
+  for arg in "$@"; do
+    tips+=" --negotiation-tip=$arg"
+  done
+  echo "$tips"
+}
+
+function trace_fetch_baseline () {
+  git -C client commit-graph write --no-progress --reachable
+  git -C client repack -adq
+
+  for tip in "$@"; do git -C client rev-parse "$tip" >> tips; done
+  for algo in noop consecutive skipping; do
+    GIT_TRACE_PACKET="$PWD/baseline.$algo" \
+    git -C client -c fetch.negotiationAlgorithm="$algo" fetch --negotiate-only $(negotiation_tips "$@") \
+      --upload-pack 'unset GIT_TRACE_PACKET; git-upload-pack' \
+      file://$PWD/server || :
+  done
+}
+
+
+(mkdir no_parents && cd no_parents
+  (git init -q server && cd server
+    commit to_fetch
+  )
+
+  (git init -q client && cd client
+    for i in $(seq 7); do
+      commit c$i
+    done
+  )
+
+  trace_fetch_baseline main
+)
+
+(mkdir two_colliding_skips && cd two_colliding_skips
+  (git init -q server && cd server
+    commit to_fetch
+  )
+
+  (git init -q client && cd client
+    for i in $(seq 11); do
+      commit c$i
+    done
+    git checkout c5
+    commit c5side
+  )
+
+  trace_fetch_baseline HEAD main
+)
+
+(mkdir multi_round && cd multi_round
+  (git init -q server && cd server
+    commit to_fetch
+  )
+
+  (git init -q client && cd client
+    for i in $(seq 8); do
+      git checkout --orphan b$i &&
+      commit b$i.c0
+    done
+
+    for j in $(seq 19); do
+      for i in $(seq 8); do
+        git checkout b$i &&
+        commit b$i.c$j
+      done
+    done
+  )
+  (cd server
+    git fetch --no-tags "$PWD/../client" b1:refs/heads/b1
+    git checkout b1
+    commit commit-on-b1
+  )
+  trace_fetch_baseline $(ls client/.git/refs/heads | sort)
+)
+
+(mkdir clock_skew && cd clock_skew
+  (git init -q server && cd server
+    commit to_fetch
+  )
+
+  (git init -q client && cd client
+    tick=2000000000
+    commit c1
+    commit c2
+
+    tick=1000000000
+    git checkout c1
+    commit old1
+    commit old2
+    commit old3
+    commit old4
+  )
+
+  trace_fetch_baseline HEAD main
+)

gix-negotiate/tests/fixtures/make_repos.sh

@@ -1,3 +1,5 @@
+use gix_testtools::Result;
+
 mod window_size {
     use gix_negotiate::window_size;
 
@@ -31,3 +33,5 @@ mod window_size {
         }
     }
 }
+
+mod baseline;