feat: add basic connectivity check

Implement a simple connectivity check in a new `gix-fsck` crate, and add
this to `gix` via a new `fsck` subcommand. Currently this is
functionally equivalent to:
`git rev-list --objects --quiet --missing=print`

Author: cesfahani

Cargo.lock

@@ -281,6 +281,7 @@ members = [
     "gix-archive",
     "gix-worktree-stream",
     "gix-revwalk",
+    "gix-fsck",
 
     "tests/tools",
 
@@ -294,6 +295,7 @@ members = [
     "gix-ref/tests",
     "gix-config/tests",
     "gix-traverse/tests",
+    "gix-fsck/tests",
 ]
 
 [workspace.dependencies]

Cargo.toml

@@ -49,6 +49,7 @@ gix-pack-for-configuration-only = { package = "gix-pack", version = "^0.44.0", p
 gix-transport-configuration-only = { package = "gix-transport", version = "^0.38.0", path = "../gix-transport", default-features = false }
 gix-archive-for-configuration-only = { package = "gix-archive", version = "^0.6.0", path = "../gix-archive", optional = true, features = ["tar", "tar_gz"] }
 gix-status = { version = "^0.2.0", path = "../gix-status" }
+gix-fsck = { version = "^0.0.0", path = "../gix-fsck" }
 serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"] }
 anyhow = "1.0.42"
 thiserror = "1.0.34"

gitoxide-core/Cargo.toml

@@ -0,0 +1,40 @@
+use std::io::{BufWriter, Write};
+
+use anyhow::Context;
+use gix::{objs::Kind, ObjectId};
+
+pub fn connectivity(mut repo: gix::Repository, spec: Option<String>, out: impl std::io::Write) -> anyhow::Result<()> {
+    let mut out = BufWriter::with_capacity(64 * 1024, out);
+    let spec = spec.unwrap_or("HEAD".into());
+
+    repo.object_cache_size_if_unset(4 * 1024 * 1024);
+    // We expect to be finding a bunch of non-existent objects here - never refresh the ODB
+    repo.objects.refresh_never();
+
+    let id = repo
+        .rev_parse_single(spec.as_str())
+        .context("Only single revisions are supported")?;
+    let commits: gix::revision::Walk<'_> = id
+        .object()?
+        .peel_to_kind(gix::object::Kind::Commit)
+        .context("Need commitish as starting point")?
+        .id()
+        .ancestors()
+        .all()?;
+
+    let missing_cb = |oid: &ObjectId, kind: Kind| {
+        writeln!(out, "{oid}: {kind}").expect("failed to write output");
+    };
+    let mut conn = gix_fsck::ConnectivityCheck::new(&repo.objects, missing_cb);
+
+    // Walk all commits, checking each one for connectivity
+    for commit in commits {
+        let commit = commit?;
+        conn.check_commit(&commit.id);
+        for parent in commit.parent_ids {
+            conn.check_commit(&parent);
+        }
+    }
+
+    Ok(())
+}

gitoxide-core/src/repository/fsck.rs

@@ -35,6 +35,7 @@ pub use clone::function::clone;
 pub use fetch::function::fetch;
 
 pub mod commitgraph;
+pub mod fsck;
 pub mod index;
 pub mod mailmap;
 pub mod odb;

gitoxide-core/src/repository/mod.rs

@@ -0,0 +1,20 @@
+[package]
+name = "gix-fsck"
+version = "0.0.0"
+repository = "https://github.com/Byron/gitoxide"
+authors = ["Cameron Esfahani <[email protected]>", "Sebastian Thiel <[email protected]>"]
+license = "MIT OR Apache-2.0"
+description = "Verifies the connectivity and validity of objects in the database"
+edition = "2021"
+include = ["src/**/*", "LICENSE-*", "CHANGELOG.md"]
+rust-version = "1.65"
+autotests = false
+
+[lib]
+doctest = false
+
+[dependencies]
+gix-hash = { version = "^0.13.1", path = "../gix-hash" }
+gix-hashtable = { version = "^0.4.0", path = "../gix-hashtable" }
+gix-object = { version = "^0.38.0", path = "../gix-object" }
+gix-odb = { version = "^0.54.0", path = "../gix-odb" }

gix-fsck/Cargo.toml

@@ -0,0 +1,6 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

gix-fsck/Changelog.md

@@ -0,0 +1 @@
+../LICENSE-APACHE

gix-fsck/LICENSE-APACHE

@@ -0,0 +1 @@
+../LICENSE-MIT

gix-fsck/LICENSE-MIT

@@ -0,0 +1,127 @@
+//! A library for performing object database integrity and connectivity checks
+#![deny(rust_2018_idioms)]
+
+use gix_hash::ObjectId;
+use gix_hashtable::HashSet;
+use gix_object::{tree::EntryMode, Kind};
+use gix_odb::{pack::Find, FindExt, Handle};
+
+pub struct ConnectivityCheck<'a, F>
+where
+    F: FnMut(&ObjectId, Kind),
+{
+    /// ODB handle to use for the check
+    db: &'a Handle,
+    /// Closure to invoke when a missing object is encountered
+    missing_cb: F,
+    /// Set of Object IDs already (or about to be) scanned during the check
+    oid_set: HashSet,
+    /// Single buffer for decoding objects from the ODB
+    /// This is slightly faster than allocating throughout the connectivity check (and reduces the memory requirements)
+    buf: Vec<u8>,
+}
+
+impl<'a, F> ConnectivityCheck<'a, F>
+where
+    F: FnMut(&ObjectId, Kind),
+{
+    /// Instantiate a connectivity check
+    pub fn new(db: &'a Handle, missing_cb: F) -> ConnectivityCheck<'a, F> {
+        ConnectivityCheck {
+            db,
+            missing_cb,
+            oid_set: HashSet::default(),
+            buf: Vec::new(),
+        }
+    }
+
+    /// Run the connectivity check on the provided commit object ID
+    /// - This will walk the trees and blobs referenced by the commit and verify they exist in the ODB
+    /// - Any objects previously encountered by this [`ConnectivityCheck`] instance will be skipped
+    /// - Any referenced blobs that are not present in the ODB will result in a call to the  `missing_cb`
+    /// - Missing commits or trees will currently result in panic
+    ///     - TODO: add support for missing trees
+    ///     - TODO: consider how to handle a missing commit (invoke `missing_cb`, or possibly return a Result?)
+    pub fn check_commit(&mut self, oid: &ObjectId) {
+        // Attempt to insert the commit ID in the set, and if already present, return immediately
+        if !self.oid_set.insert(*oid) {
+            return;
+        }
+        // Obtain the commit's tree ID
+        let tree_id = {
+            let commit = self.db.find_commit(oid, &mut self.buf).expect("failed to find commit");
+            commit.tree()
+        };
+
+        // Attempt to insert the tree ID in the set, and if already present, return immediately
+        if self.oid_set.insert(tree_id) {
+            self.check_tree(&tree_id);
+        }
+    }
+
+    fn check_tree(&mut self, oid: &ObjectId) {
+        let tree = match self.db.find_tree(oid, &mut self.buf) {
+            Ok(tree) => tree,
+            Err(_) => {
+                // Tree is missing, so invoke `missing_cb`
+                (self.missing_cb)(oid, Kind::Tree);
+                return;
+            }
+        };
+
+        // Keeping separate sets for trees and blobs for now...
+        // This is about a wash when compared to using a HashMap<ObjectID, Kind>
+        struct TreeEntries {
+            trees: HashSet<ObjectId>,
+            blobs: HashSet<ObjectId>,
+        }
+
+        // Build up a set of trees and a set of blobs
+        let entries: TreeEntries = {
+            let mut entries = TreeEntries {
+                trees: HashSet::default(),
+                blobs: HashSet::default(),
+            };
+
+            // For each entry in the tree
+            for entry_ref in tree.entries.iter() {
+                match entry_ref.mode {
+                    EntryMode::Tree => {
+                        let tree_id = entry_ref.oid.to_owned();
+                        // Check if the tree has already been encountered
+                        if self.oid_set.insert(tree_id) {
+                            entries.trees.insert(tree_id);
+                        }
+                    }
+                    EntryMode::Blob | EntryMode::BlobExecutable | EntryMode::Link => {
+                        let blob_id = entry_ref.oid.to_owned();
+                        // Check if the blob has already been encountered
+                        if self.oid_set.insert(blob_id) {
+                            entries.blobs.insert(blob_id);
+                        }
+                    }
+                    EntryMode::Commit => {
+                        // This implies a submodule (OID is the commit hash of the submodule)
+                        // Skip it as it's not in this repository!
+                    }
+                }
+            }
+            entries
+        };
+
+        for tree_id in entries.trees.iter() {
+            self.check_tree(tree_id);
+        }
+        for blob_id in entries.blobs.iter() {
+            self.check_blob(blob_id);
+        }
+    }
+
+    fn check_blob(&mut self, oid: &ObjectId) {
+        // Check if the blob is missing from the ODB
+        if !self.db.contains(oid) {
+            // Blob is missing, so invoke `missing_cb`
+            (self.missing_cb)(oid, Kind::Blob);
+        }
+    }
+}

gix-fsck/src/lib.rs

@@ -0,0 +1,22 @@
+[package]
+name = "gix-fsck-tests"
+version = "0.0.0"
+repository = "https://github.com/Byron/gitoxide"
+authors = ["Cameron Esfahani <[email protected]>", "Sebastian Thiel <[email protected]>"]
+license = "MIT OR Apache-2.0"
+description = "Tests for gix-odb with feature-toggle support"
+edition = "2021"
+rust-version = "1.65"
+publish = false
+
+[[test]]
+name = "integrate"
+path = "integrate.rs"
+
+[dev-dependencies]
+gix-fsck = { path = ".." }
+gix-hash = { path = "../../gix-hash" }
+gix-hashtable = { path = "../../gix-hashtable" }
+gix-object = { path = "../../gix-object" }
+gix-odb = { path = "../../gix-odb" }
+gix-testtools = { path = "../../tests/tools"}

gix-fsck/tests/Cargo.toml

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2dbf9142109ea28a9b170efafa7e7a5845a7ef5f024063ded42513b9f29b3d3
+size 13696

gix-fsck/tests/fixtures/generated-archives/make_test_repos.tar.xz

@@ -0,0 +1,65 @@
+#!/bin/bash
+set -x
+set -euo pipefail
+
+# Stolen from `set-static-git-environment` in tests/helpers.sh
+export GIT_AUTHOR_DATE="2020-09-09 09:06:03 +0800"
+export GIT_COMMITTER_DATE="${GIT_AUTHOR_DATE}"
+export GIT_AUTHOR_NAME="Sebastian Thiel"
+export GIT_COMMITTER_NAME="${GIT_AUTHOR_NAME}"
+export GIT_AUTHOR_EMAIL="[email protected]"
+export GIT_COMMITTER_EMAIL="${GIT_AUTHOR_EMAIL}"
+
+# Ignore the system config
+export GIT_CONFIG_SYSTEM=""
+# We override the global config with our own local one (see below)
+export GIT_CONFIG_GLOBAL="$PWD/.gitconfig"
+
+# The default value for `init.defaultBranch` may soon change, so to be safe, set it explicitly
+git config --global init.defaultBranch main
+# We need to be able to do partial clones, so enable it
+# Note: This used to only work at global scope, so to be safe, we'll keep it there (no harm in doing so)
+git config --global uploadpack.allowFilter true
+# Disable the reflog, as that will cause the absolute path of the remote to be inserted
+git config --global core.logAllRefUpdates false
+
+# Helper for _nearly_ reproducible clones
+# - this fixes up the remote URL to be relative (`git` will resolve it to an absolute path upon a clone operation)
+# - clones made with this helper will still have an index (`.git/index`)
+# - the index contains lots of non-reproducible fields (e.g. mtime, ctime, inode number, uid, gid)
+# - we _could_ remove the index file and later rebuild it from the working tree, but no current need to do so
+function git_static_clone() {
+    git clone --no-local --no-hardlinks "$@"
+    # Get aboslute paths of the clone destination and the remote
+    CLONE_PATH="$(realpath "${@: -1}")"
+    REMOTE_PATH="$(realpath "${@: (-2):1}")"
+    # Get the remote path relative to the clone path
+    REMOTE_REL_PATH="$(realpath --relative-to="$CLONE_PATH" "$REMOTE_PATH")"
+    (
+        cd "$CLONE_PATH"
+        # Fixup the remote URL to be relative
+        git remote set-url origin "$REMOTE_REL_PATH"
+    )
+}
+
+# First build out a base repository
+git init base
+(
+    cd base
+
+    echo "blob 1" > blob-1
+    git add -A
+    git commit -m "commit 1"
+    echo "blob-2" > blob-2
+    git add -A
+    git commit -m "commit 2"
+    git rm blob-1
+    git add -A
+    git commit -m "commit 3"
+)
+
+# Blobless clone
+git_static_clone --filter=blob:none ./base blobless
+
+# Treeless (and blobless) clone
+git_static_clone --filter=tree:0 ./base treeless