reduce memory usage of pathological packs

Cargo.toml

@@ -157,10 +157,10 @@ gix = { version = "^0.44.1", path = "gix", default-features = false }
 time = "0.3.19"
 
 clap = { version = "4.1.1", features = ["derive", "cargo"] }
-prodash = { version = "23.1", optional = true, default-features = false }
+prodash = { version = "25.0.0", optional = true, default-features = false }
 is-terminal = { version = "0.4.0", optional = true }
 env_logger = { version = "0.10.0", default-features = false }
-crosstermion = { version = "0.10.1", optional = true, default-features = false }
+crosstermion = { version = "0.11.0", optional = true, default-features = false }
 futures-lite = { version = "1.12.0", optional = true, default-features = false, features = ["std"] }
 
 # for progress

Makefile

@@ -151,6 +151,7 @@ unit-tests: ## run all unit tests
 	cd gix-ref/tests && cargo test --all-features
 	cd gix-odb && cargo test && cargo test --all-features
 	cd gix-object && cargo test && cargo test --features verbose-object-parsing-errors
+	cd gix-pack && cargo test --all-features
 	cd gix-pack/tests && cargo test --features internal-testing-to-avoid-being-run-by-cargo-test-all \
 				&& cargo test --features "internal-testing-gix-features-parallel"
 	cd gix-index/tests && cargo test --features internal-testing-to-avoid-being-run-by-cargo-test-all \

gitoxide-core/src/hours/mod.rs

@@ -344,10 +344,10 @@ where
                 }
                 None => Vec::new(),
             };
-            if let Some(mut progress) = change_progress {
+            if let Some(progress) = change_progress {
                 progress.show_throughput(start);
             }
-            if let Some(mut progress) = lines_progress {
+            if let Some(progress) = lines_progress {
                 progress.show_throughput(start);
             }
 

gitoxide-core/src/organize.rs

@@ -12,15 +12,12 @@ pub enum Mode {
     Simulate,
 }
 
-fn find_git_repository_workdirs<P: Progress>(
+fn find_git_repository_workdirs(
     root: impl AsRef<Path>,
-    mut progress: P,
+    mut progress: impl Progress,
     debug: bool,
     threads: Option<usize>,
-) -> impl Iterator<Item = (PathBuf, gix::Kind)>
-where
-    P::SubProgress: Sync,
-{
+) -> impl Iterator<Item = (PathBuf, gix::Kind)> {
     progress.init(None, progress::count("filesystem items"));
     fn is_repository(path: &Path) -> Option<gix::Kind> {
         // Can be git dir or worktree checkout (file)
@@ -211,10 +208,7 @@ pub fn discover<P: Progress>(
     mut progress: P,
     debug: bool,
     threads: Option<usize>,
-) -> anyhow::Result<()>
-where
-    <P::SubProgress as Progress>::SubProgress: Sync,
-{
+) -> anyhow::Result<()> {
     for (git_workdir, _kind) in
         find_git_repository_workdirs(source_dir, progress.add_child("Searching repositories"), debug, threads)
     {
@@ -229,10 +223,7 @@ pub fn run<P: Progress>(
     destination: impl AsRef<Path>,
     mut progress: P,
     threads: Option<usize>,
-) -> anyhow::Result<()>
-where
-    <P::SubProgress as Progress>::SubProgress: Sync,
-{
+) -> anyhow::Result<()> {
     let mut num_errors = 0usize;
     let destination = destination.as_ref().canonicalize()?;
     for (path_to_move, kind) in

gitoxide-core/src/pack/explode.rs

@@ -90,6 +90,7 @@ enum Error {
 }
 
 #[allow(clippy::large_enum_variant)]
+#[derive(Clone)]
 enum OutputWriter {
     Loose(loose::Store),
     Sink(odb::Sink),
@@ -177,51 +178,57 @@ pub fn pack_or_pack_index(
             }
         });
 
-    let pack::index::traverse::Outcome{mut progress, ..} = bundle
+    let pack::index::traverse::Outcome { progress, .. } = bundle
         .index
         .traverse(
             &bundle.pack,
             progress,
             &should_interrupt,
             {
                 let object_path = object_path.map(|p| p.as_ref().to_owned());
-                move || {
-                    let out = OutputWriter::new(object_path.clone(), sink_compress, object_hash);
-                    let loose_odb = verify.then(|| object_path.as_ref().map(|path| loose::Store::at(path, object_hash))).flatten();
-                    let mut read_buf = Vec::new();
-                    move |object_kind, buf, index_entry, progress| {
-                        let written_id = out.write_buf(object_kind, buf).map_err(|err| {
-                            Error::Write{source: Box::new(err) as Box<dyn std::error::Error + Send + Sync>,
+                let out = OutputWriter::new(object_path.clone(), sink_compress, object_hash);
+                let loose_odb = verify
+                    .then(|| object_path.as_ref().map(|path| loose::Store::at(path, object_hash)))
+                    .flatten();
+                let mut read_buf = Vec::new();
+                move |object_kind, buf, index_entry, progress| {
+                    let written_id = out.write_buf(object_kind, buf).map_err(|err| Error::Write {
+                        source: Box::new(err) as Box<dyn std::error::Error + Send + Sync>,
+                        kind: object_kind,
+                        id: index_entry.oid,
+                    })?;
+                    if written_id != index_entry.oid {
+                        if let object::Kind::Tree = object_kind {
+                            progress.info(format!(
+                                "The tree in pack named {} was written as {} due to modes 100664 and 100640 rewritten as 100644.",
+                                index_entry.oid, written_id
+                            ));
+                        } else {
+                            return Err(Error::ObjectEncodeMismatch {
                                 kind: object_kind,
-                                id: index_entry.oid,
-                            }
-                        })?;
-                        if written_id != index_entry.oid {
-                            if let object::Kind::Tree = object_kind {
-                                progress.info(format!(
-                                    "The tree in pack named {} was written as {} due to modes 100664 and 100640 rewritten as 100644.",
-                                    index_entry.oid, written_id
-                                ));
-                            } else {
-                                return Err(Error::ObjectEncodeMismatch{kind: object_kind, actual: index_entry.oid, expected:written_id});
-                            }
+                                actual: index_entry.oid,
+                                expected: written_id,
+                            });
                         }
-                        if let Some(verifier) = loose_odb.as_ref() {
-                            let obj = verifier
-                                .try_find(written_id, &mut read_buf)
-                                .map_err(|err| Error::WrittenFileCorrupt{source:err, id:written_id})?
-                                .ok_or(Error::WrittenFileMissing{id:written_id})?;
-                            obj.verify_checksum(written_id)?;
-                        }
-                        Ok(())
                     }
+                    if let Some(verifier) = loose_odb.as_ref() {
+                        let obj = verifier
+                            .try_find(written_id, &mut read_buf)
+                            .map_err(|err| Error::WrittenFileCorrupt {
+                                source: err,
+                                id: written_id,
+                            })?
+                            .ok_or(Error::WrittenFileMissing { id: written_id })?;
+                        obj.verify_checksum(written_id)?;
+                    }
+                    Ok(())
                 }
             },
             pack::index::traverse::Options {
                 traversal: algorithm,
                 thread_limit,
                 check: check.into(),
-                make_pack_lookup_cache:             pack::cache::lru::StaticLinkedList::<64>::default,
+                make_pack_lookup_cache: pack::cache::lru::StaticLinkedList::<64>::default,
             },
         )
         .with_context(|| "Failed to explode the entire pack - some loose objects may have been created nonetheless")?;

gitoxide-core/src/query/engine/update.rs

@@ -46,14 +46,14 @@ pub fn update(
     };
     let commit_counter = db_progress.counter().expect("shared counter available");
 
-    let mut change_progress = {
+    let change_progress = {
         let mut p = progress.add_child("find changes");
         p.init(None, progress::count("modified files"));
         p
     };
     let change_counter = change_progress.counter().expect("shared counter available");
 
-    let mut lines_progress = {
+    let lines_progress = {
         let mut p = progress.add_child("find changes");
         p.init(None, progress::count("diff lines"));
         p

gitoxide-core/src/repository/verify.rs

@@ -31,7 +31,7 @@ pub fn integrity(
     }: Context,
 ) -> anyhow::Result<()> {
     #[cfg_attr(not(feature = "serde"), allow(unused))]
-    let mut outcome = repo.objects.store_ref().verify_integrity(
+    let outcome = repo.objects.store_ref().verify_integrity(
         progress,
         should_interrupt,
         gix::odb::pack::index::verify::integrity::Options {

gix-features/Cargo.toml

@@ -5,7 +5,7 @@ repository = "https://github.com/Byron/gitoxide"
 version = "0.29.0"
 authors = ["Sebastian Thiel <[email protected]>"]
 license = "MIT/Apache-2.0"
-edition = "2018"
+edition = "2021"
 rust-version = "1.64"
 
 [lib]
@@ -123,7 +123,7 @@ crc32fast = { version = "1.2.1", optional = true }
 sha1 = { version = "0.10.0", optional = true }
 
 # progress
-prodash = { version = "23.1", optional = true, default-features = false }
+prodash = { version = "25.0.0", optional = true, default-features = false }
 bytesize = { version = "1.0.1", optional = true }
 
 # pipe