single and multi-threaded index tests (#293)

Author: Byron

Makefile

@@ -140,6 +140,8 @@ unit-tests: ## run all unit tests
 	cd git-object && cargo test && cargo test --features verbose-object-parsing-errors
 	cd git-pack && cargo test --features internal-testing-to-avoid-being-run-by-cargo-test-all \
 				&& cargo test --features "internal-testing-git-features-parallel"
+	cd git-index && cargo test --features internal-testing-to-avoid-being-run-by-cargo-test-all \
+				&& cargo test --features "internal-testing-git-features-parallel"
 	cd git-packetline && cargo test \
 					  && cargo test --features blocking-io,maybe-async/is_sync --test blocking-packetline \
 					  && cargo test --features "async-io" --test async-packetline

etc/check-package-size.sh

@@ -18,7 +18,7 @@ echo "in root: gitoxide CLI"
 #indent cargo diet -n --package-size-limit 25KB - fails right now because of dotted profile.dev.package
 (enter cargo-smart-release && indent cargo diet -n --package-size-limit 85KB)
 (enter git-actor && indent cargo diet -n --package-size-limit 5KB)
-(enter git-index && indent cargo diet -n --package-size-limit 10KB)
+(enter git-index && indent cargo diet -n --package-size-limit 15KB)
 (enter git-tempfile && indent cargo diet -n --package-size-limit 25KB)
 (enter git-lock && indent cargo diet -n --package-size-limit 15KB)
 (enter git-config && indent cargo diet -n --package-size-limit 65KB)

git-features/src/parallel/mod.rs

@@ -111,15 +111,15 @@ pub fn optimize_chunk_size_and_thread_limit(
 
 /// Always returns 1, available when the `parallel` feature toggle is unset.
 #[cfg(not(feature = "parallel"))]
-fn num_threads(_thread_limit: Option<usize>) -> usize {
+pub fn num_threads(_thread_limit: Option<usize>) -> usize {
     1
 }
 
 /// Returns the amount of threads the system can effectively use as the amount of its logical cores.
 ///
 /// Only available with the `parallel` feature toggle set.
 #[cfg(feature = "parallel")]
-fn num_threads(thread_limit: Option<usize>) -> usize {
+pub fn num_threads(thread_limit: Option<usize>) -> usize {
     let logical_cores = num_cpus::get();
     thread_limit
         .map(|l| if l == 0 { logical_cores } else { l })

git-index/Cargo.toml

@@ -11,9 +11,22 @@ edition = "2021"
 doctest = false
 test = true
 
+[[test]]
+name = "multi-threaded"
+path = "tests/index-multi-threaded.rs"
+required-features = ["internal-testing-git-features-parallel"]
+
+[[test]]
+name = "single-threaded"
+path = "tests/index-single-threaded.rs"
+required-features = ["internal-testing-to-avoid-being-run-by-cargo-test-all"]
+
 [features]
 serde1 = ["serde"]
 
+internal-testing-git-features-parallel = ["git-features/parallel"]
+internal-testing-to-avoid-being-run-by-cargo-test-all = []
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]

git-index/src/decode/mod.rs

@@ -37,7 +37,7 @@ pub struct Options {
     ///
     /// This applies to loading extensions in parallel to entries if the common EOIE extension is available.
     /// It also allows to use multiple threads for loading entries if the IEOT extension is present.
-    pub num_threads: Option<usize>,
+    pub thread_limit: Option<usize>,
     /// The minimum size in bytes to load extensions in their own thread, assuming there is enough `num_threads` available.
     pub min_extension_block_in_bytes_for_threading: usize,
 }
@@ -48,22 +48,24 @@ impl State {
         timestamp: FileTime,
         Options {
             object_hash,
-            num_threads: _,
+            thread_limit,
             min_extension_block_in_bytes_for_threading: _,
         }: Options,
     ) -> Result<(Self, git_hash::ObjectId), Error> {
         let (version, num_entries, post_header_data) = header::decode(data, object_hash)?;
         let start_of_extensions = extension::end_of_index_entry::decode(data, object_hash);
 
+        let num_threads = git_features::parallel::num_threads(thread_limit);
         let path_backing_buffer_size = entries::estimate_path_storage_requirements_in_bytes(
             num_entries,
             data.len(),
             start_of_extensions,
             object_hash,
             version,
         );
+
         let (entries, ext, data) = match start_of_extensions {
-            Some(offset) => {
+            Some(offset) if num_threads > 1 => {
                 let start_of_extensions = &data[offset..];
                 let index_offsets_table = extension::index_entry_offset_table::find(start_of_extensions, object_hash);
                 let (entries_res, (ext, data)) = match index_offsets_table {
@@ -89,7 +91,7 @@ impl State {
                 };
                 (entries_res?.0, ext, data)
             }
-            None => {
+            None | Some(_) => {
                 let (entries, data) = entries::load_all(
                     post_header_data,
                     num_entries,

git-index/src/lib.rs

@@ -110,3 +110,12 @@ pub(crate) mod util {
         data.split_at(pos).into()
     }
 }
+
+#[test]
+fn size_of_entry() {
+    assert_eq!(std::mem::size_of::<crate::Entry>(), 80);
+
+    // the reason we have our own time is half the size.
+    assert_eq!(std::mem::size_of::<crate::entry::Time>(), 8);
+    assert_eq!(std::mem::size_of::<filetime::FileTime>(), 16);
+}

git-index/tests/index-multi-threaded.rs

@@ -0,0 +1 @@
+mod index;

git-index/tests/index-single-threaded.rs

@@ -0,0 +1 @@
+mod index;

git-index/tests/file/mod.rs renamed to git-index/tests/index/file/mod.rs

@@ -3,7 +3,7 @@ mod init {
     use git_testtools::hex_to_id;
 
     fn file(name: &str) -> git_index::File {
-        git_index::File::at(crate::index_fixture_path(name), git_index::decode::Options::default()).unwrap()
+        git_index::File::at(crate::index::fixture_path(name), git_index::decode::Options::default()).unwrap()
     }
 
     #[test]

git-index/tests/index.rs renamed to git-index/tests/index/mod.rs

@@ -2,7 +2,7 @@ use std::path::{Path, PathBuf};
 
 mod file;
 
-pub fn index_fixture_path(name: &str) -> PathBuf {
+pub fn fixture_path(name: &str) -> PathBuf {
     let dir = git_testtools::scripted_fixture_repo_read_only(Path::new("make_index").join(name).with_extension("sh"))
         .expect("script works");
     dir.join(".git").join("index")