remove byteorder from git-pack (#293)

It's sufficiently well supported using the standard library now.

Author: Byron

Cargo.lock

@@ -43,7 +43,6 @@ git-tempfile = { version ="^1.0.0", path = "../git-tempfile" }
 
 smallvec = "1.3.0"
 memmap2 = "0.5.0"
-byteorder = "1.2.3"
 serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"] }
 bytesize = "1.0.1"
 os_str_bytes = "6.0.0"

git-pack/Cargo.toml

@@ -1,5 +1,3 @@
-use byteorder::{BigEndian, ByteOrder};
-
 use crate::data;
 
 pub(crate) const N32_SIZE: usize = std::mem::size_of::<u32>();
@@ -11,13 +9,13 @@ pub fn decode(data: &[u8; 12]) -> Result<(data::Version, u32), decode::Error> {
         return Err(decode::Error::Corrupt("Pack data type not recognized".into()));
     }
     ofs += N32_SIZE;
-    let kind = match BigEndian::read_u32(&data[ofs..ofs + N32_SIZE]) {
+    let kind = match crate::read_u32(&data[ofs..ofs + N32_SIZE]) {
         2 => data::Version::V2,
         3 => data::Version::V3,
         v => return Err(decode::Error::UnsupportedVersion(v)),
     };
     ofs += N32_SIZE;
-    let num_objects = BigEndian::read_u32(&data[ofs..ofs + N32_SIZE]);
+    let num_objects = crate::read_u32(&data[ofs..ofs + N32_SIZE]);
 
     Ok((kind, num_objects))
 }

git-pack/src/data/header.rs

@@ -1,7 +1,5 @@
 use std::mem::size_of;
 
-use byteorder::{BigEndian, ByteOrder};
-
 use crate::{
     data,
     index::{self, EntryIndex, FAN_LEN},
@@ -39,7 +37,7 @@ impl index::File {
                     let (ofs, oid) = c.split_at(N32_SIZE);
                     Entry {
                         oid: git_hash::ObjectId::from(oid),
-                        pack_offset: BigEndian::read_u32(ofs) as u64,
+                        pack_offset: crate::read_u32(ofs) as u64,
                         crc32: None,
                     }
                 }),
@@ -59,7 +57,7 @@ impl index::File {
             .map(move |(oid, crc32, ofs32)| Entry {
                 oid: git_hash::ObjectId::from(oid),
                 pack_offset: self.pack_offset_from_offset_v2(ofs32, pack64_offset),
-                crc32: Some(BigEndian::read_u32(crc32)),
+                crc32: Some(crate::read_u32(crc32)),
             }),
             _ => panic!("Cannot use iter_v2() on index of type {:?}", self.version),
         }
@@ -94,7 +92,7 @@ impl index::File {
             }
             index::Version::V1 => {
                 let start = V1_HEADER_SIZE + index * (N32_SIZE + self.hash_len);
-                BigEndian::read_u32(&self.data[start..][..N32_SIZE]) as u64
+                crate::read_u32(&self.data[start..][..N32_SIZE]) as u64
             }
         }
     }
@@ -110,7 +108,7 @@ impl index::File {
         match self.version {
             index::Version::V2 => {
                 let start = self.offset_crc32_v2() + index * N32_SIZE;
-                Some(BigEndian::read_u32(&self.data[start..start + N32_SIZE]))
+                Some(crate::read_u32(&self.data[start..start + N32_SIZE]))
             }
             index::Version::V1 => None,
         }
@@ -153,14 +151,13 @@ impl index::File {
         let mut ofs: Vec<_> = match self.version {
             index::Version::V1 => self.iter().map(|e| e.pack_offset).collect(),
             index::Version::V2 => {
-                let mut v = Vec::with_capacity(self.num_objects as usize);
-                let mut ofs32 = &self.data[self.offset_pack_offset_v2()..];
-                let pack_offset_64 = self.offset_pack_offset64_v2();
-                for _ in 0..self.num_objects {
-                    v.push(self.pack_offset_from_offset_v2(ofs32, pack_offset_64));
-                    ofs32 = &ofs32[4..];
-                }
-                v
+                let offset32_start = &self.data[self.offset_pack_offset_v2()..];
+                let pack_offset_64_start = self.offset_pack_offset64_v2();
+                offset32_start
+                    .chunks(N32_SIZE)
+                    .take(self.num_objects as usize)
+                    .map(|offset| self.pack_offset_from_offset_v2(offset, pack_offset_64_start))
+                    .collect()
             }
         };
         ofs.sort_unstable();
@@ -185,10 +182,10 @@ impl index::File {
     #[inline]
     fn pack_offset_from_offset_v2(&self, offset: &[u8], pack64_offset: usize) -> data::Offset {
         debug_assert_eq!(self.version, index::Version::V2);
-        let ofs32 = BigEndian::read_u32(offset);
+        let ofs32 = crate::read_u32(offset);
         if (ofs32 & N32_HIGH_BIT) == N32_HIGH_BIT {
             let from = pack64_offset + (ofs32 ^ N32_HIGH_BIT) as usize * N64_SIZE;
-            BigEndian::read_u64(&self.data[from..][..N64_SIZE])
+            crate::read_u64(&self.data[from..][..N64_SIZE])
         } else {
             ofs32 as u64
         }

git-pack/src/index/access.rs

@@ -1,7 +1,5 @@
 use std::{mem::size_of, path::Path};
 
-use byteorder::{BigEndian, ByteOrder};
-
 use crate::index::{self, Version, FAN_LEN, V2_SIGNATURE};
 
 /// Returned by [`index::File::at()`].
@@ -57,7 +55,7 @@ impl index::File {
             let d = {
                 if let Version::V2 = kind {
                     let (vd, dr) = d.split_at(N32_SIZE);
-                    let version = BigEndian::read_u32(vd);
+                    let version = crate::read_u32(vd);
                     if version != Version::V2 as u32 {
                         return Err(Error::UnsupportedVersion { version });
                     }
@@ -87,7 +85,7 @@ impl index::File {
 fn read_fan(d: &[u8]) -> ([u32; FAN_LEN], usize) {
     let mut fan = [0; FAN_LEN];
     for (c, f) in d.chunks(N32_SIZE).zip(fan.iter_mut()) {
-        *f = BigEndian::read_u32(c);
+        *f = crate::read_u32(c);
     }
     (fan, FAN_LEN * N32_SIZE)
 }

git-pack/src/index/init.rs

@@ -3,7 +3,6 @@ use std::{cmp::Ordering, io};
 pub(crate) const LARGE_OFFSET_THRESHOLD: u64 = 0x7fff_ffff;
 pub(crate) const HIGH_BIT: u32 = 0x8000_0000;
 
-use byteorder::{BigEndian, WriteBytesExt};
 use git_features::{
     hash,
     progress::{self, Progress},
@@ -35,15 +34,15 @@ pub(crate) fn write_to(
         hash::Write::new(out, kind.hash()),
     ));
     out.write_all(V2_SIGNATURE)?;
-    out.write_u32::<BigEndian>(kind as u32)?;
+    out.write_all(&(kind as u32).to_be_bytes())?;
 
     progress.init(Some(4), progress::steps());
     let start = std::time::Instant::now();
     let _info = progress.add_child("writing fan-out table");
     let fan_out = fanout(entries_sorted_by_oid.iter().map(|e| e.data.id.first_byte()));
 
     for value in fan_out {
-        out.write_u32::<BigEndian>(value)?;
+        out.write_all(&value.to_be_bytes())?;
     }
 
     progress.inc();
@@ -55,15 +54,15 @@ pub(crate) fn write_to(
     progress.inc();
     let _info = progress.add_child("writing crc32");
     for entry in &entries_sorted_by_oid {
-        out.write_u32::<BigEndian>(entry.data.crc32)?;
+        out.write_all(&entry.data.crc32.to_be_bytes())?;
     }
 
     progress.inc();
     let _info = progress.add_child("writing offsets");
     {
         let mut offsets64 = Vec::<u64>::new();
         for entry in &entries_sorted_by_oid {
-            out.write_u32::<BigEndian>(if entry.offset > LARGE_OFFSET_THRESHOLD {
+            let offset: u32 = if entry.offset > LARGE_OFFSET_THRESHOLD {
                 assert!(
                     offsets64.len() < LARGE_OFFSET_THRESHOLD as usize,
                     "Encoding breakdown - way too many 64bit offsets"
@@ -72,10 +71,11 @@ pub(crate) fn write_to(
                 ((offsets64.len() - 1) as u32) | HIGH_BIT
             } else {
                 entry.offset as u32
-            })?;
+            };
+            out.write_all(&offset.to_be_bytes())?;
         }
         for value in offsets64 {
-            out.write_u64::<BigEndian>(value)?;
+            out.write_all(&value.to_be_bytes())?;
         }
     }
 

git-pack/src/index/write/encode.rs

@@ -32,6 +32,7 @@ pub mod cache;
 pub mod data;
 
 mod find_traits;
+
 pub use find_traits::{Find, FindExt};
 
 ///
@@ -54,3 +55,15 @@ mod mmap {
         }
     }
 }
+
+use std::convert::TryInto;
+
+#[inline]
+fn read_u32(b: &[u8]) -> u32 {
+    u32::from_be_bytes(b.try_into().unwrap())
+}
+
+#[inline]
+fn read_u64(b: &[u8]) -> u64 {
+    u64::from_be_bytes(b.try_into().unwrap())
+}

git-pack/src/lib.rs

@@ -1,7 +1,5 @@
 use std::path::{Path, PathBuf};
 
-use byteorder::{BigEndian, ByteOrder};
-
 use crate::{
     data,
     multi_index::{EntryIndex, File, PackIndex, Version},
@@ -102,15 +100,15 @@ impl File {
 
         const HIGH_BIT: u32 = 1 << 31;
 
-        let pack_index = BigEndian::read_u32(&self.data[start..][..4]);
+        let pack_index = crate::read_u32(&self.data[start..][..4]);
         let offset = &self.data[start + 4..][..4];
-        let ofs32 = BigEndian::read_u32(offset);
+        let ofs32 = crate::read_u32(offset);
         let pack_offset = if (ofs32 & HIGH_BIT) == HIGH_BIT {
             // We determine if large offsets are actually larger than 4GB and if not, we don't use the high-bit to signal anything
             // but allow the presence of the large-offset chunk to signal what's happening.
             if let Some(offsets_64) = self.large_offsets_ofs {
                 let from = offsets_64 + (ofs32 ^ HIGH_BIT) as usize * 8;
-                BigEndian::read_u64(&self.data[from..][..8])
+                crate::read_u64(&self.data[from..][..8])
             } else {
                 ofs32 as u64
             }

git-pack/src/multi_index/access.rs

@@ -108,8 +108,6 @@ pub mod index_names {
 pub mod fanout {
     use std::convert::TryInto;
 
-    use byteorder::{BigEndian, WriteBytesExt};
-
     use crate::multi_index;
 
     /// The size of the fanout table
@@ -138,7 +136,7 @@ pub mod fanout {
         let fanout = crate::index::write::encode::fanout(sorted_entries.iter().map(|e| e.id.first_byte()));
 
         for value in fanout {
-            out.write_u32::<BigEndian>(value)?;
+            out.write_all(&value.to_be_bytes())?;
         }
         Ok(())
     }
@@ -178,8 +176,6 @@ pub mod lookup {
 pub mod offsets {
     use std::{convert::TryInto, ops::Range};
 
-    use byteorder::{BigEndian, WriteBytesExt};
-
     use crate::multi_index;
 
     /// The id uniquely identifying the offsets table.
@@ -199,7 +195,7 @@ pub mod offsets {
         let mut num_large_offsets = 0u32;
 
         for entry in sorted_entries {
-            out.write_u32::<BigEndian>(entry.pack_index)?;
+            out.write_all(&entry.pack_index.to_be_bytes())?;
 
             let offset: u32 = if large_offsets_needed {
                 if entry.pack_offset > LARGE_OFFSET_THRESHOLD {
@@ -215,7 +211,7 @@ pub mod offsets {
                     .try_into()
                     .expect("without large offsets, pack-offset fits u32")
             };
-            out.write_u32::<BigEndian>(offset)?;
+            out.write_all(&offset.to_be_bytes())?;
         }
         Ok(())
     }
@@ -231,8 +227,6 @@ pub mod offsets {
 pub mod large_offsets {
     use std::ops::Range;
 
-    use byteorder::{BigEndian, WriteBytesExt};
-
     use crate::{index::write::encode::LARGE_OFFSET_THRESHOLD, multi_index};
 
     /// The id uniquely identifying the large offsets table (with 64 bit offsets)
@@ -267,7 +261,7 @@ pub mod large_offsets {
             .iter()
             .filter_map(|e| (e.pack_offset > LARGE_OFFSET_THRESHOLD).then(|| e.pack_offset))
         {
-            out.write_u64::<BigEndian>(offset)?;
+            out.write_all(&offset.to_be_bytes())?;
             num_large_offsets = num_large_offsets
                 .checked_sub(1)
                 .expect("BUG: wrote more offsets the previously found");

git-pack/src/multi_index/chunk.rs

@@ -1,7 +1,5 @@
 use std::{convert::TryFrom, path::Path};
 
-use byteorder::{BigEndian, ByteOrder};
-
 use crate::multi_index::{chunk, File, Version};
 
 mod error {
@@ -90,7 +88,7 @@ impl TryFrom<&Path> for File {
             let (_num_base_files, data) = data.split_at(1); // TODO: handle base files once it's clear what this does
 
             let (num_indices, _) = data.split_at(4);
-            let num_indices = BigEndian::read_u32(num_indices);
+            let num_indices = crate::read_u32(num_indices);
 
             (version, object_hash, num_chunks, num_indices)
         };