Merge pull request #50 from Byron/edward-shen/odb-zlib-ng

Use zlib-ng by default

Author: Byron

Cargo.lock

@@ -57,7 +57,6 @@ prodash-render-line-termion = ["prodash-render-line", "prodash/render-line-termi
 prodash-render-tui = ["prodash/render-tui"]
 prodash-render-line = ["prodash/render-line"]
 
-
 [dependencies]
 anyhow = "1.0.31"
 

Cargo.toml

@@ -12,6 +12,7 @@ include = ["src/**/*"]
 doctest = false
 
 [features]
+default = ["flate2/zlib-ng-compat"]
 serde1 = ["serde", "git-object/serde1"]
 
 [package.metadata.docs.rs]
@@ -22,7 +23,7 @@ git-object = { version = "^0.6.0", path = "../git-object" }
 git-hash = { version = "^0.1.0", path = "../git-hash" }
 git-features = { version = "^0.11.0", path = "../git-features" }
 
-miniz_oxide = "0.4.1"
+flate2 = { version = "1.0.17", default-features = false }
 smallvec = "1.3.0"
 filebuffer = "0.4.0"
 byteorder = "1.2.3"

git-odb/Cargo.toml

@@ -73,7 +73,7 @@ impl Db {
             })?;
             (
                 inflate
-                    .once(&compressed[..bytes_read], &mut decompressed[..], true)
+                    .once(&compressed[..bytes_read], &mut decompressed[..])
                     .map_err(|e| Error::DecompressFile {
                         source: e,
                         path: path.to_owned(),

git-odb/src/loose/db/locate.rs

@@ -1,7 +1,6 @@
-use super::stream;
-use crate::{loose, zlib};
+use crate::loose;
+use flate2::Decompress;
 use git_object as object;
-use miniz_oxide::inflate::decompress_to_vec_zlib;
 use object::borrowed;
 use smallvec::SmallVec;
 use std::{io::Read, path::PathBuf};
@@ -11,7 +10,7 @@ use std::{io::Read, path::PathBuf};
 #[allow(missing_docs)]
 pub enum Error {
     #[error("decompression of object data failed")]
-    Decompress(#[from] zlib::Error),
+    Decompress(#[from] flate2::DecompressError),
     #[error(transparent)]
     Parse(#[from] borrowed::Error),
     #[error("Could not {action} data at '{path}'")]
@@ -41,9 +40,9 @@ impl loose::Object {
     ///
     /// **Note**: This is most useful for big blobs as these won't be read into memory in full. Use [`decode()`][loose::Object::decode()] for
     /// Trees, Tags and Commits instead for convenient access to their payload.
-    pub fn stream(&mut self) -> Result<stream::Reader<'_>, Error> {
+    pub fn stream(&mut self) -> Result<loose::object::stream::Reader<'_>, Error> {
         match &self.path {
-            Some(path) => Ok(stream::Reader::from_file(
+            Some(path) => Ok(loose::object::stream::Reader::from_file(
                 self.header_size,
                 std::fs::File::open(path).map_err(|source| Error::Io {
                     source,
@@ -53,7 +52,7 @@ impl loose::Object {
             )),
             None => {
                 self.decompress_all()?;
-                Ok(stream::Reader::from_data(
+                Ok(loose::object::stream::Reader::from_data(
                     self.header_size,
                     &self.decompressed_data.as_slice(),
                 ))
@@ -94,7 +93,9 @@ impl loose::Object {
             })?;
             self.compressed_data = SmallVec::from(buf);
         }
-        self.decompressed_data = SmallVec::from(decompress_to_vec_zlib(&self.compressed_data[..]).unwrap());
+        let mut vec = Vec::with_capacity(std::cmp::min(self.compressed_data.len() * 2, usize::MAX));
+        Decompress::new(true).decompress_vec(&self.compressed_data[..], &mut vec, flate2::FlushDecompress::None)?;
+        self.decompressed_data = SmallVec::from(vec);
         self.compressed_data = Default::default();
         self.decompressed_data.shrink_to_fit();
         assert!(self.decompressed_data.len() == total_size);

git-odb/src/loose/object/decode.rs

@@ -130,7 +130,7 @@ impl File {
         assert!(offset < self.data.len(), "entry offset out of bounds");
 
         zlib::Inflate::default()
-            .once(&self.data[offset..], out, true)
+            .once(&self.data[offset..], out)
             .map_err(Into::into)
             .map(|(_, consumed_in, _)| consumed_in)
     }

git-odb/src/pack/data/decode.rs

@@ -1,7 +1,5 @@
-use crate::{
-    hash, pack,
-    zlib::stream::inflate::{Inflate, InflateReaderBoxed},
-};
+use crate::{hash, pack, zlib::stream::inflate::InflateReaderBoxed};
+use flate2::Decompress;
 use git_features::hash::Sha1;
 use git_object::owned;
 use std::{fs, io};
@@ -16,6 +14,8 @@ pub enum Error {
     PackParse(#[from] pack::data::parse::Error),
     #[error("pack checksum in trailer was {expected}, but actual checksum was {actual}")]
     ChecksumMismatch { expected: owned::Id, actual: owned::Id },
+    #[error("pack is incomplete: it was decompressed into {actual} bytes but {expected} bytes where expected.")]
+    IncompletePack { actual: u64, expected: u64 },
 }
 
 /// An item of the iteration produced by [`Iter`]
@@ -51,7 +51,7 @@ pub struct Entry {
 /// The iterator used as part of [Bundle::write_stream_to_directory(…)][pack::Bundle::write_stream_to_directory()].
 pub struct Iter<R> {
     read: R,
-    decompressor: Option<Box<Inflate>>,
+    decompressor: Option<Box<Decompress>>,
     offset: u64,
     had_error: bool,
     kind: pack::data::Version,
@@ -183,9 +183,12 @@ where
         .map_err(Error::from)?;
 
         // Decompress object to learn it's compressed bytes
-        let mut decompressor = self.decompressor.take().unwrap_or_default();
+        let mut decompressor = self
+            .decompressor
+            .take()
+            .unwrap_or_else(|| Box::new(Decompress::new(true)));
         let compressed_buf = self.compressed_buf.take().unwrap_or_else(|| Vec::with_capacity(4096));
-        decompressor.reset();
+        decompressor.reset(true);
         let mut decompressed_reader = InflateReaderBoxed {
             inner: read_and_pass_to(
                 &mut self.read,
@@ -199,14 +202,15 @@ where
         };
 
         let bytes_copied = io::copy(&mut decompressed_reader, &mut io::sink())?;
-        debug_assert_eq!(
-            bytes_copied, entry.decompressed_size,
-            "We should have decompressed {} bytes, but got {} instead",
-            entry.decompressed_size, bytes_copied
-        );
+        if bytes_copied != entry.decompressed_size {
+            return Err(Error::IncompletePack {
+                actual: bytes_copied,
+                expected: entry.decompressed_size,
+            });
+        }
 
         let pack_offset = self.offset;
-        let compressed_size = decompressed_reader.decompressor.total_in;
+        let compressed_size = decompressed_reader.decompressor.total_in();
         self.offset += entry.header_size() as u64 + compressed_size;
         self.decompressor = Some(decompressed_reader.decompressor);
 

git-odb/src/pack/data/iter.rs

@@ -105,7 +105,7 @@ fn decompress_all_at_once(b: &[u8], decompressed_len: usize) -> Result<Vec<u8>,
     let mut out = Vec::new();
     out.resize(decompressed_len, 0);
     zlib::Inflate::default()
-        .once(&b, &mut out, true)
+        .once(&b, &mut out)
         .map_err(|err| Error::ZlibInflate {
             source: err,
             message: "Failed to decompress entry",

git-odb/src/pack/tree/traverse/resolve.rs

@@ -1,63 +1,39 @@
-use miniz_oxide::{
-    inflate::core::DecompressorOxide,
-    inflate::{
-        core::inflate_flags::{
-            TINFL_FLAG_HAS_MORE_INPUT, TINFL_FLAG_PARSE_ZLIB_HEADER, TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF,
-        },
-        TINFLStatus,
-    },
-};
+use flate2::Decompress;
 
 #[derive(thiserror::Error, Debug)]
 pub enum Error {
     #[error("Could not write all bytes when decompressing content")]
     WriteInflated(#[from] std::io::Error),
     #[error("Could not decode zip stream, status was '{0:?}'")]
-    Inflate(miniz_oxide::inflate::TINFLStatus),
+    Inflate(#[from] flate2::DecompressError),
 }
 
 /// Decompress a few bytes of a zlib stream without allocation
 pub struct Inflate {
-    state: DecompressorOxide,
+    state: Decompress,
     pub is_done: bool,
 }
 
 impl Default for Inflate {
     fn default() -> Self {
         Inflate {
-            state: DecompressorOxide::default(),
+            state: Decompress::new(true),
             is_done: false,
         }
     }
 }
 
 impl Inflate {
-    /// Run the decompressor exactly once. Cannot be run mutliple times
-    pub fn once(
-        &mut self,
-        input: &[u8],
-        out: &mut [u8],
-        parse_header: bool,
-    ) -> Result<(TINFLStatus, usize, usize), Error> {
-        let (status, in_consumed, out_consumed) = miniz_oxide::inflate::core::decompress(
-            &mut self.state,
-            input,
-            out,
-            0,
-            if parse_header { TINFL_FLAG_PARSE_ZLIB_HEADER } else { 0 }
-                | TINFL_FLAG_HAS_MORE_INPUT
-                | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF,
-        );
-
-        use miniz_oxide::inflate::TINFLStatus::*;
-        match status {
-            Failed | FailedCannotMakeProgress | BadParam | Adler32Mismatch => return Err(Error::Inflate(status)),
-            HasMoreOutput | NeedsMoreInput => {}
-            Done => {
-                self.is_done = true;
-            }
-        };
-        Ok((status, in_consumed, out_consumed))
+    /// Run the decompressor exactly once. Cannot be run multiple times
+    pub fn once(&mut self, input: &[u8], out: &mut [u8]) -> Result<(flate2::Status, usize, usize), Error> {
+        let before_in = self.state.total_in();
+        let before_out = self.state.total_out();
+        let status = self.state.decompress(input, out, flate2::FlushDecompress::None)?;
+        Ok((
+            status,
+            (self.state.total_in() - before_in) as usize,
+            (self.state.total_out() - before_out) as usize,
+        ))
     }
 }