Merge pull request #140 from mulimoen/feature/dcpl-2021

The DCPL branch

Author: mulimoen

.github/workflows/ci.yml

@@ -162,9 +162,6 @@ jobs:
         run: |
           [ "${{matrix.mpi}}" != "serial" ] && FEATURES=mpio
           cargo test -vv --features="$FEATURES"
-      - name: Test const generics
-        if: matrix.rust == 'nightly'
-        run: cargo test -p hdf5-types --features hdf5-types/const_generics
 
   msi:
     name: msi
@@ -209,6 +206,22 @@ jobs:
       - name: Build and test all crates
         run: cargo test -vv
 
+  msrv:
+    name: Minimal Supported Rust Version
+    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with: {submodules: true}
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with: {toolchain: 1.51, profile: minimal, override: true}
+      - name: Build and test all crates
+        run:
+          cargo test --workspace -vv --features=hdf5-sys/static --exclude=hdf5-derive
+
   wine:
     name: wine
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -2,6 +2,65 @@
 
 ## Unreleased
 
+### Added
+
+- Complete rewrite of `DatasetBuilder`; dataset creation API is now different and not
+  backwards-compatible (however, it integrates all of the new features and is more
+  flexible and powerful). It is now possible to create and write the datasets in one step.
+  Refer to the API docs for full reference.
+- New `Extents` type matching HDF5 extents types: null (no elements), scalar, simple (fixed
+  dimensionality); it is used to query and specify shapes of datasets. Extents objects
+  are convertible from numbers and also tuples, slices and vectors of indices - all of
+  which can be used whenever extents are required (e.g., when creating a new dataset).
+- New `Selection` type and the API around it closely matching HDF5 selection API - this
+  includes 'all' selection, point-wise selection and hyperslab selection (only 'regular'
+  hyperslabs are supported - that is, hyperslabs that can be represented as a single
+  multi-dimensional box some of whose dimensions may be infinite). Selection objects
+  are convertible from numbers and ranges and tuples and arrays of numbers and ranges;
+  one can also use `s!` macro from `ndarray` crate. Selections can be provided when
+  reading and writing slices.
+- LZF / Blosc filters have been added, they have to be enabled by `"lzf"` / `"blosc"`
+  cargo features and depend on `lzf-sys` / `blosc-sys` crates respectively. Blosc filter
+  is a meta-filter providing multi-threaded access to the best-in-class compression codecs
+  like Zstd and LZ4 and is recommended to use as a default when compression is critical.
+- New `Filter` type to unify all of the filters API; if LZF / Blosc filters are enabled,
+  this enum also contains the corresponding variants. It is now also possible to provide
+  user-defined filters with custom filter IDs and parameters.
+- Dataset creation property list (DCPL) API is now supported; this provides access to all
+  of the properties that can be specified at dataset creation time (e.g., layout, chunking,
+  fill values, external file linking, virtual maps, object time tracking, attribute
+  creation order, and a few other low-level settings).
+- As part of DCPL change, virtual dataset maps (VDS API in HDF5 1.10+) are now supported.
+- Link creation property list (LCPL) API is now also wrapped.
+- File creation property list (FCPL) API has been extended to include a few previously
+  missing properties (object time tracking, attribute creation order and few other
+  low-level settings).
+- Added `h5-alloc` feature to `hdf5-types` crate - uses the HDF5 allocator for
+  varlen types and dynamic values. This may be necessary on platforms where different
+  allocators may be used in different libraries (e.g. dynamic libraries on Windows),
+  or if `libhdf5` is compiled with the memchecker option enabled. This option is
+  force-enabled by default if using a dll version of the library on Windows.
+- New `DynValue` type which represents a dynamic self-describing HDF5 object that
+  also knows how to deallocate itself; it supports all of the HDF5 types including
+  compound types, strings and arrays.
+- Added methods to `Dataset`: `layout`, `dapl`, `access_plist`, `dcpl`, `create_plist`.
+  
+ ### Changed
+  
+- `Dataspace` type has been reworked and can be now constructed from an extents object 
+  and sliced with a selection object.
+- `Dataset::fill_value` now returns an object of the newly added `DynValue` type; this
+  object is self-describing and knows how to free itself.
+- Automatic chunking now uses a fill-from-back approach instead of the previously
+  used method which is used in `h5py`.
+- Removed `Filters` type (there's now `Filter` that represents a single filter).
+- `write_slice`, `read_slice`, `read_slice_1d`, `read_slice_2d` now take any object
+  convertible to `Selection` (instead of `SliceInfo`).
+- `Dataset::chunks` has been renamed to `Dataset::chunk`
+- Const generics support (MSRV 1.51): `hdf5-types` now uses const generics for array types,
+  allowing fixed-size arrays of arbitrary sizes.
+- The `ndarray` dependency has been updated to `0.15`.
+
 ## 0.7.1
 
 ### Added

Cargo.toml

@@ -14,6 +14,8 @@ edition = "2018"
 [features]
 default = []
 mpio = ["mpi-sys", "hdf5-sys/mpio"]
+lzf = ["lzf-sys", "errno"]
+blosc = ["blosc-sys"]
 
 [workspace]
 members = [".", "hdf5-types", "hdf5-derive", "hdf5-sys", "hdf5-src"]
@@ -24,21 +26,25 @@ bitflags = "1.2"
 lazy_static = "1.4"
 libc = "0.2"
 parking_lot = "0.11"
-ndarray = ">=0.13,<0.15"
+ndarray = "0.15"
 num-integer = "0.1"
 num-traits = "0.2"
 mpi-sys = { version = "0.1", optional = true }
+errno = { version = "0.2", optional = true }
 hdf5-sys = { path = "hdf5-sys", version = "0.7.1" }  # !V
 hdf5-types = { path = "hdf5-types", version = "0.7.1" }  # !V
 hdf5-derive = { path = "hdf5-derive", version = "0.7.1" }  # !V
+blosc-sys = { version = "0.1", package = "blosc-src", optional = true }
+lzf-sys = { version = "0.1", optional = true }
+cfg-if = "1.0"
 
 [dev-dependencies]
 paste = "1.0"
-pretty_assertions = "0.6"
+pretty_assertions = "0.7"
 rand = { version = "0.8", features = ["small_rng"] }
 regex = "1.3"
 scopeguard = "1.0"
 tempfile = "3.2"
 
 [package.metadata.docs.rs]
-features = ["hdf5-sys/static", "hdf5-sys/zlib"]
+features = ["hdf5-sys/static", "hdf5-sys/zlib", "blosc", "lzf"]

README.md

@@ -49,10 +49,10 @@ fn main() -> hdf5::Result<()> {
     {
         // write
         let file = hdf5::File::create("pixels.h5")?;
-        let colors = file.new_dataset::<Color>().create("colors", 2)?;
+        let colors = file.new_dataset::<Color>().shape(2).create("colors")?;
         colors.write(&[RED, BLUE])?;
         let group = file.create_group("dir")?;
-        let pixels = group.new_dataset::<Pixel>().create("pixels", (2, 2))?;
+        let pixels = group.new_dataset::<Pixel>().shape((2, 2)).create("pixels")?;
         pixels.write(&arr2(&[
             [Pixel { xy: (1, 2), color: RED }, Pixel { xy: (3, 4), color: BLUE }],
             [Pixel { xy: (5, 6), color: GREEN }, Pixel { xy: (7, 8), color: RED }],

examples/simple.rs

@@ -23,10 +23,10 @@ fn main() -> hdf5::Result<()> {
     {
         // write
         let file = hdf5::File::create("pixels.h5")?;
-        let colors = file.new_dataset::<Color>().create("colors", 2)?;
+        let colors = file.new_dataset::<Color>().shape(2).create("colors")?;
         colors.write(&[RED, BLUE])?;
         let group = file.create_group("dir")?;
-        let pixels = group.new_dataset::<Pixel>().create("pixels", (2, 2))?;
+        let pixels = group.new_dataset::<Pixel>().shape((2, 2)).create("pixels")?;
         pixels.write(&arr2(&[
             [Pixel { xy: (1, 2), color: RED }, Pixel { xy: (3, 4), color: BLUE }],
             [Pixel { xy: (5, 6), color: GREEN }, Pixel { xy: (7, 8), color: RED }],

hdf5-sys/Cargo.toml

@@ -29,7 +29,7 @@ static = ["hdf5-src"]
 deprecated = ["hdf5-src/deprecated"]
 
 [build-dependencies]
-libloading = "0.6"
+libloading = "0.7"
 regex = { version = "1.3", features = ["std"] }
 
 [target.'cfg(all(unix, not(target_os = "macos")))'.build-dependencies]

hdf5-sys/build.rs

@@ -85,7 +85,7 @@ impl Display for RuntimeError {
 
 #[allow(non_snake_case, non_camel_case_types)]
 fn get_runtime_version_single<P: AsRef<Path>>(path: P) -> Result<Version, Box<dyn Error>> {
-    let lib = libloading::Library::new(path.as_ref())?;
+    let lib = unsafe { libloading::Library::new(path.as_ref()) }?;
 
     type H5open_t = unsafe extern "C" fn() -> c_int;
     let H5open = unsafe { lib.get::<H5open_t>(b"H5open")? };
@@ -124,28 +124,26 @@ fn validate_runtime_version(config: &Config) {
     }
     for link_path in &link_paths {
         if let Ok(paths) = fs::read_dir(link_path) {
-            for path in paths {
-                if let Ok(path) = path {
-                    let path = path.path();
-                    if let Some(filename) = path.file_name() {
-                        let filename = filename.to_str().unwrap_or("");
-                        if path.is_file() && libfiles.contains(&filename) {
-                            println!("Attempting to load: {:?}", path);
-                            match get_runtime_version_single(&path) {
-                                Ok(version) => {
-                                    println!("    => runtime version = {:?}", version);
-                                    if version == config.header.version {
-                                        println!("HDF5 library runtime version matches headers.");
-                                        return;
-                                    }
-                                    panic!(
-                                        "Invalid HDF5 runtime version (expected: {:?}).",
-                                        config.header.version
-                                    );
-                                }
-                                Err(err) => {
-                                    println!("    => {}", err);
+            for path in paths.flatten() {
+                let path = path.path();
+                if let Some(filename) = path.file_name() {
+                    let filename = filename.to_str().unwrap_or("");
+                    if path.is_file() && libfiles.contains(&filename) {
+                        println!("Attempting to load: {:?}", path);
+                        match get_runtime_version_single(&path) {
+                            Ok(version) => {
+                                println!("    => runtime version = {:?}", version);
+                                if version == config.header.version {
+                                    println!("HDF5 library runtime version matches headers.");
+                                    return;
                                 }
+                                panic!(
+                                    "Invalid HDF5 runtime version (expected: {:?}).",
+                                    config.header.version
+                                );
+                            }
+                            Err(err) => {
+                                println!("    => {}", err);
                             }
                         }
                     }

hdf5-sys/src/lib.rs

@@ -2,6 +2,7 @@
 #![cfg_attr(feature = "cargo-clippy", allow(clippy::unreadable_literal))]
 #![cfg_attr(feature = "cargo-clippy", allow(clippy::missing_safety_doc))]
 #![cfg_attr(feature = "cargo-clippy", allow(clippy::cognitive_complexity))]
+#![cfg_attr(feature = "cargo-clippy", allow(clippy::upper_case_acronyms))]
 
 macro_rules! extern_static {
     ($dest:ident, $src:ident) => {

hdf5-types/Cargo.toml

@@ -8,13 +8,17 @@ description = "Native Rust equivalents of HDF5 types."
 repository = "https://github.com/aldanor/hdf5-rust"
 homepage = "https://github.com/aldanor/hdf5-rust"
 edition = "2018"
+build = "build.rs"
 
 [features]
-const_generics = []
+h5-alloc = []
 
 [dependencies]
 ascii = "1.0"
 libc = "0.2"
+hdf5-sys = { version = "0.7.1", path = "../hdf5-sys" }  # !V
+cfg-if = "1.0.0"
 
 [dev-dependencies]
 quickcheck = { version = "1.0", default-features = false }
+unindent = "0.1"

hdf5-types/build.rs

@@ -0,0 +1,6 @@
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    if std::env::var_os("DEP_HDF5_MSVC_DLL_INDIRECTION").is_some() {
+        println!("cargo:rustc-cfg=windows_dll");
+    }
+}