PR to get CI run

.github/workflows/ci.yml

@@ -4,9 +4,15 @@ on:
   push:
     branches:
       - master
+      - conda_feb2021
+      - conda_nov2021
+      - conda_may2022
   pull_request:
     branches:
       - master
+      - conda_feb2021
+      - conda_nov2021
+      - conda_may2022
   schedule:
     - cron: '0 18 * * *'
 
@@ -123,7 +129,7 @@ jobs:
           [ "${{runner.os}}" != "Windows" ] && export RUSTFLAGS="-C link-args=-Wl,-rpath,$CONDA_PREFIX/lib"
           [ "${{matrix.mpi}}" == "mpich" ] && [ "${{runner.os}}" == "Linux" ] && export MPICH_CC=$(which gcc)
           [ "${{matrix.mpi}}" == "openmpi" ] && [ "${{runner.os}}" == "Linux" ] && export OMPI_CC=$(which gcc)
-          cargo test -vv --features="$FEATURES"
+          cargo test -vv --features="$FEATURES" attribute_tests -- --test-threads 1 --nocapture
 
   static:
     name: static
@@ -154,6 +160,26 @@ jobs:
           cargo r --example chunking --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc
         if: matrix.rust != 'stable-gnu'
 
+  conda_dl:
+    runs-on: ${{matrix.os}}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {os: ubuntu, rust: stable}
+          - {os: windows, rust: stable-msvc}
+          - {os: windows, rust: stable-gnu}
+          - {os: macos, rust: stable}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with: {submodules: true}
+      - name: Install Rust (${{matrix.rust}})
+        uses: actions-rs/toolchain@v1
+        with: {toolchain: '${{matrix.rust}}', profile: minimal, override: true}
+      - name: Build and test all crates
+        run: cargo test --workspace -v --features conda --exclude hdf5-derive
+
   apt:
     name: apt
     runs-on: ubuntu-${{matrix.ubuntu}}

Cargo.toml

@@ -10,17 +10,15 @@ repository = "https://github.com/aldanor/hdf5-rust"
 homepage = "https://github.com/aldanor/hdf5-rust"
 build = "build.rs"
 edition = "2018"
+include = ["src/**/*", "Cargo.*", "LICENSE-*", "README.*", "CHANGELOG.md", "build.rs"]
 categories = ["science", "filesystem"]
 
 [features]
 default = []
 mpio = ["mpi-sys", "hdf5-sys/mpio"]
 lzf = ["lzf-sys", "errno"]
 blosc = ["blosc-sys"]
-# The features with version numbers such as 1.10.3, 1.12.0 are metafeatures
-# and is only available when the HDF5 library is at least this version.
-# Features have_direct and have_parallel are also metafeatures and dependent
-# on the HDF5 library which is linked against.
+conda = ["hdf5-sys/conda"] 
 
 [workspace]
 members = [".", "hdf5-types", "hdf5-derive", "hdf5-sys", "hdf5-src"]
@@ -30,8 +28,8 @@ default-members = [".", "hdf5-types", "hdf5-derive", "hdf5-sys"]
 bitflags = "1.2"
 lazy_static = "1.4"
 libc = "0.2"
-parking_lot = "0.11"
-ndarray = "0.15"
+parking_lot = ">=0.11,<0.13"
+ndarray = ">=0.15,<0.17"
 paste = "1.0"
 mpi-sys = { version = "0.1", optional = true }
 errno = { version = "0.2", optional = true }
@@ -46,7 +44,7 @@ cfg-if = "1.0"
 paste = "1.0"
 pretty_assertions = "1.0"
 rand = { version = "0.8", features = ["small_rng"] }
-regex = "1.3"
+regex = { version = "1.3", default-features = false, features = ["std"] }
 scopeguard = "1.0"
 tempfile = "3.2"
 

hdf5-derive/Cargo.toml

@@ -8,6 +8,7 @@ description = "Derive macro for HDF5 structs and enums."
 repository = "https://github.com/aldanor/hdf5-rust"
 homepage = "https://github.com/aldanor/hdf5-rust"
 edition = "2018"
+include = ["src/*.rs"]
 readme = "README.md"
 categories = ["procedural-macro-helpers"]
 

hdf5-sys/Cargo.toml

@@ -28,10 +28,16 @@ threadsafe = ["hdf5-src/threadsafe"]
 zlib = ["libz-sys", "hdf5-src/zlib"]
 static = ["hdf5-src"]
 deprecated = ["hdf5-src/deprecated"]
+conda = ["attohttpc", "sha2", "bzip2", "tar"]
 
 [build-dependencies]
-libloading = "0.7"
-regex = { version = "1.3", features = ["std"] }
+libloading = ">=0.7,<1"
+regex = { version = "1.3", default-features = false, features = ["std", "unicode-perl"] }
+sha2 = { version = ">=0.9, <0.11", optional = true }
+attohttpc = { version = ">=0.12, <1", default-features = false, features = ["compress", "tls-rustls"], optional = true }
+bzip2 = { version = ">=0.3.3, <0.5", optional = true }
+tar = { version = "*", optional = true }
+
 
 [target.'cfg(all(unix, not(target_os = "macos")))'.build-dependencies]
 pkg-config = "0.3"

hdf5-sys/build.rs

@@ -637,6 +637,14 @@ impl Config {
 }
 
 fn main() {
+    #[cfg(feature = "conda")]
+    if env::var("CARGO_FEATURE_CONDA").is_ok() {
+        println!("cargo:rerun-if-changed=build.rs");
+        println!("cargo:rerun-if-env-changed=CARGO_FEATURE_CONDA");
+        conda_dl::conda_static();
+        return;
+    }
+
     if feature_enabled("STATIC") && std::env::var_os("HDF5_DIR").is_none() {
         get_build_and_emit();
     } else {
@@ -680,3 +688,178 @@ fn get_build_and_emit() {
     let config = Config { header, inc_dir: "".into(), link_paths: Vec::new() };
     config.emit_cfg_flags();
 }
+
+/// Download HDF5 binary builds from Conda and statically link to them.
+#[cfg(feature = "conda")]
+mod conda_dl {
+    use super::*;
+
+    use std::io::{self, Read};
+    use std::path::PathBuf;
+    use std::time::Duration;
+
+    use bzip2::read::BzDecoder;
+    use sha2::Digest;
+    use tar::Archive;
+
+    #[cfg(target_os = "linux")]
+    mod conda {
+        pub const INC_PATH: &'static str = "include";
+        pub const LIB_PATH: &'static str = "lib";
+
+        pub const DLS: &[(&'static str, &'static str, &'static str)] = &[
+            (
+                "hdf5-1.12.1-h70be1eb_2.tar.bz2",
+                "https://anaconda.org/anaconda/hdf5/1.12.1/download/linux-64/hdf5-1.12.1-h70be1eb_2.tar.bz2",
+                "30a88eb6e50bb2201d06734f74d3f98624f3b7ff1371607f770bb71d5dd86c61",
+            ),
+            (
+                "zlib-1.2.13-h166bdaf_4.tar.bz2",
+                "https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-h166bdaf_4.tar.bz2",
+                "282ce274ebe6da1fbd52efbb61bd5a93dec0365b14d64566e6819d1691b75300",
+            ),
+        ];
+    }
+
+    #[cfg(all(target_os = "macos", target_arch = "x86_64"))]
+    mod conda {
+        pub const INC_PATH: &'static str = "include";
+        pub const LIB_PATH: &'static str = "lib";
+
+        pub const DLS: &[(&'static str, &'static str, &'static str)] = &[
+            (
+                "hdf5-1.12.0-h964e04d_0.tar.bz2",
+                "https://anaconda.org/anaconda/hdf5/1.12.0/download/osx-64/hdf5-1.12.0-h964e04d_0.tar.bz2",
+                "1d6e4058ab8ad0ea70dd755b62310a956f9a026ab6a3336ff4bd605c806d7c60",
+            ),
+            (
+                "zlib-1.2.11-h6c3fc93_1014.tar.bz2",
+                "https://anaconda.org/conda-forge/zlib/1.2.11/download/osx-64/zlib-1.2.11-h6c3fc93_1014.tar.bz2",
+                "825ad325db6febfcc5ec1051a57a50be5504613c737ef3be8ccf38ec4dd12b70",
+            ),
+        ];
+    }
+
+    #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
+    mod conda {
+        pub const INC_PATH: &'static str = "include";
+        pub const LIB_PATH: &'static str = "lib";
+
+        pub const DLS: &[(&'static str, &'static str, &'static str)] = &[
+            (
+                "hdf5-static-1.12.1-nompi_h18eddcc_102.tar.bz2",
+                "https://anaconda.org/conda-forge/hdf5-static/1.12.1/download/osx-arm64/hdf5-static-1.12.1-nompi_h18eddcc_102.tar.bz2",
+                "ee6e87b831eefdc5d7d5ec39705c94bbec55b0f1fc11058242fb18a462fb1484",
+            ),
+            (
+                "hdf5-1.12.1-nompi_had0e5e0_101.tar.bz2",
+                "https://anaconda.org/conda-forge/hdf5/1.12.1/download/osx-arm64/hdf5-1.12.1-nompi_had0e5e0_101.tar.bz2",
+                "479a943c508ea796b708d4eaaa6bf03b5168c877a454190d67a75f28a44a2461",
+            ),
+            (
+                "zlib-1.2.11-hee7b306_1013.tar.bz2",
+                "https://anaconda.org/conda-forge/zlib/1.2.11/download/osx-arm64/zlib-1.2.11-hee7b306_1013.tar.bz2",
+                "04cbcc43aaf9b1ba31eddb0a93adb1a025156542fd4ba2b7b66b4ba4f4126d50",
+            ),
+        ];
+    }
+
+    #[cfg(target_os = "windows")]
+    mod conda {
+        pub const INC_PATH: &'static str = "Library\\include";
+        pub const LIB_PATH: &'static str = "Library\\lib";
+
+        pub const DLS: &[(&'static str, &'static str, &'static str)] = &[
+            (
+                "hdf5-1.12.0-h1756f20_0.tar.bz2",
+                "https://anaconda.org/anaconda/hdf5/1.12.0/download/win-64/hdf5-1.12.0-h1756f20_0.tar.bz2",
+                "747997999fc56b5c878cc8ee224b486266ac1c77ddf8407529a91eb851abf3d7",
+            ),
+            (
+                "zlib-1.2.11-vc14h1cdd9ab_1.tar.bz2",
+                "https://repo.continuum.io/pkgs/main/win-64/zlib-1.2.11-vc14h1cdd9ab_1.tar.bz2",
+                "cf7f895c0ff7f238ed02182a89864386bfc198674dc280002ff4a47ae0993b0e",
+            ),
+        ];
+    }
+
+    fn download(uri: &str, filename: &str, out_dir: &Path) {
+        let out = PathBuf::from(out_dir.join(filename));
+
+        // Download the tarball.
+        let f = fs::File::create(&out).unwrap();
+        let writer = io::BufWriter::new(f);
+
+        let req = attohttpc::get(uri).read_timeout(Duration::new(90, 0));
+
+        let response = req.send().unwrap();
+
+        if !response.is_success() {
+            panic!("Unexpected response code {:?} for {}", response.status(), uri);
+        }
+
+        response.write_to(writer).unwrap();
+    }
+
+    fn calc_sha256(path: &Path) -> String {
+        let mut f = io::BufReader::new(fs::File::open(path).unwrap());
+        let mut buf = Vec::new();
+        f.read_to_end(&mut buf).unwrap();
+
+        let digest = sha2::Sha256::digest(&buf);
+        format!("{:x}", digest)
+    }
+
+    fn extract<P: AsRef<Path>, P2: AsRef<Path>>(archive_path: P, extract_to: P2) {
+        let file = fs::File::open(archive_path).unwrap();
+        let unzipped = BzDecoder::new(file);
+        let mut a = Archive::new(unzipped);
+        a.unpack(extract_to).unwrap();
+    }
+
+    pub fn conda_static() {
+        let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
+
+        for (archive, uri, sha256) in conda::DLS {
+            let archive_path = out_dir.join(archive);
+            if archive_path.exists() && calc_sha256(&archive_path) == *sha256 {
+                println!("Use existings archive");
+            } else {
+                println!("Download archive");
+                download(uri, archive, &out_dir);
+                extract(&archive_path, &out_dir);
+
+                let sum = calc_sha256(&archive_path);
+                if sum != *sha256 {
+                    panic!(
+                        "check sum of downloaded archive {} is incorrect: sha256sum={}",
+                        archive, sum
+                    );
+                }
+            }
+        }
+
+        println!("cargo:rustc-link-search={}", out_dir.join(conda::LIB_PATH).display());
+
+        #[cfg(target_os = "windows")]
+        {
+            println!("cargo:rustc-link-lib=static=zlibstatic");
+            println!("cargo:rustc-link-lib=static=libhdf5");
+        }
+
+        #[cfg(target_os = "linux")]
+        println!("cargo:rustc-link-lib=static=hdf5");
+
+        #[cfg(target_os = "macos")]
+        println!("cargo:rustc-link-lib=static=hdf5");
+
+        #[cfg(not(target_os = "windows"))]
+        println!("cargo:rustc-link-lib=static=z");
+
+        let inc_dir = out_dir.join(conda::INC_PATH);
+
+        let header = Header::parse(&inc_dir);
+        let cfg = Config { inc_dir, link_paths: Vec::new(), header };
+        cfg.emit_cfg_flags();
+    }
+}