diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88a72e462..c84014fae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,9 +4,15 @@ on: push: branches: - master + - conda_feb2021 + - conda_nov2021 + - conda_may2022 pull_request: branches: - master + - conda_feb2021 + - conda_nov2021 + - conda_may2022 schedule: - cron: '0 18 * * *' @@ -123,7 +129,7 @@ jobs: [ "${{runner.os}}" != "Windows" ] && export RUSTFLAGS="-C link-args=-Wl,-rpath,$CONDA_PREFIX/lib" [ "${{matrix.mpi}}" == "mpich" ] && [ "${{runner.os}}" == "Linux" ] && export MPICH_CC=$(which gcc) [ "${{matrix.mpi}}" == "openmpi" ] && [ "${{runner.os}}" == "Linux" ] && export OMPI_CC=$(which gcc) - cargo test -vv --features="$FEATURES" + cargo test -vv --features="$FEATURES" attribute_tests -- --test-threads 1 --nocapture static: name: static @@ -154,6 +160,26 @@ jobs: cargo r --example chunking --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc if: matrix.rust != 'stable-gnu' + conda_dl: + runs-on: ${{matrix.os}}-latest + strategy: + fail-fast: false + matrix: + include: + - {os: ubuntu, rust: stable} + - {os: windows, rust: stable-msvc} + - {os: windows, rust: stable-gnu} + - {os: macos, rust: stable} + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: {submodules: true} + - name: Install Rust (${{matrix.rust}}) + uses: actions-rs/toolchain@v1 + with: {toolchain: '${{matrix.rust}}', profile: minimal, override: true} + - name: Build and test all crates + run: cargo test --workspace -v --features conda --exclude hdf5-derive + apt: name: apt runs-on: ubuntu-${{matrix.ubuntu}} diff --git a/Cargo.toml b/Cargo.toml index af3f1e8bc..7215812c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ repository = "https://github.com/aldanor/hdf5-rust" homepage = "https://github.com/aldanor/hdf5-rust" build = "build.rs" edition = "2018" +include = ["src/**/*", "Cargo.*", "LICENSE-*", "README.*", "CHANGELOG.md", "build.rs"] categories = ["science", "filesystem"] [features] @@ -17,10 +18,7 @@ default = [] mpio = ["mpi-sys", "hdf5-sys/mpio"] lzf = ["lzf-sys", "errno"] blosc = ["blosc-sys"] -# The features with version numbers such as 1.10.3, 1.12.0 are metafeatures -# and is only available when the HDF5 library is at least this version. -# Features have_direct and have_parallel are also metafeatures and dependent -# on the HDF5 library which is linked against. +conda = ["hdf5-sys/conda"] [workspace] members = [".", "hdf5-types", "hdf5-derive", "hdf5-sys", "hdf5-src"] @@ -30,8 +28,8 @@ default-members = [".", "hdf5-types", "hdf5-derive", "hdf5-sys"] bitflags = "1.2" lazy_static = "1.4" libc = "0.2" -parking_lot = "0.11" -ndarray = "0.15" +parking_lot = ">=0.11,<0.13" +ndarray = ">=0.15,<0.17" paste = "1.0" mpi-sys = { version = "0.1", optional = true } errno = { version = "0.2", optional = true } @@ -46,7 +44,7 @@ cfg-if = "1.0" paste = "1.0" pretty_assertions = "1.0" rand = { version = "0.8", features = ["small_rng"] } -regex = "1.3" +regex = { version = "1.3", default-features = false, features = ["std"] } scopeguard = "1.0" tempfile = "3.2" diff --git a/hdf5-derive/Cargo.toml b/hdf5-derive/Cargo.toml index d39558f5d..9f6173db4 100644 --- a/hdf5-derive/Cargo.toml +++ b/hdf5-derive/Cargo.toml @@ -8,6 +8,7 @@ description = "Derive macro for HDF5 structs and enums." repository = "https://github.com/aldanor/hdf5-rust" homepage = "https://github.com/aldanor/hdf5-rust" edition = "2018" +include = ["src/*.rs"] readme = "README.md" categories = ["procedural-macro-helpers"] diff --git a/hdf5-sys/Cargo.toml b/hdf5-sys/Cargo.toml index a318eb70a..44e7011d1 100644 --- a/hdf5-sys/Cargo.toml +++ b/hdf5-sys/Cargo.toml @@ -28,10 +28,16 @@ threadsafe = ["hdf5-src/threadsafe"] zlib = ["libz-sys", "hdf5-src/zlib"] static = ["hdf5-src"] deprecated = ["hdf5-src/deprecated"] +conda = ["attohttpc", "sha2", "bzip2", "tar"] [build-dependencies] -libloading = "0.7" -regex = { version = "1.3", features = ["std"] } +libloading = ">=0.7,<1" +regex = { version = "1.3", default-features = false, features = ["std", "unicode-perl"] } +sha2 = { version = ">=0.9, <0.11", optional = true } +attohttpc = { version = ">=0.12, <1", default-features = false, features = ["compress", "tls-rustls"], optional = true } +bzip2 = { version = ">=0.3.3, <0.5", optional = true } +tar = { version = "*", optional = true } + [target.'cfg(all(unix, not(target_os = "macos")))'.build-dependencies] pkg-config = "0.3" diff --git a/hdf5-sys/build.rs b/hdf5-sys/build.rs index 13d73b189..ba7bd1353 100644 --- a/hdf5-sys/build.rs +++ b/hdf5-sys/build.rs @@ -637,6 +637,14 @@ impl Config { } fn main() { + #[cfg(feature = "conda")] + if env::var("CARGO_FEATURE_CONDA").is_ok() { + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-env-changed=CARGO_FEATURE_CONDA"); + conda_dl::conda_static(); + return; + } + if feature_enabled("STATIC") && std::env::var_os("HDF5_DIR").is_none() { get_build_and_emit(); } else { @@ -680,3 +688,178 @@ fn get_build_and_emit() { let config = Config { header, inc_dir: "".into(), link_paths: Vec::new() }; config.emit_cfg_flags(); } + +/// Download HDF5 binary builds from Conda and statically link to them. +#[cfg(feature = "conda")] +mod conda_dl { + use super::*; + + use std::io::{self, Read}; + use std::path::PathBuf; + use std::time::Duration; + + use bzip2::read::BzDecoder; + use sha2::Digest; + use tar::Archive; + + #[cfg(target_os = "linux")] + mod conda { + pub const INC_PATH: &'static str = "include"; + pub const LIB_PATH: &'static str = "lib"; + + pub const DLS: &[(&'static str, &'static str, &'static str)] = &[ + ( + "hdf5-1.12.1-h70be1eb_2.tar.bz2", + "https://anaconda.org/anaconda/hdf5/1.12.1/download/linux-64/hdf5-1.12.1-h70be1eb_2.tar.bz2", + "30a88eb6e50bb2201d06734f74d3f98624f3b7ff1371607f770bb71d5dd86c61", + ), + ( + "zlib-1.2.13-h166bdaf_4.tar.bz2", + "https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-h166bdaf_4.tar.bz2", + "282ce274ebe6da1fbd52efbb61bd5a93dec0365b14d64566e6819d1691b75300", + ), + ]; + } + + #[cfg(all(target_os = "macos", target_arch = "x86_64"))] + mod conda { + pub const INC_PATH: &'static str = "include"; + pub const LIB_PATH: &'static str = "lib"; + + pub const DLS: &[(&'static str, &'static str, &'static str)] = &[ + ( + "hdf5-1.12.0-h964e04d_0.tar.bz2", + "https://anaconda.org/anaconda/hdf5/1.12.0/download/osx-64/hdf5-1.12.0-h964e04d_0.tar.bz2", + "1d6e4058ab8ad0ea70dd755b62310a956f9a026ab6a3336ff4bd605c806d7c60", + ), + ( + "zlib-1.2.11-h6c3fc93_1014.tar.bz2", + "https://anaconda.org/conda-forge/zlib/1.2.11/download/osx-64/zlib-1.2.11-h6c3fc93_1014.tar.bz2", + "825ad325db6febfcc5ec1051a57a50be5504613c737ef3be8ccf38ec4dd12b70", + ), + ]; + } + + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + mod conda { + pub const INC_PATH: &'static str = "include"; + pub const LIB_PATH: &'static str = "lib"; + + pub const DLS: &[(&'static str, &'static str, &'static str)] = &[ + ( + "hdf5-static-1.12.1-nompi_h18eddcc_102.tar.bz2", + "https://anaconda.org/conda-forge/hdf5-static/1.12.1/download/osx-arm64/hdf5-static-1.12.1-nompi_h18eddcc_102.tar.bz2", + "ee6e87b831eefdc5d7d5ec39705c94bbec55b0f1fc11058242fb18a462fb1484", + ), + ( + "hdf5-1.12.1-nompi_had0e5e0_101.tar.bz2", + "https://anaconda.org/conda-forge/hdf5/1.12.1/download/osx-arm64/hdf5-1.12.1-nompi_had0e5e0_101.tar.bz2", + "479a943c508ea796b708d4eaaa6bf03b5168c877a454190d67a75f28a44a2461", + ), + ( + "zlib-1.2.11-hee7b306_1013.tar.bz2", + "https://anaconda.org/conda-forge/zlib/1.2.11/download/osx-arm64/zlib-1.2.11-hee7b306_1013.tar.bz2", + "04cbcc43aaf9b1ba31eddb0a93adb1a025156542fd4ba2b7b66b4ba4f4126d50", + ), + ]; + } + + #[cfg(target_os = "windows")] + mod conda { + pub const INC_PATH: &'static str = "Library\\include"; + pub const LIB_PATH: &'static str = "Library\\lib"; + + pub const DLS: &[(&'static str, &'static str, &'static str)] = &[ + ( + "hdf5-1.12.0-h1756f20_0.tar.bz2", + "https://anaconda.org/anaconda/hdf5/1.12.0/download/win-64/hdf5-1.12.0-h1756f20_0.tar.bz2", + "747997999fc56b5c878cc8ee224b486266ac1c77ddf8407529a91eb851abf3d7", + ), + ( + "zlib-1.2.11-vc14h1cdd9ab_1.tar.bz2", + "https://repo.continuum.io/pkgs/main/win-64/zlib-1.2.11-vc14h1cdd9ab_1.tar.bz2", + "cf7f895c0ff7f238ed02182a89864386bfc198674dc280002ff4a47ae0993b0e", + ), + ]; + } + + fn download(uri: &str, filename: &str, out_dir: &Path) { + let out = PathBuf::from(out_dir.join(filename)); + + // Download the tarball. + let f = fs::File::create(&out).unwrap(); + let writer = io::BufWriter::new(f); + + let req = attohttpc::get(uri).read_timeout(Duration::new(90, 0)); + + let response = req.send().unwrap(); + + if !response.is_success() { + panic!("Unexpected response code {:?} for {}", response.status(), uri); + } + + response.write_to(writer).unwrap(); + } + + fn calc_sha256(path: &Path) -> String { + let mut f = io::BufReader::new(fs::File::open(path).unwrap()); + let mut buf = Vec::new(); + f.read_to_end(&mut buf).unwrap(); + + let digest = sha2::Sha256::digest(&buf); + format!("{:x}", digest) + } + + fn extract, P2: AsRef>(archive_path: P, extract_to: P2) { + let file = fs::File::open(archive_path).unwrap(); + let unzipped = BzDecoder::new(file); + let mut a = Archive::new(unzipped); + a.unpack(extract_to).unwrap(); + } + + pub fn conda_static() { + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + + for (archive, uri, sha256) in conda::DLS { + let archive_path = out_dir.join(archive); + if archive_path.exists() && calc_sha256(&archive_path) == *sha256 { + println!("Use existings archive"); + } else { + println!("Download archive"); + download(uri, archive, &out_dir); + extract(&archive_path, &out_dir); + + let sum = calc_sha256(&archive_path); + if sum != *sha256 { + panic!( + "check sum of downloaded archive {} is incorrect: sha256sum={}", + archive, sum + ); + } + } + } + + println!("cargo:rustc-link-search={}", out_dir.join(conda::LIB_PATH).display()); + + #[cfg(target_os = "windows")] + { + println!("cargo:rustc-link-lib=static=zlibstatic"); + println!("cargo:rustc-link-lib=static=libhdf5"); + } + + #[cfg(target_os = "linux")] + println!("cargo:rustc-link-lib=static=hdf5"); + + #[cfg(target_os = "macos")] + println!("cargo:rustc-link-lib=static=hdf5"); + + #[cfg(not(target_os = "windows"))] + println!("cargo:rustc-link-lib=static=z"); + + let inc_dir = out_dir.join(conda::INC_PATH); + + let header = Header::parse(&inc_dir); + let cfg = Config { inc_dir, link_paths: Vec::new(), header }; + cfg.emit_cfg_flags(); + } +}