Merge pull request #1963 from joshtriplett/zlib-rs-default

Switch to zlib-rs by default and drop other zlib backends

Author: Byron

.github/workflows/ci.yml

@@ -67,7 +67,7 @@ jobs:
         continue-on-error: true
       - name: Check for unrecognized *-sys dependencies
         run: |
-          ! grep -qP '(?<!\blinux-raw)-sys\b' tree.txt
+          ! grep -qP '(?<!\b(linux-raw|libz-rs))-sys\b' tree.txt
       - name: Wrap cc1 (and cc1plus if present) to record calls
         run: |
           set -o noclobber  # Catch any collisions with existing entries in /usr/local.
@@ -402,7 +402,7 @@ jobs:
       - name: features of gix-features
         run: |
           set +x
-          for feature in progress parallel io-pipe crc32 zlib zlib-rust-backend cache-efficiency-debug; do
+          for feature in progress parallel io-pipe crc32 zlib cache-efficiency-debug; do
             (cd gix-features && cargo build --features "$feature" --target "$TARGET")
           done
       - name: crates with 'wasm' feature

.github/workflows/release.yml

@@ -231,7 +231,6 @@ jobs:
       - name: Install packages (Ubuntu)
         # Because openssl doesn't work on musl by default, we resort to max-pure.
         # And that won't need any dependency, so we can skip this or use `continue-on-error`.
-        # Once we want to support better zlib performance, we might have to re-add it.
         if: matrix.os == 'ubuntu-latest-disabled'
         run: |
           sudo apt-get update
@@ -541,8 +540,6 @@ jobs:
           msystem: MINGW${{ startsWith(matrix.target, 'i686-') && '32' || '64' }}
           pacboy: cc:p
           path-type: inherit
-      - name: Install prerequisites
-        run: vcpkg install zlib:x64-windows-static-md
       - name: 'Installation from crates.io: gitoxide'
         run: cargo +${{ matrix.rust }} install --target ${{ matrix.target }} --no-default-features --features max-pure --target-dir install-artifacts --debug --force gitoxide
         shell: msys2 {0}

Cargo.lock

@@ -41,14 +41,15 @@ max = ["max-control", "fast", "gitoxide-core-tools-query", "gitoxide-core-tools-
 
 ## Like `max`, but only Rust is allowed.
 ##
-## This is the most compatible build as it won't need a C compiler or C toolchains to build. It's also not the fastest as or the most feature-rich in terms of available
-## transports as it uses Rust's HTTP implementation.
+## This is the most compatible build as it won't need a C compiler or C toolchains to build. Thanks to zlib-rs, you don't have to trade off between compatibility and performance.
+##
+## This uses Rust's HTTP implementation.
 ##
 ## As fast as possible, with TUI progress, progress line rendering with auto-configuration, all transports available but less mature pure Rust HTTP implementation, all `ein` tools, CLI colors and local-time support, JSON output, regex support for rev-specs.
-max-pure = ["max-control", "gix-features/zlib-rust-backend", "http-client-reqwest", "gitoxide-core-blocking-client"]
+max-pure = ["max-control", "http-client-reqwest", "gitoxide-core-blocking-client"]
 
 ## Like `max`, but with more control for configuration. See the *Package Maintainers* headline for more information.
-max-control = ["tracing", "fast-safe", "pretty-cli", "gitoxide-core-tools", "prodash-render-line", "prodash-render-tui", "prodash/render-line-autoconfigure", "gix/revparse-regex"]
+max-control = ["tracing", "fast", "pretty-cli", "gitoxide-core-tools", "prodash-render-line", "prodash-render-tui", "prodash/render-line-autoconfigure", "gix/revparse-regex"]
 
 ## All the good stuff, with less fanciness for smaller binaries.
 ##
@@ -60,7 +61,7 @@ lean = ["fast", "tracing", "pretty-cli", "http-client-curl", "gitoxide-core-tool
 ## This build is essentially limited to local operations without any fanciness.
 ##
 ## Optimized for size, no parallelism thus much slower, progress line rendering.
-small = ["pretty-cli", "gix-features/zlib-rust-backend", "prodash-render-line", "is-terminal"]
+small = ["pretty-cli", "prodash-render-line", "is-terminal"]
 
 ## Like lean, but uses Rusts async implementations for networking.
 ##
@@ -74,37 +75,20 @@ small = ["pretty-cli", "gix-features/zlib-rust-backend", "prodash-render-line",
 lean-async = ["fast", "tracing", "pretty-cli", "gitoxide-core-tools", "gitoxide-core-tools-query", "gitoxide-core-tools-corpus", "gitoxide-core-async-client", "prodash-render-line"]
 
 #! ### Package Maintainers
-#! `*-control` features leave it to you to configure C libraries, involving choices for `zlib` and transport implementation.
+#! `*-control` features leave it to you to configure C libraries, involving choices for HTTP transport implementation.
 #!
 #! Additional features *can* be provided with `--features` and are handled by the [`gix-features` crate](https://docs.rs/gix-features/latest).
-#! If nothing else is specified, the Rust implementation is used. ! Note that only one feature of each section can be enabled at a time.
-#!
-#! * **zlib**
-#!     - `gix-features/zlib-ng`
-#!     - `gix-features/zlib-ng-compat`
-#!     - `gix-features/zlib-stock`
-#!     - `gix-features/zlib-rust-backend` (*default if no choice is made*)
-#! * **HTTP** - see the *Building Blocks for mutually exclusive networking* headline
-#!
-#! #### Examples
-#!
-#! * `cargo build --release --no-default-features --features max-control,gix-features/zlib-stock,gitoxide-core-blocking-client,http-client-curl`
-#!     - Create a build just like `max`, but using the stock `zlib` library instead of `zlib-ng`
-#! * `cargo build --release --no-default-features --features max-control,http-client-reqwest,gitoxide-core-blocking-client,gix-features/zlib-ng`
-#!     - Create a build just like `max-pure`, but with faster compression due to `zlib-ng`.
+#! Note that only one HTTP transport can be enabled at a time. See the *Building Blocks for mutually exclusive networking* headline.
 
 #! ### Building Blocks
 #! Typical combinations of features of our dependencies, some of which are referred to in the `gitoxide` crate's code for conditional compilation.
 
-## Makes the crate execute as fast as possible by supporting parallel computation of otherwise long-running functions
-## as well as a faster zlib backend.
+## Makes the crate execute as fast as possible by supporting parallel computation of otherwise long-running functions.
 ## If disabled, the binary will be visibly smaller.
 fast = ["gix/max-performance", "gix/comfort"]
 
-## Makes the crate execute as fast as possible by supporting parallel computation of otherwise long-running functions
-## as well as a faster zlib backend.
-## If disabled, the binary will be visibly smaller.
-fast-safe = ["gix/max-performance-safe", "gix/comfort"]
+## Deprecated: identical to `fast`, as the fastest zlib backend is now the pure-Rust zlib-rs.
+fast-safe = ["fast"]
 
 ## Enable tracing in `gitoxide-core`.
 tracing = ["dep:tracing-forest", "dep:tracing-subscriber", "dep:tracing", "gix-features/tracing", "gix-features/tracing-detail"]
@@ -201,8 +185,8 @@ gix-ref = { opt-level = 3 }
 gix-hash = { opt-level = 3 }
 gix-actor = { opt-level = 3 }
 gix-config = { opt-level = 3 }
-miniz_oxide = { opt-level = 3 }
 sha1-checked = { opt-level = 3 }
+zlib-rs = { opt-level = 3 }
 
 [profile.release]
 overflow-checks = false

Cargo.toml

@@ -418,8 +418,8 @@ Please take a look at the [`SHORTCOMINGS.md` file](https://github.com/GitoxideLa
 
 * **itertools** _(MIT Licensed)_
   * We use the `izip!` macro in code
-* **deflate2** _(MIT Licensed)_
-  * We use various abstractions to implement decompression and compression directly on top of the rather low-level `miniz_oxide` crate
+* **flate2** _(MIT Licensed)_
+  * We use the high-level `flate2` library to implement decompression and compression, which builds on the high-performance `zlib-rs` crate.
 
 ## 🙏 Special Thanks 🙏
 

README.md

@@ -4,15 +4,13 @@ ARG GITOXIDE_VERSION=0.36.0
 FROM rust:alpine AS bootstrap_os
   # hadolint ignore=DL3018
   RUN apk upgrade --update-cache --available \
-      && apk add --no-cache --virtual .runtime-gitoxide libressl zlib-ng \
-             libressl3.8-libcrypto
+      && apk add --no-cache --virtual .runtime-gitoxide libressl libressl3.8-libcrypto
 
 
 FROM bootstrap_os AS bootstrap_build_deps
   # hadolint ignore=DL3018
   RUN apk add --no-cache --virtual .rust-builder cmake gcc musl-dev make pkgconfig \
-      && apk add --no-cache --virtual .bootstrap-gitoxide libressl-dev zlib-ng \
-             libressl3.8-libcrypto
+      && apk add --no-cache --virtual .bootstrap-gitoxide libressl-dev libressl3.8-libcrypto
 
 
 FROM bootstrap_build_deps AS bootstrap_builder

etc/docker/Dockerfile.alpine

@@ -23,7 +23,8 @@ tar = ["dep:tar", "dep:gix-path"]
 tar_gz = ["tar", "dep:flate2"]
 
 ## Enable the `zip` archive format.
-zip = ["dep:zip"]
+## This also enables the `flate2` dependency in order to enable `zlib-rs` on it.
+zip = ["dep:flate2", "dep:zip"]
 
 
 [dependencies]
@@ -32,10 +33,8 @@ gix-object = { version = "^0.48.0", path = "../gix-object" }
 gix-path = { version = "^0.10.15", path = "../gix-path", optional = true }
 gix-date = { version = "^0.9.4", path = "../gix-date" }
 
-flate2 = { version = "1.1.1", optional = true }
-zip = { version = "2.6.1", optional = true, default-features = false, features = [
-    "deflate",
-] }
+flate2 = { version = "1.1.1", optional = true, default-features = false, features = ["zlib-rs"] }
+zip = { version = "2.6.1", optional = true, default-features = false, features = ["deflate"] }
 jiff = { version = "0.2.6", default-features = false, features = ["std"] }
 
 thiserror = "2.0.0"

gix-archive/Cargo.toml

@@ -54,30 +54,23 @@ io-pipe = ["dep:bytes"]
 ## provide a proven and fast `crc32` implementation.
 crc32 = ["dep:crc32fast"]
 
-#! ### Mutually Exclusive ZLIB
-
-## Enable the usage of zlib related utilities to compress or decompress data.
-## The base `zlib` feature uses the `flate2` Rust crate; the other mutually exclusive features select the `flate2 backend.
-## Note that a competitive Zlib implementation is critical to `gitoxide's` object database performance.
-## Enabling this without enabling one of the other features below will use a low-performance pure-Rust backend.
-zlib = ["dep:flate2", "flate2?/rust_backend", "dep:thiserror"]
-## Use the C-based zlib-ng backend, which can compress and decompress significantly faster.
-zlib-ng = ["zlib", "flate2?/zlib-ng"]
-## Use the high-performance rust-based zlib backend on par with zlib-ng.
-## As of zlib-rs 0.5.0 (used by flate2 1.1.1), this no longer exports C symbols
-## by default, so it doesn't conflict with any other zlib library that might be
-## loaded into the same address space.
-zlib-rs = ["zlib", "flate2?/zlib-rs"]
-## Use zlib-ng via its zlib-compat API. Useful if you already need zlib for C
-## code elsewhere in your dependencies. Otherwise, use zlib-ng.
-zlib-ng-compat = ["zlib", "flate2?/zlib-ng-compat"]
-## Use a slower C-based backend which can compress and decompress significantly faster than the rust version.
-## Unlike `zlib-ng-compat`, this allows using dynamic linking with system `zlib` libraries and doesn't require cmake.
-zlib-stock = ["zlib", "flate2?/zlib"]
-## Pure Rust backend, available for completeness even though it's the default
-## if neither of the above options are set. Low performance, but pure Rust, so it
-## may build in environments where other backends don't.
-zlib-rust-backend = ["zlib", "flate2?/rust_backend"]
+## Enable the usage of zlib-related utilities to compress or decompress data.
+## This enables the `flate2` crate, and always uses the high-performance `zlib-rs` backend.
+## Note that the various past features for selecting zlib backends are now deprecated and do nothing.
+zlib = ["dep:flate2", "dep:thiserror"]
+## Deprecated: gix always uses zlib-rs.
+zlib-ng = ["zlib"]
+## Deprecated: gix always uses zlib-rs now. As of zlib-rs 0.5.0 (used by flate2
+## 1.1.1), this no longer exports C symbols # by default, so it doesn't
+## conflict with any other zlib library that might be loaded into the same
+## address space.
+zlib-rs = ["zlib"]
+## Deprecated: gix always uses zlib-rs.
+zlib-ng-compat = ["zlib"]
+## Deprecated: gix always uses zlib-rs.
+zlib-stock = ["zlib"]
+## Deprecated: gix always uses zlib-rs.
+zlib-rust-backend = ["zlib"]
 
 #! ### Other
 
@@ -128,7 +121,7 @@ bytesize = { version = "2.0.1", optional = true }
 bytes = { version = "1.0.0", optional = true }
 
 # zlib module
-flate2 = { version = "1.1.1", optional = true, default-features = false }
+flate2 = { version = "1.1.1", optional = true, default-features = false, features = ["zlib-rs"] }
 thiserror = { version = "2.0.0", optional = true }
 
 once_cell = { version = "1.21.3", optional = true }

gix-features/Cargo.toml

@@ -370,11 +370,10 @@ impl Url {
 }
 
 /// This module contains extensions to the [Url] struct which are only intended to be used
-/// for testing code. Do not use this module in production! For all intends and purposes the APIs of
+/// for testing code. Do not use this module in production! For all intents and purposes, the APIs of
 /// all functions and types exposed by this module are considered unstable and are allowed to break
 /// even in patch releases!
 #[doc(hidden)]
-#[cfg(debug_assertions)]
 pub mod testing {
     use bstr::BString;
 

gix-url/src/lib.rs

@@ -220,14 +220,11 @@ blocking-http-transport-reqwest-native-tls = [
 #! ZIP might not compile on all platforms, so it depends on the end-user who compiles the application to chose these based on their needs.
 
 ## Activate features that maximize performance, like using threads, but leave everything else that might affect compatibility out to allow users more fine-grained
-## control over performance features like which `zlib*` implementation to use.
+## control.
 ## No C toolchain is involved.
 max-control = ["parallel", "pack-cache-lru-static", "pack-cache-lru-dynamic"]
 
-## Activate features that maximize performance, like usage of threads, `and access to caching in object databases, skipping the ones known to cause compile failures
-## on some platforms.
-## Note that this configuration still uses a pure Rust zlib implementation which isn't the fastest compared to its C-alternatives.
-## No C toolchain is involved.
+## Deprecated: gix always uses zlib-rs, so this is equivalent to `max-performance`
 max-performance-safe = ["max-control"]
 
 ## The tempfile registry uses a better implementation of a thread-safe hashmap, relying on an external crate.
@@ -247,25 +244,20 @@ pack-cache-lru-static = ["gix-pack/pack-cache-lru-static"]
 ## Provide a hash-map based LRU cache whose eviction is based a memory cap calculated from object data.
 pack-cache-lru-dynamic = ["gix-pack/pack-cache-lru-dynamic"]
 
-## Activate other features that maximize performance, like usage of threads, `zlib-ng` and access to caching in object databases.
-## Note that some platforms might suffer from compile failures, which is when `max-performance-safe` should be used.
-max-performance = ["max-performance-safe", "zlib-ng"]
+## Enable all features required for performance. Currently, this is equivalent to `max-control`, as gix always uses zlib-rs.
+max-performance = ["max-control"]
 
-## Use the C-based zlib-ng backend, which can compress and decompress significantly faster.
-## Note that this will cause duplicate symbol errors if the application also depends on `zlib` - use `zlib-ng-compat` in that case.
-zlib-ng = ["gix-features/zlib-ng"]
+## Deprecated: gix always uses zlib-rs.
+zlib-ng = ["gix-features/zlib"]
 
-## Use the high-performance rust-based zlib backend en par with zlib-ng.
-## Note that this will cause duplicate symbol errors if the application also depends on `zlib`, without remedy.
-zlib-rs = ["gix-features/zlib-rs"]
+## Deprecated: gix always uses zlib-rs.
+zlib-rs = ["gix-features/zlib"]
 
-## Use zlib-ng via its zlib-compat API. Useful if you already need zlib for C
-## code elsewhere in your dependencies. Otherwise, use `zlib-ng`.
-zlib-ng-compat = ["gix-features/zlib-ng-compat"]
+## Deprecated: gix always uses zlib-rs.
+zlib-ng-compat = ["gix-features/zlib"]
 
-## Use a slower C-based backend which can compress and decompress significantly faster than the rust version.
-## Unlike `zlib-ng-compat`, this allows using dynamic linking with system `zlib` libraries and doesn't require cmake.
-zlib-stock = ["gix-features/zlib-stock"]
+## Deprecated: gix always uses zlib-rs.
+zlib-stock = ["gix-features/zlib"]
 
 #! #### Other
 #!

gix/Cargo.toml

@@ -103,10 +103,6 @@ check:
     cargo check -p gix-features --features io-pipe
     cargo check -p gix-features --features crc32
     cargo check -p gix-features --features zlib
-    cargo check -p gix-features --features zlib,zlib-ng
-    cargo check -p gix-features --features zlib,zlib-ng-compat
-    cargo check -p gix-features --features zlib-stock
-    cargo check -p gix-features --features zlib,zlib-stock
     cargo check -p gix-features --features cache-efficiency-debug
     cd gix-commitgraph; \
       set -ex; \

justfile

@@ -551,15 +551,8 @@ title "gix commit-graph"
                   expect_run $WITH_FAILURE test -e "${PACK_FILE}".idx
                 }
 
-                if test "$kind" = "small" ; then
-                  suffix=miniz-oxide
-                elif test "$kind" = "max-pure"; then
-                  suffix=miniz-oxide-max
-                else
-                  suffix=zlib-ng
-                fi
                 it "creates all pack objects, but the broken ones" && {
-                  WITH_SNAPSHOT="$snapshot/broken-with-objects-dir-skip-checks-success-tree-$suffix" \
+                  WITH_SNAPSHOT="$snapshot/broken-with-objects-dir-skip-checks-success-tree" \
                   expect_run_sh $SUCCESSFULLY 'find . -type f | sort'
                 }
               )