Stop installing setuptools and wheel (#243)

Currently the buildpack performs a system site-packages install of not
only pip, but also setuptools and wheel. This has historically been
necessary for pip to be able to build source distributions (sdists) for
packages that don't ship with compatible wheels.

However:
- Thanks to PEP 518, packages can now (and many already do) specify an
  explicit build backend using `[build-system]` in their
  `pyproject.toml`. The dependencies specified in that config (such as
  setuptools and wheel) will be installed by pip into an isolated and
  ephemeral build environment as part of the source distribution build
  process. Such packages therefore don't need/use globally installed
  setuptools/wheel versions.
- As of pip v22.1, pip will now default to the isolated build environment
  mode (along with a fallback legacy setuptools build backend), if the
  setuptools package isn't installed globally. This means that packages
  that haven't yet migrated to a PEP 518 `pyproject.toml` build backend
  config can still build even if setuptools isn't installed globally.

There are a small number of rarely used packages in the wild that
aren't compatible with build isolation mode, however, these typically
require more build dependencies than just setuptools, which means they
wouldn't have worked with this buildpack anyway.

As such, it's no longer necessary for us to install setuptools and wheel
globally. This matches the behaviour of the `venv` and `ensurepip` modules
in Python 3.12+, where setuptools and wheel installation has also been
removed. And it also matches the default behaviour of Poetry too, whose
`install --sync` command removes any implicitly installed packages in the
current environment (other than pip).

See:
https://peps.python.org/pep-0518/
https://pip.pypa.io/en/stable/reference/build-system/
https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/#build-isolation
pypa/pip#10717
python/cpython#101039
pypa/get-pip#218
astral-sh/uv#2252

GUS-W-16437776.

Author: edmorley

Diff for: CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Removed
+
+- Stopped explicitly installing setuptools and wheel. They will be automatically installed by pip into an isolated build environment if they are required for building a package. ([#243](https://github.com/heroku/buildpacks-python/pull/243))
+
 ## [0.13.0] - 2024-08-01
 
 ### Changed

Diff for: requirements/setuptools.txt

@@ -129,13 +129,13 @@ fn on_python_layer_error(error: PythonLayerError) {
         PythonLayerError::BootstrapPipCommand(error) => match error {
             StreamedCommandError::Io(io_error) => log_io_error(
                 "Unable to bootstrap pip",
-                "running the command to install pip, setuptools and wheel",
+                "running the command to install pip",
                 &io_error,
             ),
             StreamedCommandError::NonZeroExitStatus(exit_status) => log_error(
                 "Unable to bootstrap pip",
                 formatdoc! {"
-                    The command to install pip, setuptools and wheel did not exit successfully ({exit_status}).
+                    The command to install pip did not exit successfully ({exit_status}).
                     
                     See the log output above for more information.
                     

Diff for: requirements/wheel.txt

@@ -1,4 +1,4 @@
-use crate::packaging_tool_versions::PackagingToolVersions;
+use crate::packaging_tool_versions::PIP_VERSION;
 use crate::python_version::PythonVersion;
 use crate::{BuildpackError, PythonBuildpack};
 use libcnb::build::BuildContext;
@@ -17,14 +17,13 @@ pub(crate) fn prepare_pip_cache(
     context: &BuildContext<PythonBuildpack>,
     env: &mut Env,
     python_version: &PythonVersion,
-    packaging_tool_versions: &PackagingToolVersions,
 ) -> Result<(), libcnb::Error<BuildpackError>> {
     let new_metadata = PipCacheLayerMetadata {
         arch: context.target.arch.clone(),
         distro_name: context.target.distro_name.clone(),
         distro_version: context.target.distro_version.clone(),
         python_version: python_version.to_string(),
-        packaging_tool_versions: packaging_tool_versions.clone(),
+        pip_version: PIP_VERSION.to_string(),
     };
 
     let layer = context.cached_layer(
@@ -74,15 +73,15 @@ pub(crate) fn prepare_pip_cache(
     Ok(())
 }
 
-// Timestamp based cache invalidation isn't used here since the Python/pip/setuptools/wheel
-// versions will change often enough that it isn't worth the added complexity. Ideally pip
-// would support cleaning up its own cache: https://github.com/pypa/pip/issues/6956
+// Timestamp based cache invalidation isn't used here since the Python and pip versions will
+// change often enough that it isn't worth the added complexity. Ideally pip would support
+// cleaning up its own cache: https://github.com/pypa/pip/issues/6956
 #[derive(Deserialize, PartialEq, Serialize)]
 #[serde(deny_unknown_fields)]
 struct PipCacheLayerMetadata {
     arch: String,
     distro_name: String,
     distro_version: String,
     python_version: String,
-    packaging_tool_versions: PackagingToolVersions,
+    pip_version: String,
 }

Diff for: src/errors.rs

@@ -1,4 +1,4 @@
-use crate::packaging_tool_versions::PackagingToolVersions;
+use crate::packaging_tool_versions::PIP_VERSION;
 use crate::python_version::PythonVersion;
 use crate::utils::{self, DownloadUnpackArchiveError, StreamedCommandError};
 use crate::{BuildpackError, PythonBuildpack};
@@ -17,7 +17,7 @@ use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::{fs, io};
 
-/// Creates a layer containing the Python runtime and the packages `pip`, `setuptools` and `wheel`.
+/// Creates a layer containing the Python runtime and pip.
 //
 // We install both Python and the packaging tools into the same layer, since:
 //  - We don't want to mix buildpack/packaging dependencies with the app's own dependencies
@@ -31,25 +31,18 @@ use std::{fs, io};
 //  - This leaves just the system site-packages directory, which exists within the Python
 //    installation directory and Python does not support moving it elsewhere.
 //  - It matches what both local and official Docker image environments do.
-#[allow(clippy::too_many_lines)]
 pub(crate) fn install_python_and_packaging_tools(
     context: &BuildContext<PythonBuildpack>,
     env: &mut Env,
     python_version: &PythonVersion,
-    packaging_tool_versions: &PackagingToolVersions,
 ) -> Result<(), libcnb::Error<BuildpackError>> {
     let new_metadata = PythonLayerMetadata {
         arch: context.target.arch.clone(),
         distro_name: context.target.distro_name.clone(),
         distro_version: context.target.distro_version.clone(),
         python_version: python_version.to_string(),
-        packaging_tool_versions: packaging_tool_versions.clone(),
+        pip_version: PIP_VERSION.to_string(),
     };
-    let PackagingToolVersions {
-        pip_version,
-        setuptools_version,
-        wheel_version,
-    } = packaging_tool_versions;
 
     let layer = context.cached_layer(
         layer_name!("python"),
@@ -71,9 +64,8 @@ pub(crate) fn install_python_and_packaging_tools(
 
     match layer.state {
         LayerState::Restored { .. } => {
-            log_info(format!("Using cached Python {python_version}"));
             log_info(format!(
-                "Using cached pip {pip_version}, setuptools {setuptools_version} and wheel {wheel_version}"
+                "Using cached Python {python_version} and pip {PIP_VERSION}"
             ));
         }
         LayerState::Empty { ref cause } => {
@@ -117,9 +109,7 @@ pub(crate) fn install_python_and_packaging_tools(
         return Ok(());
     }
 
-    log_info(format!(
-        "Installing pip {pip_version}, setuptools {setuptools_version} and wheel {wheel_version}"
-    ));
+    log_info(format!("Installing pip {PIP_VERSION}"));
 
     let python_stdlib_dir = layer_path.join(format!(
         "lib/python{}.{}",
@@ -140,9 +130,7 @@ pub(crate) fn install_python_and_packaging_tools(
                 "--no-cache-dir",
                 "--no-input",
                 "--quiet",
-                format!("pip=={pip_version}").as_str(),
-                format!("setuptools=={setuptools_version}").as_str(),
-                format!("wheel=={wheel_version}").as_str(),
+                format!("pip=={PIP_VERSION}").as_str(),
             ])
             .current_dir(&context.app_dir)
             .env_clear()
@@ -170,7 +158,7 @@ struct PythonLayerMetadata {
     distro_name: String,
     distro_version: String,
     python_version: String,
-    packaging_tool_versions: PackagingToolVersions,
+    pip_version: String,
 }
 
 /// Compare cached layer metadata to the new layer metadata to determine if the cache should be
@@ -189,25 +177,15 @@ fn cache_invalidation_reasons(
         distro_name: cached_distro_name,
         distro_version: cached_distro_version,
         python_version: cached_python_version,
-        packaging_tool_versions:
-            PackagingToolVersions {
-                pip_version: cached_pip_version,
-                setuptools_version: cached_setuptools_version,
-                wheel_version: cached_wheel_version,
-            },
+        pip_version: cached_pip_version,
     } = cached_metadata;
 
     let PythonLayerMetadata {
         arch,
         distro_name,
         distro_version,
         python_version,
-        packaging_tool_versions:
-            PackagingToolVersions {
-                pip_version,
-                setuptools_version,
-                wheel_version,
-            },
+        pip_version,
     } = new_metadata;
 
     let mut reasons = Vec::new();
@@ -236,18 +214,6 @@ fn cache_invalidation_reasons(
         ));
     }
 
-    if cached_setuptools_version != setuptools_version {
-        reasons.push(format!(
-            "The setuptools version has changed from {cached_setuptools_version} to {setuptools_version}"
-        ));
-    }
-
-    if cached_wheel_version != wheel_version {
-        reasons.push(format!(
-            "The wheel version has changed from {cached_wheel_version} to {wheel_version}"
-        ));
-    }
-
     reasons
 }
 
@@ -423,11 +389,7 @@ mod tests {
             distro_name: "ubuntu".to_string(),
             distro_version: "22.04".to_string(),
             python_version: "3.11.0".to_string(),
-            packaging_tool_versions: PackagingToolVersions {
-                pip_version: "A.B.C".to_string(),
-                setuptools_version: "D.E.F".to_string(),
-                wheel_version: "G.H.I".to_string(),
-            },
+            pip_version: "A.B.C".to_string(),
         }
     }
 
@@ -462,11 +424,7 @@ mod tests {
             distro_name: "debian".to_string(),
             distro_version: "12".to_string(),
             python_version: "3.11.1".to_string(),
-            packaging_tool_versions: PackagingToolVersions {
-                pip_version: "A.B.C-new".to_string(),
-                setuptools_version: "D.E.F-new".to_string(),
-                wheel_version: "G.H.I-new".to_string(),
-            },
+            pip_version: "A.B.C-new".to_string(),
         };
         assert_eq!(
             cache_invalidation_reasons(&cached_metadata, &new_metadata),
@@ -475,8 +433,6 @@ mod tests {
                 "The OS has changed from ubuntu-22.04 to debian-12",
                 "The Python version has changed from 3.11.0 to 3.11.1",
                 "The pip version has changed from A.B.C to A.B.C-new",
-                "The setuptools version has changed from D.E.F to D.E.F-new",
-                "The wheel version has changed from G.H.I to G.H.I-new"
             ]
         );
     }

Diff for: src/layers/pip_cache.rs

@@ -13,7 +13,6 @@ use crate::layers::pip_dependencies::PipDependenciesLayerError;
 use crate::layers::python::{self, PythonLayerError};
 use crate::layers::{pip_cache, pip_dependencies};
 use crate::package_manager::{DeterminePackageManagerError, PackageManager};
-use crate::packaging_tool_versions::PackagingToolVersions;
 use crate::python_version::PythonVersionError;
 use libcnb::build::{BuildContext, BuildResult, BuildResultBuilder};
 use libcnb::detect::{DetectContext, DetectResult, DetectResultBuilder};
@@ -53,7 +52,6 @@ impl Buildpack for PythonBuildpack {
         log_header("Determining Python version");
         let python_version = python_version::determine_python_version(&context.app_dir)
             .map_err(BuildpackError::PythonVersion)?;
-        let packaging_tool_versions = PackagingToolVersions::default();
 
         // We inherit the current process's env vars, since we want `PATH` and `HOME` from the OS
         // to be set (so that later commands can find tools like Git in the base image), along
@@ -62,26 +60,16 @@ impl Buildpack for PythonBuildpack {
         // making sure that buildpack env vars take precedence in layers envs and command usage.
         let mut env = Env::from_current();
 
-        // Create the layer containing the Python runtime, and the packages `pip`, `setuptools` and `wheel`.
-        log_header("Installing Python and packaging tools");
-        python::install_python_and_packaging_tools(
-            &context,
-            &mut env,
-            &python_version,
-            &packaging_tool_versions,
-        )?;
+        // Create the layer containing the Python runtime and pip.
+        log_header("Installing Python and pip");
+        python::install_python_and_packaging_tools(&context, &mut env, &python_version)?;
 
         // Create the layers for the application dependencies and package manager cache.
         // In the future support will be added for package managers other than pip.
         let dependencies_layer_dir = match package_manager {
             PackageManager::Pip => {
                 log_header("Installing dependencies using pip");
-                pip_cache::prepare_pip_cache(
-                    &context,
-                    &mut env,
-                    &python_version,
-                    &packaging_tool_versions,
-                )?;
+                pip_cache::prepare_pip_cache(&context, &mut env, &python_version)?;
                 pip_dependencies::install_dependencies(&context, &mut env)?
             }
         };

Diff for: src/layers/python.rs

@@ -1,36 +1,10 @@
-use serde::{Deserialize, Serialize};
 use std::str;
 
 // We store these versions in requirements files so that Dependabot can update them.
 // Each file must contain a single package specifier in the format `package==1.2.3`,
 // from which we extract/validate the version substring at compile time.
-const PIP_VERSION: &str = extract_requirement_version(include_str!("../requirements/pip.txt"));
-const SETUPTOOLS_VERSION: &str =
-    extract_requirement_version(include_str!("../requirements/setuptools.txt"));
-const WHEEL_VERSION: &str = extract_requirement_version(include_str!("../requirements/wheel.txt"));
-
-/// The versions of various packaging tools used during the build.
-/// These are always installed, and are independent of the chosen package manager.
-/// Strings are used instead of a semver version, since these packages don't use
-/// semver, and we never introspect the version parts anyway.
-#[allow(clippy::struct_field_names)]
-#[derive(Clone, Deserialize, PartialEq, Serialize)]
-#[serde(deny_unknown_fields)]
-pub(crate) struct PackagingToolVersions {
-    pub(crate) pip_version: String,
-    pub(crate) setuptools_version: String,
-    pub(crate) wheel_version: String,
-}
-
-impl Default for PackagingToolVersions {
-    fn default() -> Self {
-        Self {
-            pip_version: PIP_VERSION.to_string(),
-            setuptools_version: SETUPTOOLS_VERSION.to_string(),
-            wheel_version: WHEEL_VERSION.to_string(),
-        }
-    }
-}
+pub(crate) const PIP_VERSION: &str =
+    extract_requirement_version(include_str!("../requirements/pip.txt"));
 
 // Extract the version substring from an exact-version package specifier (such as `foo==1.2.3`).
 // This function should only be used to extract the version constants from the buildpack's own