diff --git a/CHANGELOG.md b/CHANGELOG.md index 5983ae2..ad64c2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added `reinstall` parameter to micropip.install to allow reinstalling + a package that is already installed + [#64](https://github.com/pyodide/micropip/pull/64) + ### Fixed - micropip now respects the `yanked` flag in the PyPI Simple API. diff --git a/micropip/install.py b/micropip/install.py new file mode 100644 index 0000000..b2927c9 --- /dev/null +++ b/micropip/install.py @@ -0,0 +1,125 @@ +import asyncio +import importlib +from pathlib import Path + +from ._compat import loadPackage, to_js +from ._vendored.packaging.src.packaging.markers import default_environment +from .constants import FAQ_URLS +from .logging import indent_log, setup_logging +from .transaction import Transaction +from .uninstall import uninstall_distributions + + +async def install( + requirements: str | list[str], + index_urls: list[str] | str, + keep_going: bool = False, + deps: bool = True, + credentials: str | None = None, + pre: bool = False, + *, + constraints: list[str] | None = None, + reinstall: bool = False, + verbose: bool | int | None = None, +) -> None: + with setup_logging().ctx_level(verbose) as logger: + + ctx = default_environment() + if isinstance(requirements, str): + requirements = [requirements] + + fetch_kwargs = {} + + if credentials: + fetch_kwargs["credentials"] = credentials + + # Note: getsitepackages is not available in a virtual environment... + # See https://github.com/pypa/virtualenv/issues/228 (issue is closed but + # problem is not fixed) + from site import getsitepackages + + wheel_base = Path(getsitepackages()[0]) + + transaction = Transaction( + ctx=ctx, # type: ignore[arg-type] + ctx_extras=[], + keep_going=keep_going, + deps=deps, + pre=pre, + fetch_kwargs=fetch_kwargs, + verbose=verbose, + index_urls=index_urls, + constraints=constraints, + reinstall=reinstall, + ) + await transaction.gather_requirements(requirements) + + if transaction.failed: + failed_requirements = ", ".join([f"'{req}'" for req in transaction.failed]) + raise ValueError( + f"Can't find a pure Python 3 wheel for: {failed_requirements}\n" + f"See: {FAQ_URLS['cant_find_wheel']}\n" + ) + + pyodide_packages, wheels = transaction.pyodide_packages, transaction.wheels + + packages_all = [pkg.name for pkg in wheels + pyodide_packages] + distributions = search_installed_packages(packages_all) + + with indent_log(): + uninstall_distributions(distributions, logger) + + logger.debug( + "Installing packages %r and wheels %r ", + transaction.pyodide_packages, + [w.filename for w in transaction.wheels], + ) + + if packages_all: + logger.info("Installing collected packages: %s", ", ".join(packages_all)) + + # Install PyPI packages + # detect whether the wheel metadata is from PyPI or from custom location + # wheel metadata from PyPI has SHA256 checksum digest. + await asyncio.gather(*(wheel.install(wheel_base) for wheel in wheels)) + + # Install built-in packages + if pyodide_packages: + # Note: branch never happens in out-of-browser testing because in + # that case REPODATA_PACKAGES is empty. + await asyncio.ensure_future( + loadPackage(to_js([name for [name, _, _] in pyodide_packages])) + ) + + packages = [f"{pkg.name}-{pkg.version}" for pkg in pyodide_packages + wheels] + + if packages: + logger.info("Successfully installed %s", ", ".join(packages)) + + importlib.invalidate_caches() + + +def search_installed_packages( + names: list[str], +) -> list[importlib.metadata.Distribution]: + """ + Get installed packages by name. + + Parameters + ---------- + names + List of distribution names to search for. + + Returns + ------- + List of distributions that were found. + If a distribution is not found, it is not included in the list. + """ + distributions = [] + for name in names: + try: + distributions.append(importlib.metadata.distribution(name)) + except importlib.metadata.PackageNotFoundError: + pass + + return distributions diff --git a/micropip/package_manager.py b/micropip/package_manager.py index 0ad6fda..56bdab7 100644 --- a/micropip/package_manager.py +++ b/micropip/package_manager.py @@ -2,6 +2,8 @@ import builtins import importlib import importlib.metadata +import logging +from collections.abc import Iterable from importlib.metadata import Distribution from pathlib import Path from typing import ( # noqa: UP035 List import is necessary due to the `list` method @@ -14,7 +16,7 @@ from ._vendored.packaging.src.packaging.markers import default_environment from .constants import FAQ_URLS from .freeze import freeze_lockfile -from .logging import setup_logging +from .logging import indent_log, setup_logging from .package import PackageDict, PackageMetadata from .transaction import Transaction @@ -48,6 +50,7 @@ async def install( index_urls: list[str] | str | None = None, *, constraints: list[str] | None = None, + reinstall: bool = False, verbose: bool | int | None = None, ) -> None: """Install the given package and all of its dependencies. @@ -140,6 +143,16 @@ async def install( Unlike ``requirements``, the package name _must_ be provided in the PEP-508 format e.g. ``pkgname@https://...``. + reinstall: + + If ``False`` (default), micropip will show an error if the requested package + is already installed, but with a incompatible version. If ``True``, + micropip will uninstall the existing packages that are not compatible with + the requested version and install the packages again. + + Note that packages that are already imported will not be reloaded, so make + sure to reload the module after reinstalling by e.g. running importlib.reload(module). + verbose: Print more information about the process. By default, micropip does not change logger level. Setting ``verbose=True`` will print similar @@ -179,6 +192,7 @@ async def install( verbose=verbose, index_urls=index_urls, constraints=constraints, + reinstall=reinstall, ) await transaction.gather_requirements(requirements) @@ -193,7 +207,15 @@ async def install( pyodide_packages, wheels = transaction.pyodide_packages, transaction.wheels - package_names = [pkg.name for pkg in wheels + pyodide_packages] + packages_all = [pkg.name for pkg in wheels + pyodide_packages] + + distributions = search_installed_packages(packages_all) + # This check is redundant because the distributions will always be an emtpy list when reinstall==False + # (no installed packages will be returned from transaction) + # But just in case. + if reinstall: + with indent_log(): + self._uninstall_distributions(distributions, logger) logger.debug( "Installing packages %r and wheels %r ", @@ -201,9 +223,9 @@ async def install( [w.filename for w in transaction.wheels], ) - if package_names: + if packages_all: logger.info( - "Installing collected packages: %s", ", ".join(package_names) + "Installing collected packages: %s", ", ".join(packages_all) ) # Install PyPI packages @@ -422,68 +444,7 @@ def uninstall( except importlib.metadata.PackageNotFoundError: logger.warning("Skipping '%s' as it is not installed.", package) - for dist in distributions: - # Note: this value needs to be retrieved before removing files, as - # dist.name uses metadata file to get the name - name = dist.name - version = dist.version - - logger.info("Found existing installation: %s %s", name, version) - - root = get_root(dist) - files = get_files_in_distribution(dist) - directories = set() - - for file in files: - if not file.is_file(): - if not file.is_relative_to(root): - # This file is not in the site-packages directory. Probably one of: - # - data_files - # - scripts - # - entry_points - # Since we don't support these, we can ignore them (except for data_files (TODO)) - logger.warning( - "skipping file '%s' that is relative to root", - ) - continue - # see PR 130, it is likely that this is never triggered since Python 3.12 - # as non existing files are not listed by get_files_in_distribution anymore. - logger.warning( - "A file '%s' listed in the metadata of '%s' does not exist.", - file, - name, - ) - - continue - - file.unlink() - - if file.parent != root: - directories.add(file.parent) - - # Remove directories in reverse hierarchical order - for directory in sorted( - directories, key=lambda x: len(x.parts), reverse=True - ): - try: - directory.rmdir() - except OSError: - logger.warning( - "A directory '%s' is not empty after uninstallation of '%s'. " - "This might cause problems when installing a new version of the package. ", - directory, - name, - ) - - if hasattr(self.compat_layer.loadedPackages, name): - delattr(self.compat_layer.loadedPackages, name) - else: - # This should not happen, but just in case - logger.warning( - "a package '%s' was not found in loadedPackages.", name - ) - - logger.info("Successfully uninstalled %s-%s", name, version) + self._uninstall_distributions(distributions, logger) importlib.invalidate_caches() @@ -524,3 +485,109 @@ def set_constraints(self, constraints: List[str]): # noqa: UP006 """ self.constraints = constraints[:] + + def _uninstall_distributions( + self, + distributions: Iterable[Distribution], + logger: logging.Logger, # TODO: move this to an attribute of the PackageManager + ) -> None: + """ + Uninstall the given package distributions. + + This function does not do any checks, so make sure that the distributions + are installed and that they are installed using a wheel file, i.e. packages + that have distribution metadata. + + This function also does not invalidate the import cache, so make sure to + call `importlib.invalidate_caches()` after calling this function. + + Parameters + ---------- + distributions + Package distributions to uninstall. + + """ + for dist in distributions: + # Note: this value needs to be retrieved before removing files, as + # dist.name uses metadata file to get the name + name = dist.name + version = dist.version + + logger.info("Found existing installation: %s %s", name, version) + + root = get_root(dist) + files = get_files_in_distribution(dist) + directories = set() + + for file in files: + if not file.is_file(): + if not file.is_relative_to(root): + # This file is not in the site-packages directory. Probably one of: + # - data_files + # - scripts + # - entry_points + # Since we don't support these, we can ignore them (except for data_files (TODO)) + logger.warning( + "skipping file '%s' that is relative to root", + ) + continue + # see PR 130, it is likely that this is never triggered since Python 3.12 + # as non existing files are not listed by get_files_in_distribution anymore. + logger.warning( + "A file '%s' listed in the metadata of '%s' does not exist.", + file, + name, + ) + + continue + + file.unlink() + + if file.parent != root: + directories.add(file.parent) + + # Remove directories in reverse hierarchical order + for directory in sorted( + directories, key=lambda x: len(x.parts), reverse=True + ): + try: + directory.rmdir() + except OSError: + logger.warning( + "A directory '%s' is not empty after uninstallation of '%s'. " + "This might cause problems when installing a new version of the package. ", + directory, + name, + ) + + if hasattr(self.compat_layer.loadedPackages, name): + delattr(self.compat_layer.loadedPackages, name) + else: + # This should not happen, but just in case + logger.warning("a package '%s' was not found in loadedPackages.", name) + + logger.info("Successfully uninstalled %s-%s", name, version) + + +def search_installed_packages( + names: list[str], +) -> list[importlib.metadata.Distribution]: + """ + Get installed packages by name. + Parameters + ---------- + names + List of distribution names to search for. + Returns + ------- + List of distributions that were found. + If a distribution is not found, it is not included in the list. + """ + distributions = [] + for name in names: + try: + distributions.append(importlib.metadata.distribution(name)) + except importlib.metadata.PackageNotFoundError: + pass + + return distributions diff --git a/micropip/transaction.py b/micropip/transaction.py index db32665..0443a51 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -45,6 +45,7 @@ class Transaction: verbose: bool | int | None = None constraints: list[str] | None = None + reinstall: bool = False def __post_init__(self) -> None: # If index_urls is None, pyodide-lock.json have to be searched first. @@ -94,7 +95,22 @@ async def add_requirement(self, req: str | Requirement) -> None: return await self.add_requirement_inner(Requirement(req)) - def check_version_satisfied(self, req: Requirement) -> tuple[bool, str]: + def check_version_satisfied( + self, req: Requirement, *, allow_reinstall: bool = False + ) -> tuple[bool, str]: + """ + Check if the installed version of a package satisfies the requirement. + Returns True if the requirement is satisfied, False otherwise. + + Parameters + ---------- + req + The requirement to check. + + allow_reinstall + If False, this function will raise exception if the package is already installed + and the installed version does not satisfy the requirement. + """ ver = None try: ver = importlib.metadata.version(req.name) @@ -110,9 +126,16 @@ def check_version_satisfied(self, req: Requirement) -> tuple[bool, str]: # installed version matches, nothing to do return True, ver - raise ValueError( - f"Requested '{req}', " f"but {req.name}=={ver} is already installed" - ) + if allow_reinstall: + return False, "" + else: + raise ValueError( + f"Requested '{req}', " + f"but {req.name}=={ver} is already installed. " + "If you want to reinstall the package with a different version, " + "use micropip.install(..., reinstall=True) to force reinstall, " + "or micropip.uninstall(...) to uninstall the package first." + ) async def add_requirement_inner( self, @@ -169,7 +192,10 @@ def eval_marker(e: dict[str, str]) -> bool: # Is some version of this package is already installed? req.name = canonicalize_name(req.name) - satisfied, ver = self.check_version_satisfied(req) + satisfied, ver = self.check_version_satisfied( + req, allow_reinstall=self.reinstall + ) + if satisfied: logger.info("Requirement already satisfied: %s (%s)", req, ver) return @@ -251,7 +277,9 @@ async def _add_requirement_from_package_index(self, req: Requirement): # Maybe while we were downloading pypi_json some other branch # installed the wheel? - satisfied, ver = self.check_version_satisfied(req) + satisfied, ver = self.check_version_satisfied( + req, allow_reinstall=self.reinstall + ) if satisfied: logger.info("Requirement already satisfied: %s (%s)", req, ver) diff --git a/micropip/uninstall.py b/micropip/uninstall.py new file mode 100644 index 0000000..3ebc8e3 --- /dev/null +++ b/micropip/uninstall.py @@ -0,0 +1,108 @@ +import importlib +import importlib.metadata +import logging +from collections.abc import Iterable +from importlib.metadata import Distribution + +from ._compat import loadedPackages +from ._utils import get_files_in_distribution, get_root +from .logging import setup_logging + + +def uninstall(packages: str | list[str], *, verbose: bool | int = False) -> None: + with setup_logging().ctx_level(verbose) as logger: + + if isinstance(packages, str): + packages = [packages] + + distributions: list[Distribution] = [] + for package in packages: + try: + dist = importlib.metadata.distribution(package) + distributions.append(dist) + except importlib.metadata.PackageNotFoundError: + logger.warning("Skipping '%s' as it is not installed.", package) + + uninstall_distributions(distributions, logger) + + importlib.invalidate_caches() + + +def uninstall_distributions( + distributions: Iterable[Distribution], + logger: logging.Logger, # TODO: move this to an attribute of the PackageManager +) -> None: + """ + Uninstall the given package distributions. + + This function does not do any checks, so make sure that the distributions + are installed and that they are installed using a wheel file, i.e. packages + that have distribution metadata. + + This function also does not invalidate the import cache, so make sure to + call `importlib.invalidate_caches()` after calling this function. + + Parameters + ---------- + distributions + Package distributions to uninstall. + + """ + for dist in distributions: + # Note: this value needs to be retrieved before removing files, as + # dist.name uses metadata file to get the name + name = dist.name + version = dist.version + + logger.info("Found existing installation: %s %s", name, version) + + root = get_root(dist) + files = get_files_in_distribution(dist) + directories = set() + + for file in files: + if not file.is_file(): + if not file.is_relative_to(root): + # This file is not in the site-packages directory. Probably one of: + # - data_files + # - scripts + # - entry_points + # Since we don't support these, we can ignore them (except for data_files (TODO)) + logger.warning( + "skipping file '%s' that is relative to root", + ) + continue + # see PR 130, it is likely that this is never triggered since Python 3.12 + # as non existing files are not listed by get_files_in_distribution anymore. + logger.warning( + "A file '%s' listed in the metadata of '%s' does not exist.", + file, + name, + ) + + continue + + file.unlink() + + if file.parent != root: + directories.add(file.parent) + + # Remove directories in reverse hierarchical order + for directory in sorted(directories, key=lambda x: len(x.parts), reverse=True): + try: + directory.rmdir() + except OSError: + logger.warning( + "A directory '%s' is not empty after uninstallation of '%s'. " + "This might cause problems when installing a new version of the package. ", + directory, + name, + ) + + if hasattr(loadedPackages, name): + delattr(loadedPackages, name) + else: + # This should not happen, but just in case + logger.warning("a package '%s' was not found in loadedPackages.", name) + + logger.info("Successfully uninstalled %s-%s", name, version) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 79d1cb6..12a5d55 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -52,6 +52,41 @@ async def _run(selenium): _run(selenium_standalone_micropip) +@integration_test_only +def test_integration_install_reinstall(selenium_standalone_micropip, pytestconfig): + @run_in_pyodide + async def _run(selenium): + import micropip + + await micropip.install("mccabe==0.6.1") + + import mccabe + + assert mccabe.__version__ == "0.6.1" + + try: + await micropip.install("mccabe==0.7.0", reinstall=False) + except ValueError as e: + assert "already installed" in str(e) + else: + raise Exception("Should raise!") + + await micropip.install("mccabe==0.7.0", reinstall=True) + + import mccabe + + # still 0.6.1 + assert mccabe.__version__ == "0.6.1" + + import importlib + + importlib.reload(mccabe) + + assert mccabe.__version__ == "0.7.0" + + _run(selenium_standalone_micropip) + + @integration_test_only def test_integration_install_yanked(selenium_standalone_micropip, pytestconfig): @run_in_pyodide diff --git a/tests/test_install.py b/tests/test_install.py index af8dae2..64baba1 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -448,3 +448,31 @@ async def run(selenium, numpy_url, shapely_url): Point(0, 0) run(selenium, numpy_wheel.url, shapely_wheel.url) + + +@pytest.mark.asyncio +async def test_reinstall_different_version( + mock_fetch: mock_fetch_cls, + mock_importlib, +) -> None: + import importlib.metadata + + import pytest + + dummy = "dummy" + version_old = "1.0.0" + version_new = "2.0.0" + + mock_fetch.add_pkg_version(dummy, version_old) + mock_fetch.add_pkg_version(dummy, version_new) + + await micropip.install(f"{dummy}=={version_new}") + assert micropip.list()[dummy].version == version_new + assert importlib.metadata.version(dummy) == version_new + + with pytest.raises(ValueError, match="already installed"): + await micropip.install(f"{dummy}=={version_old}", reinstall=False) + + await micropip.install(f"{dummy}=={version_old}", reinstall=True) + assert micropip.list()[dummy].version == version_old + assert importlib.metadata.version(dummy) == version_old