diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2a474e02..cdc899a2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - pyodide-version: ['0.27.0a2'] + pyodide-version: ["0.27.0a2"] test-config: [ # FIXME: recent version of chrome gets timeout {runner: selenium, runtime: chrome, runtime-version: "125" }, @@ -28,6 +28,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + submodules: recursive - uses: actions/setup-python@v5 with: @@ -88,6 +90,7 @@ jobs: - uses: actions/checkout@cbb722410c2e876e24abbe8de2cc27693e501dcb # v4.2.2 with: ref: ${{ github.event.pull_request.head.sha }} + submodules: recursive - id: check-integration-test-trigger name: Check integration test trigger @@ -110,15 +113,25 @@ jobs: environment: PyPi-deploy steps: - uses: actions/checkout@v4 + with: + submodules: recursive - uses: actions/setup-python@v5 with: python-version: 3.12 + + # IMPORTANT: always build sdist, and then the wheel from + # the sdist (like it is currently done here). This is + # because we want to ensure that no extra files get + # copied, which can be the case with building in-tree. + # The MANIFEST.in file ensures that the sdist doesn't + # contain any unnecessary files. - name: Install requirements and build wheel shell: bash -l {0} run: | python -m pip install build twine python -m build . + - name: Publish package uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/remote_package_index_test.yml b/.github/workflows/remote_package_index_test.yml index fb8739f2..b3413455 100644 --- a/.github/workflows/remote_package_index_test.yml +++ b/.github/workflows/remote_package_index_test.yml @@ -23,6 +23,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + submodules: recursive - uses: actions/setup-python@v5 with: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..8a4e61dd --- /dev/null +++ b/.gitmodules @@ -0,0 +1,5 @@ +# d8e3b31b734926ebbcaff654279f6855a73e052f, for 24.2 release +# https://github.com/pypa/packaging/releases/tag/24.2 +[submodule "micropip/_vendored/packaging"] + path = micropip/_vendored/packaging + url = https://github.com/pypa/packaging/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 81404672..13d710c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (the ones that starts with `../` or `./`) [#174](https://github.com/pyodide/micropip/pull/174) +### Added + +- `micropip` now vendors `pypa/packaging` for better reliability. + [#178](https://github.com/pyodide/micropip/pull/178) + ## [0.8.0] - 2024/12/15 ### Added diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..93030c1d --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,16 @@ +exclude micropip/_vendored/packaging/.github/* +exclude micropip/_vendored/packaging/docs/* +exclude micropip/_vendored/packaging/tasks/* +exclude micropip/_vendored/packaging/tests/* +exclude micropip/_vendored/packaging/.pre-commit-config.yaml +exclude micropip/_vendored/packaging/.readthedocs.yml +exclude micropip/_vendored/packaging/CHANGELOG.rst +exclude micropip/_vendored/packaging/CONTRIBUTING.rst +exclude micropip/_vendored/packaging/noxfile.py +exclude micropip/_vendored/packaging/pyproject.toml + +include micropip/_vendored/packaging/LICENSE +include micropip/_vendored/packaging/LICENSE.APACHE +include micropip/_vendored/packaging/LICENSE.BSD + +include micropip/_vendored/packaging/README.rst diff --git a/micropip/_compat/_compat_not_in_pyodide.py b/micropip/_compat/_compat_not_in_pyodide.py index 96cc53bf..d2dca77c 100644 --- a/micropip/_compat/_compat_not_in_pyodide.py +++ b/micropip/_compat/_compat_not_in_pyodide.py @@ -14,6 +14,7 @@ class CompatibilityNotInPyodide(CompatibilityLayer): # Vendored from packaging + # TODO: use packaging APIs here instead? _canonicalize_regex = re.compile(r"[-_.]+") class HttpStatusError(Exception): diff --git a/micropip/_utils.py b/micropip/_utils.py index 70d4724e..e9bd886c 100644 --- a/micropip/_utils.py +++ b/micropip/_utils.py @@ -4,14 +4,19 @@ from pathlib import Path from sysconfig import get_config_var, get_platform -from packaging.requirements import Requirement -from packaging.tags import Tag -from packaging.tags import sys_tags as sys_tags_orig -from packaging.utils import BuildTag, InvalidWheelFilename, canonicalize_name -from packaging.utils import parse_wheel_filename as parse_wheel_filename_orig -from packaging.version import InvalidVersion, Version - from ._compat import REPODATA_PACKAGES +from ._vendored.packaging.src.packaging.requirements import Requirement +from ._vendored.packaging.src.packaging.tags import Tag +from ._vendored.packaging.src.packaging.tags import sys_tags as sys_tags_orig +from ._vendored.packaging.src.packaging.utils import ( + BuildTag, + InvalidWheelFilename, + canonicalize_name, +) +from ._vendored.packaging.src.packaging.utils import ( + parse_wheel_filename as parse_wheel_filename_orig, +) +from ._vendored.packaging.src.packaging.version import InvalidVersion, Version def get_dist_info(dist: Distribution) -> Path: diff --git a/micropip/_vendored/packaging b/micropip/_vendored/packaging new file mode 160000 index 00000000..d8e3b31b --- /dev/null +++ b/micropip/_vendored/packaging @@ -0,0 +1 @@ +Subproject commit d8e3b31b734926ebbcaff654279f6855a73e052f diff --git a/micropip/externals/mousebender/simple.py b/micropip/externals/mousebender/simple.py index 134bf057..3c460683 100644 --- a/micropip/externals/mousebender/simple.py +++ b/micropip/externals/mousebender/simple.py @@ -7,13 +7,12 @@ import warnings from typing import Any, Dict, List, Optional, Union, Literal, TypeAlias, TypedDict -import packaging.utils +import micropip._vendored.packaging.src.packaging.utils as packaging_utils ACCEPT_JSON_V1 = "application/vnd.pypi.simple.v1+json" - class UnsupportedAPIVersion(Exception): """The major version of an API response is not supported.""" @@ -92,7 +91,7 @@ class ProjectDetails_1_0(TypedDict): """A :class:`~typing.TypedDict` for a project details response (:pep:`691`).""" meta: _Meta_1_0 - name: packaging.utils.NormalizedName + name: packaging_utils.NormalizedName files: list[ProjectFileDetails_1_0] @@ -100,7 +99,7 @@ class ProjectDetails_1_1(TypedDict): """A :class:`~typing.TypedDict` for a project details response (:pep:`700`).""" meta: _Meta_1_1 - name: packaging.utils.NormalizedName + name: packaging_utils.NormalizedName files: list[ProjectFileDetails_1_1] # PEP 700 versions: List[str] @@ -235,6 +234,6 @@ def from_project_details_html(html: str, name: str) -> ProjectDetails_1_0: files.append(details) return { "meta": {"api-version": "1.0"}, - "name": packaging.utils.canonicalize_name(name), + "name": packaging_utils.canonicalize_name(name), "files": files, } \ No newline at end of file diff --git a/micropip/freeze.py b/micropip/freeze.py index b4fa9a36..caa8dfbf 100644 --- a/micropip/freeze.py +++ b/micropip/freeze.py @@ -4,9 +4,8 @@ from copy import deepcopy from typing import Any -from packaging.utils import canonicalize_name - from ._utils import fix_package_dependencies +from ._vendored.packaging.src.packaging.utils import canonicalize_name def freeze_lockfile( diff --git a/micropip/install.py b/micropip/install.py index 5e32b501..ac4a0d8d 100644 --- a/micropip/install.py +++ b/micropip/install.py @@ -4,9 +4,8 @@ from pathlib import Path from typing import Any -from packaging.markers import default_environment - from ._compat import loadPackage, to_js +from ._vendored.packaging.src.packaging.markers import default_environment from .constants import FAQ_URLS from .logging import setup_logging from .transaction import Transaction diff --git a/micropip/metadata.py b/micropip/metadata.py index 6809e50a..0809fc1e 100644 --- a/micropip/metadata.py +++ b/micropip/metadata.py @@ -7,8 +7,8 @@ from collections.abc import Iterable from pathlib import Path -from packaging.requirements import Requirement -from packaging.utils import canonicalize_name +from ._vendored.packaging.src.packaging.requirements import Requirement +from ._vendored.packaging.src.packaging.utils import canonicalize_name def safe_name(name): diff --git a/micropip/package.py b/micropip/package.py index d39afa89..c00476ef 100644 --- a/micropip/package.py +++ b/micropip/package.py @@ -3,7 +3,7 @@ from dataclasses import astuple, dataclass from typing import Any -from packaging.utils import canonicalize_name +from ._vendored.packaging.src.packaging.utils import canonicalize_name __all__ = ["PackageDict"] diff --git a/micropip/package_index.py b/micropip/package_index.py index 562a2521..187e777b 100644 --- a/micropip/package_index.py +++ b/micropip/package_index.py @@ -9,11 +9,10 @@ from typing import Any from urllib.parse import urljoin, urlparse, urlunparse -from packaging.utils import InvalidWheelFilename -from packaging.version import InvalidVersion, Version - from ._compat import HttpStatusError, fetch_string_and_headers from ._utils import is_package_compatible, parse_version +from ._vendored.packaging.src.packaging.utils import InvalidWheelFilename +from ._vendored.packaging.src.packaging.version import InvalidVersion, Version from .externals.mousebender.simple import from_project_details_html from .types import DistributionMetadata from .wheelinfo import WheelInfo diff --git a/micropip/transaction.py b/micropip/transaction.py index f29c48a3..f830f051 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -6,12 +6,11 @@ from importlib.metadata import PackageNotFoundError from urllib.parse import urlparse -from packaging.requirements import Requirement -from packaging.utils import canonicalize_name - from . import package_index from ._compat import REPODATA_PACKAGES from ._utils import best_compatible_tag_index, check_compatible +from ._vendored.packaging.src.packaging.requirements import Requirement +from ._vendored.packaging.src.packaging.utils import canonicalize_name from .constants import FAQ_URLS from .package import PackageMetadata from .package_index import ProjectInfo diff --git a/micropip/wheelinfo.py b/micropip/wheelinfo.py index 8e35f69b..3483bef4 100644 --- a/micropip/wheelinfo.py +++ b/micropip/wheelinfo.py @@ -7,10 +7,6 @@ from typing import Any, Literal from urllib.parse import ParseResult, urlparse -from packaging.requirements import Requirement -from packaging.tags import Tag -from packaging.version import Version - from ._compat import ( fetch_bytes, get_dynlibs, @@ -18,6 +14,9 @@ loadedPackages, ) from ._utils import parse_wheel_filename +from ._vendored.packaging.src.packaging.requirements import Requirement +from ._vendored.packaging.src.packaging.tags import Tag +from ._vendored.packaging.src.packaging.version import Version from .metadata import Metadata, safe_name, wheel_dist_info_dir from .types import DistributionMetadata diff --git a/pyproject.toml b/pyproject.toml index d761a269..936b6986 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,8 @@ classifiers = [ "Operating System :: OS Independent", ] dynamic = ["version"] -dependencies = ["packaging>=23.0"] +dependencies = [] + [project.optional-dependencies] test = [ "pytest-httpserver", @@ -64,7 +65,12 @@ known-first-party = [ ] [tool.mypy] +exclude = ["micropip/_vendored/"] python_version = "3.12" show_error_codes = true warn_unreachable = true ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "micropip._vendored.*" +warn_unreachable = false diff --git a/tests/conftest.py b/tests/conftest.py index a1a3a518..ff24ab97 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,10 +10,11 @@ from typing import Any import pytest -from packaging.utils import parse_wheel_filename from pytest_httpserver import HTTPServer from pytest_pyodide import spawn_web_server +from micropip._vendored.packaging.src.packaging.utils import parse_wheel_filename + def pytest_addoption(parser): parser.addoption( diff --git a/tests/test_install.py b/tests/test_install.py index 9b7f7bc8..ca5d224c 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -1,9 +1,9 @@ import pytest from conftest import mock_fetch_cls -from packaging.utils import parse_wheel_filename from pytest_pyodide import run_in_pyodide import micropip +from micropip._vendored.packaging.src.packaging.utils import parse_wheel_filename def test_install_custom_url(selenium_standalone_micropip, wheel_catalog): diff --git a/tests/test_transaction.py b/tests/test_transaction.py index 714ac46e..ce199e92 100644 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -1,6 +1,7 @@ import pytest from conftest import SNOWBALL_WHEEL -from packaging.tags import Tag + +from micropip._vendored.packaging.src.packaging.tags import Tag @pytest.mark.parametrize( @@ -188,8 +189,7 @@ def _pypi_metadata(package, versions_to_tags): def test_last_version_from_pypi(): pytest.importorskip("packaging") - from packaging.requirements import Requirement - + from micropip._vendored.packaging.src.packaging.requirements import Requirement from micropip.transaction import find_wheel requirement = Requirement("dummy_module") @@ -209,8 +209,7 @@ def test_find_wheel_invalid_version(): it should be skipped instead of producing an error """ pytest.importorskip("packaging") - from packaging.requirements import Requirement - + from micropip._vendored.packaging.src.packaging.requirements import Requirement from micropip.transaction import find_wheel requirement = Requirement("dummy_module") @@ -245,8 +244,7 @@ def test_find_wheel_invalid_version(): @pytest.mark.parametrize(*_best_tag_test_cases) def test_best_tag_from_pypi(package, version, incompatible_tags, compatible_tags): pytest.importorskip("packaging") - from packaging.requirements import Requirement - + from micropip._vendored.packaging.src.packaging.requirements import Requirement from micropip.transaction import find_wheel requirement = Requirement(package) @@ -280,8 +278,7 @@ def test_last_version_and_best_tag_from_pypi( package, old_version, new_version, old_tags, new_tags ): pytest.importorskip("packaging") - from packaging.requirements import Requirement - + from micropip._vendored.packaging.src.packaging.requirements import Requirement from micropip.transaction import find_wheel requirement = Requirement(package) diff --git a/tests/test_uninstall.py b/tests/test_uninstall.py index f862345c..fc3d7a54 100644 --- a/tests/test_uninstall.py +++ b/tests/test_uninstall.py @@ -1,7 +1,7 @@ # isort: skip_file from pytest_pyodide import run_in_pyodide -from packaging.utils import parse_wheel_filename +from micropip._vendored.packaging.src.packaging.utils import parse_wheel_filename TEST_PACKAGE_NAME = "test-wheel-uninstall"