Remove types-request, loky, and pyyaml from runtime dependencies

pyodide_build/common.py

@@ -204,9 +204,9 @@ def get_num_cores() -> int:
     Return the number of CPUs the current process can use.
     If the number of CPUs cannot be determined, return 1.
     """
-    import loky
+    from .vendor.loky import cpu_count
 
-    return loky.cpu_count()
+    return cpu_count()
 
 
 def make_zip_archive(

pyodide_build/io.py

@@ -162,10 +162,11 @@ def from_yaml(cls, path: Path) -> "MetaConfig":
         path
             path to the meta.yaml file
         """
-        import yaml
+        from ruamel.yaml import YAML
 
-        stream = path.read_bytes()
-        config_raw = yaml.safe_load(stream)
+        yaml = YAML(typ="safe")
+
+        config_raw = yaml.load(path)
 
         config = cls(**config_raw)
         if config.source.path:
@@ -180,7 +181,10 @@ def to_yaml(self, path: Path) -> None:
         path
             path to the meta.yaml file
         """
-        import yaml
+        from ruamel.yaml import YAML
+
+        yaml = YAML()
+        yaml.representer.ignore_aliases = lambda *_: True
 
         with open(path, "w") as f:
             yaml.dump(self.model_dump(by_alias=True, exclude_unset=True), f)

pyodide_build/tests/test_common.py

@@ -7,7 +7,6 @@
     environment_substitute_args,
     extract_wheel_metadata_file,
     find_missing_executables,
-    get_num_cores,
     make_zip_archive,
     parse_top_level_import_name,
     repack_zip_archive,
@@ -71,16 +70,6 @@ def test_environment_var_substitution(monkeypatch):
     )
 
 
-@pytest.mark.parametrize("num_cpus", [1, 2, 3])
-def test_get_num_cores(monkeypatch, num_cpus):
-    import loky
-
-    with monkeypatch.context() as m:
-        m.setattr(loky, "cpu_count", lambda: num_cpus)
-
-        assert get_num_cores() == num_cpus
-
-
 @pytest.mark.parametrize(
     "compression_level, expected_compression_type",
     [(6, zipfile.ZIP_DEFLATED), (0, zipfile.ZIP_STORED)],

pyodide_build/vendor/loky.py

@@ -0,0 +1,211 @@
+# Copied from: https://github.com/joblib/loky/blob/master/loky/backend/context.py
+
+###############################################################################
+# Basic context management with LokyContext
+#
+# author: Thomas Moreau and Olivier Grisel
+#
+# adapted from multiprocessing/context.py
+#  * Create a context ensuring loky uses only objects that are compatible
+#  * Add LokyContext to the list of context of multiprocessing so loky can be
+#    used with multiprocessing.set_start_method
+#  * Implement a CFS-aware amd physical-core aware cpu_count function.
+#
+import math
+import os
+import subprocess
+import sys
+import traceback
+import warnings
+from concurrent.futures.process import _MAX_WINDOWS_WORKERS
+
+# Cache for the number of physical cores to avoid repeating subprocess calls.
+# It should not change during the lifetime of the program.
+physical_cores_cache = None
+
+
+def cpu_count(only_physical_cores=False):
+    """Return the number of CPUs the current process can use.
+
+    The returned number of CPUs accounts for:
+     * the number of CPUs in the system, as given by
+       ``multiprocessing.cpu_count``;
+     * the CPU affinity settings of the current process
+       (available on some Unix systems);
+     * Cgroup CPU bandwidth limit (available on Linux only, typically
+       set by docker and similar container orchestration systems);
+     * the value of the LOKY_MAX_CPU_COUNT environment variable if defined.
+    and is given as the minimum of these constraints.
+
+    If ``only_physical_cores`` is True, return the number of physical cores
+    instead of the number of logical cores (hyperthreading / SMT). Note that
+    this option is not enforced if the number of usable cores is controlled in
+    any other way such as: process affinity, Cgroup restricted CPU bandwidth
+    or the LOKY_MAX_CPU_COUNT environment variable. If the number of physical
+    cores is not found, return the number of logical cores.
+
+    Note that on Windows, the returned number of CPUs cannot exceed 61 (or 60 for
+    Python < 3.10), see:
+    https://bugs.python.org/issue26903.
+
+    It is also always larger or equal to 1.
+    """
+    # Note: os.cpu_count() is allowed to return None in its docstring
+    os_cpu_count = os.cpu_count() or 1
+    if sys.platform == "win32":
+        # On Windows, attempting to use more than 61 CPUs would result in a
+        # OS-level error. See https://bugs.python.org/issue26903. According to
+        # https://learn.microsoft.com/en-us/windows/win32/procthread/processor-groups
+        # it might be possible to go beyond with a lot of extra work but this
+        # does not look easy.
+        os_cpu_count = min(os_cpu_count, _MAX_WINDOWS_WORKERS)
+
+    cpu_count_user = _cpu_count_user(os_cpu_count)
+    aggregate_cpu_count = max(min(os_cpu_count, cpu_count_user), 1)
+
+    if not only_physical_cores:
+        return aggregate_cpu_count
+
+    if cpu_count_user < os_cpu_count:
+        # Respect user setting
+        return max(cpu_count_user, 1)
+
+    cpu_count_physical, exception = _count_physical_cores()
+    if cpu_count_physical != "not found":
+        return cpu_count_physical
+
+    # Fallback to default behavior
+    if exception is not None:
+        # warns only the first time
+        warnings.warn(
+            "Could not find the number of physical cores for the "
+            f"following reason:\n{exception}\n"
+            "Returning the number of logical cores instead. You can "
+            "silence this warning by setting LOKY_MAX_CPU_COUNT to "
+            "the number of cores you want to use.",
+            stacklevel=2,
+        )
+        traceback.print_tb(exception.__traceback__)
+
+    return aggregate_cpu_count
+
+
+def _cpu_count_cgroup(os_cpu_count):
+    # Cgroup CPU bandwidth limit available in Linux since 2.6 kernel
+    cpu_max_fname = "/sys/fs/cgroup/cpu.max"
+    cfs_quota_fname = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"
+    cfs_period_fname = "/sys/fs/cgroup/cpu/cpu.cfs_period_us"
+    if os.path.exists(cpu_max_fname):
+        # cgroup v2
+        # https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html
+        with open(cpu_max_fname) as fh:
+            cpu_quota_us, cpu_period_us = fh.read().strip().split()
+    elif os.path.exists(cfs_quota_fname) and os.path.exists(cfs_period_fname):
+        # cgroup v1
+        # https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html#management
+        with open(cfs_quota_fname) as fh:
+            cpu_quota_us = fh.read().strip()
+        with open(cfs_period_fname) as fh:
+            cpu_period_us = fh.read().strip()
+    else:
+        # No Cgroup CPU bandwidth limit (e.g. non-Linux platform)
+        cpu_quota_us = "max"
+        cpu_period_us = 100_000  # unused, for consistency with default values
+
+    if cpu_quota_us == "max":
+        # No active Cgroup quota on a Cgroup-capable platform
+        return os_cpu_count
+    else:
+        cpu_quota_us = int(cpu_quota_us)
+        cpu_period_us = int(cpu_period_us)
+        if cpu_quota_us > 0 and cpu_period_us > 0:
+            return math.ceil(cpu_quota_us / cpu_period_us)
+        else:  # pragma: no cover
+            # Setting a negative cpu_quota_us value is a valid way to disable
+            # cgroup CPU bandwidth limits
+            return os_cpu_count
+
+
+def _cpu_count_affinity(os_cpu_count):
+    # Number of available CPUs given affinity settings
+    if hasattr(os, "sched_getaffinity"):
+        try:
+            return len(os.sched_getaffinity(0))
+        except NotImplementedError:
+            pass
+
+    # This can happen for platforms that do not implement any kind of CPU
+    # infinity such as macOS-based platforms.
+    return os_cpu_count
+
+
+def _cpu_count_user(os_cpu_count):
+    """Number of user defined available CPUs"""
+    cpu_count_affinity = _cpu_count_affinity(os_cpu_count)
+
+    cpu_count_cgroup = _cpu_count_cgroup(os_cpu_count)
+
+    # User defined soft-limit passed as a loky specific environment variable.
+    cpu_count_loky = int(os.environ.get("LOKY_MAX_CPU_COUNT", os_cpu_count))
+
+    return min(cpu_count_affinity, cpu_count_cgroup, cpu_count_loky)
+
+
+def _count_physical_cores():
+    """Return a tuple (number of physical cores, exception)
+
+    If the number of physical cores is found, exception is set to None.
+    If it has not been found, return ("not found", exception).
+
+    The number of physical cores is cached to avoid repeating subprocess calls.
+    """
+    exception = None
+
+    # First check if the value is cached
+    global physical_cores_cache
+    if physical_cores_cache is not None:
+        return physical_cores_cache, exception
+
+    # Not cached yet, find it
+    try:
+        if sys.platform == "linux":
+            cpu_info = subprocess.run(
+                "lscpu --parse=core".split(), capture_output=True, text=True
+            )
+            cpu_info = cpu_info.stdout.splitlines()
+            cpu_info = {line for line in cpu_info if not line.startswith("#")}
+            cpu_count_physical = len(cpu_info)
+        elif sys.platform == "win32":
+            cpu_info = subprocess.run(
+                "wmic CPU Get NumberOfCores /Format:csv".split(),
+                capture_output=True,
+                text=True,
+            )
+            cpu_info = cpu_info.stdout.splitlines()
+            cpu_info = [
+                l.split(",")[1] for l in cpu_info if (l and l != "Node,NumberOfCores")
+            ]
+            cpu_count_physical = sum(map(int, cpu_info))
+        elif sys.platform == "darwin":
+            cpu_info = subprocess.run(
+                "sysctl -n hw.physicalcpu".split(),
+                capture_output=True,
+                text=True,
+            )
+            cpu_info = cpu_info.stdout
+            cpu_count_physical = int(cpu_info)
+        else:
+            raise NotImplementedError(f"unsupported platform: {sys.platform}")
+
+        # if cpu_count_physical < 1, we did not find a valid value
+        if cpu_count_physical < 1:
+            raise ValueError(f"found {cpu_count_physical} physical cores < 1")
+
+    except Exception as e:
+        exception = e
+        cpu_count_physical = "not found"
+
+    # Put the result in cache
+    physical_cores_cache = cpu_count_physical
+
+    return cpu_count_physical, exception

pyproject.toml

@@ -14,24 +14,22 @@ classifiers = [
 license = {text = "MPL-2.0"}
 requires-python = ">=3.12"
 dependencies = [
-    "pyyaml",
-    "ruamel.yaml",
-    "packaging",
-    "wheel",
     "build~=1.2.0",
-    "virtualenv",
-    "pydantic>=2,<3",
     "pyodide-cli~=0.2.1",
+    "pyodide-lock==0.1.0a6",
+    "auditwheel-emscripten~=0.0.9",
+    "pydantic>=2,<3",
     "cmake>=3.24",
-    "unearth~=0.6",
+    "wheel",
+    "ruamel.yaml",
+    "packaging",
+    "virtualenv",
     "requests",
-    "types-requests",
     "typer",
-    "auditwheel-emscripten~=0.0.9",
-    "pyodide-lock==0.1.0a6",
-    "resolvelib",
     "rich",
-    "loky",
+    # TODO: make this a extra dependency
+    "resolvelib",
+    "unearth~=0.6",
 ]
 dynamic = ["version"]
 
@@ -63,7 +61,7 @@ test = [
     "pytest",
     "pytest-httpserver",
     "pytest-cov",
-    "packaging",
+    "types-requests",
 ]
 
 [tool.hatch.version]
@@ -76,7 +74,7 @@ exclude = [
 
 [tool.mypy]
 python_version = "3.12"
-mypy_path = ["src/py", "pyodide-build"]
+mypy_path = ["pyodide_build"]
 show_error_codes = true
 warn_unreachable = true
 enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
@@ -100,17 +98,7 @@ strict_equality = true
 
 [[tool.mypy.overrides]]
 module = [
-  "_pyodide_core",
-  "docutils.parsers.rst",
-  "js",
   "loky",
-  "matplotlib.*",
-  "PIL.*",
-  "pyodide_js",
-  "pyodide_js._api",
-  "pytest_pyodide",
-  "pytest_pyodide.runner",
-  "pytest_pyodide.utils",
   "ruamel.yaml",
   "termcolor",
   "test",
@@ -161,12 +149,7 @@ extend-immutable-calls = ["typer.Argument", "typer.Option"]
 
 [tool.ruff.lint.isort]
 known-first-party = [
-  "pyodide",
-  "pyodide_js",
-  "micropip",
   "pyodide_build",
-  "_pyodide",
-  "js",
 ]
 known-third-party = [
   "build",