diff --git a/pyodide_build/common.py b/pyodide_build/common.py index c30c0e2..6f02c68 100644 --- a/pyodide_build/common.py +++ b/pyodide_build/common.py @@ -451,7 +451,9 @@ def to_bool(value: str) -> bool: return value.lower() not in {"", "0", "false", "no", "off"} -def download_and_unpack_archive(url: str, path: Path, descr: str) -> None: +def download_and_unpack_archive( + url: str, path: Path, descr: str, *, exists_ok: bool = False +) -> None: """ Download the cross-build environment from the given URL and extract it to the given path. @@ -465,7 +467,7 @@ def download_and_unpack_archive(url: str, path: Path, descr: str) -> None: """ logger.info("Downloading %s from %s", descr, url) - if path.exists(): + if not exists_ok and path.exists(): raise FileExistsError(f"Path {path} already exists") try: diff --git a/pyodide_build/config.py b/pyodide_build/config.py index 9786220..8407fa0 100644 --- a/pyodide_build/config.py +++ b/pyodide_build/config.py @@ -179,6 +179,7 @@ def _get_make_environment_vars(self) -> Mapping[str, str]: "cpythoninstall": "CPYTHONINSTALL", "rustflags": "RUSTFLAGS", "rust_toolchain": "RUST_TOOLCHAIN", + "rust_emscripten_target_url": "RUST_EMSCRIPTEN_TARGET_URL", "cflags": "SIDE_MODULE_CFLAGS", "cxxflags": "SIDE_MODULE_CXXFLAGS", "ldflags": "SIDE_MODULE_LDFLAGS", @@ -210,6 +211,7 @@ def _get_make_environment_vars(self) -> Mapping[str, str]: "cxxflags", "ldflags", "rust_toolchain", + "rust_emscripten_target_url", "meson_cross_file", "skip_emscripten_version_check", "build_dependency_index_url", @@ -229,6 +231,7 @@ def _get_make_environment_vars(self) -> Mapping[str, str]: "cargo_build_target": "wasm32-unknown-emscripten", "cargo_target_wasm32_unknown_emscripten_linker": "emcc", "rust_toolchain": "nightly-2025-02-01", + "rust_emscripten_target_url": "", # Other configuration "pyodide_jobs": "1", "skip_emscripten_version_check": "0", diff --git a/pyodide_build/pypabuild.py b/pyodide_build/pypabuild.py index 534a054..e9c4e43 100644 --- a/pyodide_build/pypabuild.py +++ b/pyodide_build/pypabuild.py @@ -295,6 +295,7 @@ def get_build_env( args["orig__name__"] = __name__ args["pythoninclude"] = get_build_flag("PYTHONINCLUDE") args["PATH"] = env["PATH"] + args["abi"] = get_build_flag("PYODIDE_ABI_VERSION") pywasmcross_env = json.dumps(args) # Store into environment variable and to disk. In most cases we will diff --git a/pyodide_build/pywasmcross.py b/pyodide_build/pywasmcross.py index e3c54e6..36a4487 100755 --- a/pyodide_build/pywasmcross.py +++ b/pyodide_build/pywasmcross.py @@ -77,6 +77,7 @@ class CrossCompileArgs(NamedTuple): target_install_dir: str = "" # The path to the target Python installation pythoninclude: str = "" # path to the cross-compiled Python include directory exports: Literal["whole_archive", "requested", "pyinit"] | list[str] = "pyinit" + abi: str = "" def is_link_cmd(line: list[str]) -> bool: @@ -93,7 +94,7 @@ def is_link_cmd(line: list[str]) -> bool: return False -def replay_genargs_handle_dashl(arg: str, used_libs: set[str]) -> str | None: +def replay_genargs_handle_dashl(arg: str, used_libs: set[str], abi: str) -> str | None: """ Figure out how to replace a `-lsomelib` argument. @@ -118,6 +119,9 @@ def replay_genargs_handle_dashl(arg: str, used_libs: set[str]) -> str | None: if arg == "-lgfortran": return None + if abi > "2025" and arg in ["-lfreetype", "-lpng"]: + arg += "-wasm-sjlj" + # WASM link doesn't like libraries being included twice # skip second one if arg in used_libs: @@ -555,7 +559,7 @@ def handle_command_generate_args( # noqa: C901 continue if arg.startswith("-l"): - result = replay_genargs_handle_dashl(arg, used_libs) + result = replay_genargs_handle_dashl(arg, used_libs, build_args.abi) elif arg.startswith("-I"): result = replay_genargs_handle_dashI(arg, build_args.target_install_dir) elif arg.startswith("-Wl"): @@ -578,7 +582,11 @@ def handle_command_generate_args( # noqa: C901 # Better to fail at compile or link time. if is_link_cmd(line): new_args.append("-Wl,--fatal-warnings") - new_args.extend(build_args.ldflags.split()) + for arg in build_args.ldflags.split(): + if arg.startswith("-l"): + arg = replay_genargs_handle_dashl(arg, used_libs, build_args.abi) + if arg: + new_args.append(arg) new_args.extend(get_export_flags(line, build_args.exports)) if "-c" in line: @@ -638,6 +646,7 @@ def compiler_main(): target_install_dir=PYWASMCROSS_ARGS["target_install_dir"], pythoninclude=PYWASMCROSS_ARGS["pythoninclude"], exports=PYWASMCROSS_ARGS["exports"], + abi=PYWASMCROSS_ARGS["abi"], ) basename = Path(sys.argv[0]).name args = list(sys.argv) diff --git a/pyodide_build/recipe/graph_builder.py b/pyodide_build/recipe/graph_builder.py index 4931252..4200181 100755 --- a/pyodide_build/recipe/graph_builder.py +++ b/pyodide_build/recipe/graph_builder.py @@ -30,6 +30,7 @@ from pyodide_build import build_env from pyodide_build.build_env import BuildArgs from pyodide_build.common import ( + download_and_unpack_archive, exit_with_stdio, extract_wheel_metadata_file, find_matching_wheels, @@ -700,13 +701,23 @@ def run(self, n_jobs: int, already_built: set[str]) -> None: self.build_queue.put((job_priority(dependent), dependent)) +def _run(cmd, *args, check=False, **kwargs): + result = subprocess.run(cmd, *args, **kwargs, check=check) + if result.returncode != 0: + logger.error("ERROR: command failed %s", " ".join(cmd)) + exit_with_stdio(result) + return result + + def _ensure_rust_toolchain(): rust_toolchain = build_env.get_build_flag("RUST_TOOLCHAIN") - result = subprocess.run( - ["rustup", "toolchain", "install", rust_toolchain], check=False - ) - if result.returncode == 0: - result = subprocess.run( + _run(["rustup", "toolchain", "install", rust_toolchain]) + _run(["rustup", "default", rust_toolchain]) + + url = build_env.get_build_flag("RUST_EMSCRIPTEN_TARGET_URL") + if not url: + # Install target with rustup target add + _run( [ "rustup", "target", @@ -714,12 +725,26 @@ def _ensure_rust_toolchain(): "wasm32-unknown-emscripten", "--toolchain", rust_toolchain, - ], - check=False, + ] ) - if result.returncode != 0: - logger.error("ERROR: rustup toolchain install failed") - exit_with_stdio(result) + return + + # Install target from url + result = _run( + ["rustup", "which", "--toolchain", rust_toolchain, "rustc"], + capture_output=True, + text=True, + ) + toolchain_root = Path(result.stdout).parents[1] + rustlib = toolchain_root / "lib/rustlib" + install_token = rustlib / "wasm32-unknown-emscripten_install-url.txt" + if install_token.exists() and install_token.read_text() == url: + return + shutil.rmtree(rustlib / "wasm32-unknown-emscripten", ignore_errors=True) + download_and_unpack_archive( + url, rustlib, "wasm32-unknown-emscripten target", exists_ok=True + ) + install_token.write_text(url) def build_from_graph( diff --git a/pyproject.toml b/pyproject.toml index abe84a2..0b947b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -133,7 +133,7 @@ lint.select = [ lint.logger-objects = ["pyodide_build.logger.logger"] -lint.ignore = ["E402", "E501", "E731", "E741", "PERF401", "PLW2901", "UP038"] +lint.ignore = ["E402", "E501", "E731", "E741", "PERF401", "PLW2901", "UP038", "PLR0912"] # line-length = 219 # E501: Recommended goal is 88 to match black target-version = "py312"