diff --git a/Makefile b/Makefile index 7b50e1ed5c0..661b3e12ba1 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,6 @@ CXX=em++ all: \ all-but-packages \ dist/pyodide-lock.json \ - dist/console.html \ dist/pyodide.d.ts \ dist/snapshot.bin \ @@ -25,8 +24,10 @@ all-but-packages: \ \ dist/package.json \ dist/python \ + dist/python_cli_entry.js \ dist/python_stdlib.zip \ dist/test.html \ + dist/console.html \ dist/module_test.html \ @@ -252,6 +253,9 @@ dist/module_test.html: src/templates/module_test.html dist/python: src/templates/python cp $< $@ +dist/python_cli_entry.js: src/templates/python_cli_entry.js + cp $< $@ + .PHONY: dist/console.html dist/console.html: src/templates/console.html cp $< $@ @@ -312,7 +316,7 @@ $(CPYTHONLIB): emsdk/emsdk/.complete @date +"[%F %T] done building cpython..." -dist/pyodide-lock.json: FORCE +dist/pyodide-lock.json: make pyodide_build @date +"[%F %T] Building packages..." make -C packages diff --git a/README.md b/README.md index 03557fa39f6..45a5f12c526 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,11 @@ Pyodide is a Python distribution for the browser and Node.js based on WebAssembl Pyodide is a port of CPython to WebAssembly/[Emscripten](https://emscripten.org/). Pyodide makes it possible to install and run Python packages in the browser with -[micropip](https://micropip.pyodide.org/). Any pure -Python package with a wheel available on PyPi is supported. Many packages with C -extensions have also been ported for use with Pyodide. These include many -general-purpose packages such as regex, PyYAML, lxml and scientific Python -packages including NumPy, pandas, SciPy, Matplotlib, and scikit-learn. +[micropip](https://micropip.pyodide.org/). Any pure Python package with a wheel +available on PyPi is supported. Many packages with C, C++, and Rust extensions +have also been ported for use with Pyodide. These include many general-purpose +packages such as regex, PyYAML, and cryptography, and scientific Python packages +including NumPy, pandas, SciPy, Matplotlib, and scikit-learn. Pyodide comes with a robust Javascript ⟺ Python foreign function interface so that you can freely mix these two languages in your code with minimal friction. diff --git a/cpython/patches/0001-Public-pymain_run_python.patch b/cpython/patches/0001-Public-pymain_run_python.patch index 60848734306..ca4220f4172 100644 --- a/cpython/patches/0001-Public-pymain_run_python.patch +++ b/cpython/patches/0001-Public-pymain_run_python.patch @@ -1,4 +1,4 @@ -From c37e3894a271ee8b406ffc41a0ce6f5c248914ee Mon Sep 17 00:00:00 2001 +From f8a9e2d38252a032bdb617db396a047154f1bb6e Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Sun, 17 Jul 2022 14:40:39 +0100 Subject: [PATCH 1/9] Public pymain_run_python @@ -23,5 +23,5 @@ index b602272b78b..d06d3c926b9 100644 { PyObject *main_importer_path = NULL; -- -2.34.1 +2.48.1 diff --git a/cpython/patches/0002-Add-emscripten-platform-support-to-ctypes.util.find_.patch b/cpython/patches/0002-Add-emscripten-platform-support-to-ctypes.util.find_.patch index e9d2db722d5..376248eb700 100644 --- a/cpython/patches/0002-Add-emscripten-platform-support-to-ctypes.util.find_.patch +++ b/cpython/patches/0002-Add-emscripten-platform-support-to-ctypes.util.find_.patch @@ -1,4 +1,4 @@ -From 83ce578076b86b0231739b81ed9231bf8d9ca800 Mon Sep 17 00:00:00 2001 +From 74f68027172f2b646b8ab9a8d73380b12f864cb7 Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Fri, 2 Dec 2022 11:36:44 +0000 Subject: [PATCH 2/9] Add emscripten platform support to @@ -44,5 +44,5 @@ index c550883e7c7..c25c8f63e77 100644 # AIX has two styles of storing shared libraries # GNU auto_tools refer to these as svr4 and aix -- -2.34.1 +2.48.1 diff --git a/cpython/patches/0003-Allow-multiprocessing.connection-top-level-import.patch b/cpython/patches/0003-Allow-multiprocessing.connection-top-level-import.patch index e46b83ff58b..e4e11571550 100644 --- a/cpython/patches/0003-Allow-multiprocessing.connection-top-level-import.patch +++ b/cpython/patches/0003-Allow-multiprocessing.connection-top-level-import.patch @@ -1,4 +1,4 @@ -From da79a0ddc090407c1ea1d8e25da67931d1863642 Mon Sep 17 00:00:00 2001 +From 32155f0ed7043dde694b7910db46f4c97f2e41d9 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 19 Dec 2022 09:09:14 -0800 Subject: [PATCH 3/9] Allow multiprocessing.connection top level import @@ -26,5 +26,5 @@ index d0582e3cd54..b96b2454d3d 100644 from . import util -- -2.34.1 +2.48.1 diff --git a/cpython/patches/0004-Make-Emscripten-trampolines-work-with-JSPI.patch b/cpython/patches/0004-Make-Emscripten-trampolines-work-with-JSPI.patch index 877508ba8d7..5aa3612cf37 100644 --- a/cpython/patches/0004-Make-Emscripten-trampolines-work-with-JSPI.patch +++ b/cpython/patches/0004-Make-Emscripten-trampolines-work-with-JSPI.patch @@ -1,4 +1,4 @@ -From b86ebf295ad48ade5810032e83f98edb0e642abf Mon Sep 17 00:00:00 2001 +From a761e476e8abab81106ed4f8b4422732404b65d2 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Wed, 28 Jun 2023 10:46:19 -0700 Subject: [PATCH 4/9] Make Emscripten trampolines work with JSPI @@ -382,5 +382,5 @@ index d0d54050286..85c6d2f71ae 100644 ) AC_SUBST([PLATFORM_HEADERS]) -- -2.34.1 +2.48.1 diff --git a/cpython/patches/0005-Use-wasm-gc-based-call-adaptor-if-available.patch b/cpython/patches/0005-Use-wasm-gc-based-call-adaptor-if-available.patch index b4e1cc3ed18..312dc9ac100 100644 --- a/cpython/patches/0005-Use-wasm-gc-based-call-adaptor-if-available.patch +++ b/cpython/patches/0005-Use-wasm-gc-based-call-adaptor-if-available.patch @@ -1,4 +1,4 @@ -From 2d2732b55d57008545e818fc1b5ef4dca5625b9e Mon Sep 17 00:00:00 2001 +From 13370805f9652e70bc19447381443afde82c9b93 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 22 Oct 2024 15:16:03 +0200 Subject: [PATCH 5/9] Use wasm-gc based call adaptor if available @@ -408,5 +408,5 @@ index a7981bc4877..ab03729266c 100644 PyStatus -- -2.34.1 +2.48.1 diff --git a/cpython/patches/0006-Fix-LONG_BIT-constant-to-be-always-32bit.patch b/cpython/patches/0006-Fix-LONG_BIT-constant-to-be-always-32bit.patch index c061e663df3..bba3540f6c1 100644 --- a/cpython/patches/0006-Fix-LONG_BIT-constant-to-be-always-32bit.patch +++ b/cpython/patches/0006-Fix-LONG_BIT-constant-to-be-always-32bit.patch @@ -1,4 +1,4 @@ -From d9e6867695c02c07420d8a15d4ae1b96ea41f553 Mon Sep 17 00:00:00 2001 +From 6f2d305e8eba14fd3c49f54790f13c19fa41860b Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Fri, 12 Jan 2024 00:52:57 +0900 Subject: [PATCH 6/9] Fix LONG_BIT constant to be always 32bit @@ -29,5 +29,5 @@ index e2bac3bf504..ae1c1a40260 100644 #define LONG_BIT (8 * SIZEOF_LONG) #endif -- -2.34.1 +2.48.1 diff --git a/cpython/patches/0007-Warn-if-ZoneInfo-is-imported-without-tzdata.patch b/cpython/patches/0007-Warn-if-ZoneInfo-is-imported-without-tzdata.patch index 4fb94537adf..8d44ff9929f 100644 --- a/cpython/patches/0007-Warn-if-ZoneInfo-is-imported-without-tzdata.patch +++ b/cpython/patches/0007-Warn-if-ZoneInfo-is-imported-without-tzdata.patch @@ -1,4 +1,4 @@ -From a35c57ed8c7f2657a7a82d900184ebe649dedfdf Mon Sep 17 00:00:00 2001 +From 566d1c20df2370c4a47f21561086fb7e5f51299c Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Thu, 25 Jul 2024 14:28:57 +0200 Subject: [PATCH 7/9] Warn if ZoneInfo is imported without tzdata @@ -25,5 +25,5 @@ index 98cdfe37ca6..35d19eae9f0 100644 # to "we cannot find this key": # -- -2.34.1 +2.48.1 diff --git a/cpython/patches/0008-Add-call-to-JsProxy_GetMethod-to-help-remove-tempora.patch b/cpython/patches/0008-Add-call-to-JsProxy_GetMethod-to-help-remove-tempora.patch index 444be946823..976454cb36e 100644 --- a/cpython/patches/0008-Add-call-to-JsProxy_GetMethod-to-help-remove-tempora.patch +++ b/cpython/patches/0008-Add-call-to-JsProxy_GetMethod-to-help-remove-tempora.patch @@ -1,4 +1,4 @@ -From ff4534318481437542f2e49348033a00354a069d Mon Sep 17 00:00:00 2001 +From f09bfe83d146697b936799d97d8fe0110c61b86d Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Thu, 25 Jul 2024 14:41:37 +0200 Subject: [PATCH 8/9] Add call to `JsProxy_GetMethod` to help remove temporary @@ -56,5 +56,5 @@ index 6b2e0aeaab9..9240b33b08a 100644 *method = PyObject_GetAttr(obj, name); return 0; -- -2.34.1 +2.48.1 diff --git a/cpython/patches/0009-Skip-wasm-gc-on-iOS-Safari-where-it-s-broken.patch b/cpython/patches/0009-Skip-wasm-gc-on-iOS-Safari-where-it-s-broken.patch new file mode 100644 index 00000000000..5bb641cc089 --- /dev/null +++ b/cpython/patches/0009-Skip-wasm-gc-on-iOS-Safari-where-it-s-broken.patch @@ -0,0 +1,59 @@ +From c627686651a46118b2a653fa61673dfc158442d4 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=C5=81ukasz=20Langa?= +Date: Fri, 21 Feb 2025 19:24:41 +0100 +Subject: [PATCH 9/9] Skip wasm-gc on iOS Safari where it's broken + +As of iOS 18.3.1, enabling wasm-gc is making the interpreter fail to load. +Downstream pyodide issue: pyodide/pyodide#5428. + +macOS Safari 18.3 does not surface the issue. + +Confirmed on device that disabling this restores interpreter function. +--- + Python/emscripten_trampoline.c | 16 +++++++++++++--- + 1 file changed, 13 insertions(+), 3 deletions(-) + +diff --git a/Python/emscripten_trampoline.c b/Python/emscripten_trampoline.c +index e78a94e5e99..ff57f9e91d8 100644 +--- a/Python/emscripten_trampoline.c ++++ b/Python/emscripten_trampoline.c +@@ -79,7 +79,13 @@ EM_JS(CountArgsFunc, _PyEM_GetCountArgsPtr, (), { + // i32.const -1 + // ) + // ) +-addOnPreRun(() => { ++ ++function getPyEMCountArgsPtr() { ++ let isIOS = globalThis.navigator && /iPad|iPhone|iPod/.test(navigator.platform); ++ if (isIOS) { ++ return 0; ++ } ++ + // Try to initialize countArgsFunc + const code = new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, // \0asm magic number +@@ -151,15 +157,19 @@ addOnPreRun(() => { + 0x41, 0x7f, // i32.const -1 + 0x0b // end function + ]); +- let ptr = 0; + try { + const mod = new WebAssembly.Module(code); + const inst = new WebAssembly.Instance(mod, { e: { t: wasmTable } }); +- ptr = addFunction(inst.exports.f); ++ return addFunction(inst.exports.f); + } catch (e) { + // If something goes wrong, we'll null out _PyEM_CountFuncParams and fall + // back to the JS trampoline. ++ return 0; + } ++} ++ ++addOnPreRun(() => { ++ const ptr = getPyEMCountArgsPtr(); + Module._PyEM_CountArgsPtr = ptr; + const offset = HEAP32[__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET / 4]; + HEAP32[(__PyRuntime + offset) / 4] = ptr; +-- +2.48.1 + diff --git a/docs/index.rst b/docs/index.rst index da23ec8db31..129819af1a5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,11 +10,12 @@ What is Pyodide? Pyodide is a port of CPython to WebAssembly/`Emscripten `_. Pyodide makes it possible to install and run Python packages in the browser with -`micropip `_. Any pure -Python package with a wheel available on PyPI is supported. Many packages with C -extensions have also been ported for use with Pyodide. These include many -general-purpose packages such as regex, pyyaml, lxml and scientific Python -packages including numpy, pandas, scipy, matplotlib, and scikit-learn. +`micropip `_. Any +pure Python package with a wheel available on PyPi is supported. Many packages +with C, C++, and Rust extensions have also been ported for use with Pyodide. +These include many general-purpose packages such as regex, PyYAML, and +cryptography, and scientific Python packages including NumPy, pandas, SciPy, +Matplotlib, and scikit-learn. Pyodide comes with a robust Javascript ⟺ Python foreign function interface so that you can freely mix these two languages in your code with minimal @@ -43,8 +44,9 @@ Using Pyodide usage/quickstart.md usage/downloading-and-deploying.md usage/index.md - usage/loading-packages.md usage/accessing-files.md + usage/loading-packages.md + usage/building-and-testing-packages.md usage/wasm-constraints.md usage/type-conversions.md usage/keyboard-interrupts.md @@ -63,12 +65,12 @@ development process including making packages to support third party libraries. :maxdepth: 1 :caption: Development - development/building-from-sources.md development/new-packages.md - development/building-and-testing-packages.md - development/contributing.md + development/building-from-sources.md development/testing.md development/debugging.md + development/contributing.md + Project diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 49059d6f0b5..1f17f640e2b 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -24,12 +24,17 @@ myst: - {{ Fix }} `mountNativeFS` API now correctly propagates the error. {pr}`5434` - {{ Fix }} `registerJsModule()` now works with non-extensible JS objects, such as ES6 modules. {pr}`5452` +- {{ Fix }} Since 0.27.1, Pyodide has been broken in iOS because iOS ships + broken wasm-gc support. Pyodide feature detects whether the runtime supports + wasm-gc and uses it if it is present. Unfortunately, iOS passes the feature + detection but wasm-gc doesn't work as expected. {pr}`5445` ### Packages - Added `h3` 4.2.1 {pr}`5436` - Upgraded `narwhals` to 1.24.1 {pr}`5386` - Upgraded `rateslib` to 1.7.0 {pr}`5400` +- Upgraded Pyxel to 2.3.6 {pr}`5427` - Added `pcodec` 0.3.3 {pr}`5432` - {{ Breaking }} `matplotlib-pyodide` is not a default backend for matplotlib anymore. diff --git a/docs/development/building-and-testing-packages.md b/docs/usage/building-and-testing-packages.md similarity index 100% rename from docs/development/building-and-testing-packages.md rename to docs/usage/building-and-testing-packages.md diff --git a/pyproject.toml b/pyproject.toml index 8c9870e8122..0ed49f664a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -150,7 +150,7 @@ markers = [ [tool._pyodide] [tool.pyodide.build] -rust_toolchain = "nightly-2025-01-15" +rust_toolchain = "nightly-2025-02-01" [tool.codespell] ignore-words = 'tools/codespell_ignore_words.txt' diff --git a/src/js/emscripten-settings.ts b/src/js/emscripten-settings.ts index 737d80e97be..9df654f0893 100644 --- a/src/js/emscripten-settings.ts +++ b/src/js/emscripten-settings.ts @@ -17,6 +17,7 @@ export interface EmscriptenSettings { readonly print?: (a: string) => void; readonly printErr?: (a: string) => void; readonly onExit?: (code: number) => void; + readonly thisProgram?: string; readonly arguments: readonly string[]; readonly instantiateWasm?: ( imports: { [key: string]: any }, @@ -49,6 +50,7 @@ export function createSettings(config: ConfigType): EmscriptenSettings { onExit(code) { settings.exitCode = code; }, + thisProgram: config._sysExecutable, arguments: config.args, API: { config } as API, // Emscripten calls locateFile exactly one time with argument diff --git a/src/js/pyodide.ts b/src/js/pyodide.ts index 181843a6412..0396cc2bcd2 100644 --- a/src/js/pyodide.ts +++ b/src/js/pyodide.ts @@ -46,6 +46,7 @@ export type ConfigType = { stdout?: (msg: string) => void; stderr?: (msg: string) => void; jsglobals?: object; + _sysExecutable?: string; args: string[]; _node_mounts: string[]; env: { [key: string]: string }; @@ -139,6 +140,11 @@ export async function loadPyodide( * Default: ``globalThis`` */ jsglobals?: object; + /** + * Determine the value of ``sys.executable``. + * @ignore + */ + _sysExecutable?: string; /** * Command line arguments to pass to Python on startup. See `Python command * line interface options diff --git a/src/templates/python b/src/templates/python index ae066bd00fc..fa3ae47fcab 100755 --- a/src/templates/python +++ b/src/templates/python @@ -1,28 +1,6 @@ #!/usr/bin/env bash -":" /* << "EOF" -This file is a bash/node polyglot. This is needed for a few reasons: - -TODO: We don't support node < 18 anymore, so maybe we can remove this? - -1. In node 14 we must pass `--experimental-wasm-bigint`. In node >14 we cannot -pass --experimental-wasm-bigint - -2. Emscripten vendors node 14 so it is desirable not to require node >= 16 - -3. We could use a bash script in a separate file to determine the flags needed, -but virtualenv looks up the current file and uses it directly. So if we make -python.sh and have it invoke python.js, then the virtualenv will invoke python.js -directly without the `--experimental-wasm-bigint` flag and so the virtualenv won't -work with node 14. - -Keeping the bash script and the JavaScript in the same file makes sure that even -inside the virtualenv the proper shell code is executed. -*/ - -/* -EOF -# bash set -e + if [[ $1 == "-m" ]] && [[ $2 == "pip" ]]; then # redirect python -m pip to execute in host environment shift 1 @@ -46,154 +24,23 @@ process.stdout.write("--"); EOF )") -exec node "$ARGS" "$0" "$@" -*/ - - -const { loadPyodide } = require("./pyodide"); -const fs = require("fs"); - -/** - * Determine which native top level directories to mount into the Emscripten - * file system. - * - * This is a bit brittle, if the machine has a top level directory with certain - * names it is possible this could break. The most surprising one here is tmp, I - * am not sure why but if we link tmp then the process silently fails. - */ -function rootDirsToMount() { - const skipDirs = ["dev", "lib", "proc", "tmp"]; - return fs - .readdirSync("/") - .filter((dir) => !skipDirs.includes(dir)) - .map((dir) => "/" + dir); -} - -function dirsToMount() { - extra_mounts = process.env["_PYODIDE_EXTRA_MOUNTS"] || ""; - return rootDirsToMount().concat(extra_mounts.split(":").filter(s => s)) -} - -async function main() { - let args = process.argv.slice(2); - try { - py = await loadPyodide({ - args, - env: Object.assign({ - PYTHONINSPECT: "", - }, process.env, { HOME: process.cwd() }), - fullStdLib: false, - _node_mounts: dirsToMount(), - // Strip out messages written to stderr while loading - // After Pyodide is loaded we will replace stdstreams with setupStreams. - stderr(e) { - if ( - [ - "warning: no blob constructor, cannot create blobs with mimetypes", - "warning: no BlobBuilder", - ].includes(e.trim()) - ) { - return; - } - console.warn(e); - } - }); - } catch (e) { - if (e.constructor.name !== "ExitStatus") { - throw e; - } - // If the user passed `--help`, `--version`, or a set of command line - // arguments that is invalid in some way, we will exit here. - process.exit(e.status); - } - py.setStdout(); - py.setStderr(); - let sideGlobals = py.runPython("{}"); - function handleExit(code) { - if (code === undefined) { - code = 0; - } - if (py._module._Py_FinalizeEx() < 0) { - code = 120; - } - // It's important to call `process.exit` immediately after - // `_Py_FinalizeEx` because otherwise any asynchronous tasks still - // scheduled will segfault. - process.exit(code); - }; - sideGlobals.set("handleExit", handleExit); - - py.runPython( - ` - from pyodide._package_loader import SITE_PACKAGES, should_load_dynlib - from pyodide.ffi import to_js - import re - dynlibs_to_load = to_js([ - str(path) for path in SITE_PACKAGES.glob("**/*.so*") - if should_load_dynlib(path) - ]) - `, - { globals: sideGlobals } - ); - const dynlibs = sideGlobals.get("dynlibs_to_load"); - for (const dynlib of dynlibs) { - try { - await py._module.API.loadDynlib(dynlib); - } catch(e) { - console.error("Failed to load lib ", dynlib); - console.error(e); - } +# Macs come with FreeBSD coreutils which doesn't have the -s option +# so feature detect and work around it. +if which grealpath > /dev/null; then + # It has brew installed gnu core utils, use that + REALPATH="grealpath -s" +elif which realpath > /dev/null && realpath --version > /dev/null 2> /dev/null && realpath --version | grep GNU > /dev/null; then + # realpath points to GNU realpath so use it. + REALPATH="realpath -s" +else + # Shim for macs without GNU coreutils + abs_path () { + echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")" } - // Warning: this sounds like it might not do anything important, but it - // fills in the GOT. There can be segfaults if we leave it out. - // See https://github.com/emscripten-core/emscripten/issues/22052 - // TODO: Fix Emscripten so this isn't needed - py._module.reportUndefinedSymbols(); + REALPATH=abs_path +fi - py.runPython( - ` - import asyncio - # Keep the event loop alive until all tasks are finished, or SystemExit or - # KeyboardInterupt is raised. - loop = asyncio.get_event_loop() - # Make sure we don't run _no_in_progress_handler before we finish _run_main. - loop._in_progress += 1 - loop._no_in_progress_handler = handleExit - loop._system_exit_handler = handleExit - loop._keyboard_interrupt_handler = lambda: handleExit(130) - # Make shutil.get_terminal_size tell the terminal size accurately. - import shutil - from js.process import stdout - import os - def get_terminal_size(fallback=(80, 24)): - columns = getattr(stdout, "columns", None) - rows = getattr(stdout, "rows", None) - if columns is None: - columns = fallback[0] - if rows is None: - rows = fallback[1] - return os.terminal_size((columns, rows)) - shutil.get_terminal_size = get_terminal_size - `, - { globals: sideGlobals } - ); +RESOLVED_DIR=$(dirname $(realpath "$0")) - let errcode; - try { - errcode = py._module._run_main(); - } catch (e) { - if (e.constructor.name === "ExitStatus") { - process.exit(e.status); - } - py._api.fatal_error(e); - } - if (errcode) { - process.exit(errcode); - } - py.runPython("loop._decrement_in_progress()", { globals: sideGlobals }); -} -main().catch((e) => { - console.error(e); - process.exit(1); -}); +exec node "$ARGS" $RESOLVED_DIR/python_cli_entry.js --this-program="$($REALPATH "$0")" "$@" diff --git a/src/templates/python_cli_entry.js b/src/templates/python_cli_entry.js new file mode 100644 index 00000000000..abc525d6a1b --- /dev/null +++ b/src/templates/python_cli_entry.js @@ -0,0 +1,160 @@ +const { loadPyodide } = require("./pyodide"); +const fs = require("fs"); + +/** + * Determine which native top level directories to mount into the Emscripten + * file system. + * + * This is a bit brittle, if the machine has a top level directory with certain + * names it is possible this could break. The most surprising one here is tmp, I + * am not sure why but if we link tmp then the process silently fails. + */ +function rootDirsToMount() { + const skipDirs = ["dev", "lib", "proc", "tmp"]; + return fs + .readdirSync("/") + .filter((dir) => !skipDirs.includes(dir)) + .map((dir) => "/" + dir); +} + +function dirsToMount() { + extra_mounts = process.env["_PYODIDE_EXTRA_MOUNTS"] || ""; + return rootDirsToMount().concat(extra_mounts.split(":").filter((s) => s)); +} + +const thisProgramFlag = "--this-program="; +const thisProgramIndex = process.argv.findIndex((x) => + x.startsWith(thisProgramFlag), +); +const args = process.argv.slice(thisProgramIndex + 1); +const _sysExecutable = process.argv[thisProgramIndex].slice( + thisProgramFlag.length, +); + +async function main() { + try { + py = await loadPyodide({ + args, + _sysExecutable, + env: Object.assign( + { + PYTHONINSPECT: "", + }, + process.env, + { HOME: process.cwd() }, + ), + fullStdLib: false, + _node_mounts: dirsToMount(), + // Strip out messages written to stderr while loading + // After Pyodide is loaded we will replace stdstreams with setupStreams. + stderr(e) { + if ( + [ + "warning: no blob constructor, cannot create blobs with mimetypes", + "warning: no BlobBuilder", + ].includes(e.trim()) + ) { + return; + } + console.warn(e); + }, + }); + } catch (e) { + if (e.constructor.name !== "ExitStatus") { + throw e; + } + // If the user passed `--help`, `--version`, or a set of command line + // arguments that is invalid in some way, we will exit here. + process.exit(e.status); + } + py.setStdout(); + py.setStderr(); + let sideGlobals = py.runPython("{}"); + function handleExit(code) { + if (code === undefined) { + code = 0; + } + if (py._module._Py_FinalizeEx() < 0) { + code = 120; + } + // It's important to call `process.exit` immediately after + // `_Py_FinalizeEx` because otherwise any asynchronous tasks still + // scheduled will segfault. + process.exit(code); + } + sideGlobals.set("handleExit", handleExit); + + py.runPython( + ` + from pyodide._package_loader import SITE_PACKAGES, should_load_dynlib + from pyodide.ffi import to_js + import re + dynlibs_to_load = to_js([ + str(path) for path in SITE_PACKAGES.glob("**/*.so*") + if should_load_dynlib(path) + ]) + `, + { globals: sideGlobals }, + ); + const dynlibs = sideGlobals.get("dynlibs_to_load"); + for (const dynlib of dynlibs) { + try { + await py._module.API.loadDynlib(dynlib); + } catch (e) { + console.error("Failed to load lib ", dynlib); + console.error(e); + } + } + // Warning: this sounds like it might not do anything important, but it + // fills in the GOT. There can be segfaults if we leave it out. + // See https://github.com/emscripten-core/emscripten/issues/22052 + // TODO: Fix Emscripten so this isn't needed + py._module.reportUndefinedSymbols(); + + py.runPython( + ` + import asyncio + # Keep the event loop alive until all tasks are finished, or SystemExit or + # KeyboardInterupt is raised. + loop = asyncio.get_event_loop() + # Make sure we don't run _no_in_progress_handler before we finish _run_main. + loop._in_progress += 1 + loop._no_in_progress_handler = handleExit + loop._system_exit_handler = handleExit + loop._keyboard_interrupt_handler = lambda: handleExit(130) + + # Make shutil.get_terminal_size tell the terminal size accurately. + import shutil + from js.process import stdout + import os + def get_terminal_size(fallback=(80, 24)): + columns = getattr(stdout, "columns", None) + rows = getattr(stdout, "rows", None) + if columns is None: + columns = fallback[0] + if rows is None: + rows = fallback[1] + return os.terminal_size((columns, rows)) + shutil.get_terminal_size = get_terminal_size + `, + { globals: sideGlobals }, + ); + + let errcode; + try { + errcode = py._module._run_main(); + } catch (e) { + if (e.constructor.name === "ExitStatus") { + process.exit(e.status); + } + py._api.fatal_error(e); + } + if (errcode) { + process.exit(errcode); + } + py.runPython("loop._decrement_in_progress()", { globals: sideGlobals }); +} +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/src/tests/test_cmdline_runner.py b/src/tests/test_cmdline_runner.py index 34211d40883..af211dbc1dc 100644 --- a/src/tests/test_cmdline_runner.py +++ b/src/tests/test_cmdline_runner.py @@ -10,9 +10,15 @@ import pytest import pyodide -from pyodide_build.build_env import emscripten_version, get_pyodide_root +from pyodide_build.build_env import ( + emscripten_version, + get_build_environment_vars, + get_pyodide_root, +) from pyodide_build.xbuildenv import CrossBuildEnvManager +PYVERSION = get_build_environment_vars(get_pyodide_root())["PYVERSION"] + only_node = pytest.mark.xfail_browsers( chrome="node only", firefox="node only", safari="node only" ) @@ -39,7 +45,7 @@ def test_python_version(selenium): [script_path, "-V"], capture_output=True, encoding="utf8", check=False ) assert result.returncode == 0 - assert result.stdout.strip() == "Python " + sys.version.partition(" ")[0] + assert result.stdout.strip() == "Python " + PYVERSION assert result.stderr == "" @@ -249,7 +255,7 @@ def test_venv_version(selenium, venv): check=False, ) assert result.returncode == 0 - assert result.stdout.strip() == "Python " + sys.version.partition(" ")[0] + assert result.stdout.strip() == "Python " + PYVERSION assert result.stderr == "" @@ -538,3 +544,52 @@ def test_cpp_exceptions(selenium, venv): print(result.stderr) assert result.returncode == 1 assert "ImportError: oops" in result.stderr + + +@only_node +def test_pip_install_sys_platform_condition_kept(selenium, venv): + """impure Python package built with Pyodide""" + result = install_pkg(venv, "regex; sys_platform == 'emscripten'") + assert result.returncode == 0 + assert ( + clean_pkg_install_stdout(result.stdout) + == dedent( + """ + Looking in links: .../dist + Processing ./dist/regex-*-cpxxx-cpxxx-pyodide_*_wasm32.whl + Installing collected packages: regex + Successfully installed regex-* + """ + ).strip() + ) + + result = subprocess.run( + [ + venv / "bin/python", + "-c", + dedent( + r""" + import regex + m = regex.match(r"(?:(?P\w+) (?P\d+)\n)+", "one 1\ntwo 2\nthree 3\n") + print(m.capturesdict()) + """ + ), + ], + capture_output=True, + encoding="utf-8", + check=False, + ) + assert result.returncode == 0 + assert ( + result.stdout + == "{'word': ['one', 'two', 'three'], 'digits': ['1', '2', '3']}" + "\n" + ) + + +@only_node +def test_pip_install_sys_platform_condition_skipped(selenium, venv): + """impure Python package built with Pyodide""" + result = install_pkg(venv, "regex; sys_platform != 'emscripten'") + assert result.returncode == 0 + ignoring = """Ignoring regex: markers 'sys_platform != "emscripten"' don't match your environment""" + assert ignoring in result.stdout