From c1b7dc10fc5cc6890624463b69754600877bad1e Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 18 Mar 2025 10:23:41 +0100 Subject: [PATCH 01/28] Add more packages to build with Github actions --- .../workflows/build-linux-aarch64-wheels.yml | 2 +- .../workflows/build-linux-amd64-wheels.yml | 2 +- .../workflows/build-macos-aarch64-wheels.yml | 2 +- .../workflows/build-macos-amd64-wheels.yml | 2 +- .../workflows/build-windows-amd64-wheels.yml | 2 +- scripts/wheelbuilder/darwin/cryptography.sh | 48 +++++++ scripts/wheelbuilder/darwin/pandas.sh | 49 +++++++ scripts/wheelbuilder/linux/cryptography.sh | 42 ++++++ scripts/wheelbuilder/packages.txt | 129 +++++++++++++++++- 9 files changed, 272 insertions(+), 6 deletions(-) create mode 100755 scripts/wheelbuilder/darwin/cryptography.sh create mode 100755 scripts/wheelbuilder/darwin/pandas.sh create mode 100755 scripts/wheelbuilder/linux/cryptography.sh diff --git a/.github/workflows/build-linux-aarch64-wheels.yml b/.github/workflows/build-linux-aarch64-wheels.yml index 003e9e311b..5100902ecd 100644 --- a/.github/workflows/build-linux-aarch64-wheels.yml +++ b/.github/workflows/build-linux-aarch64-wheels.yml @@ -29,10 +29,10 @@ jobs: dnf install -y /usr/bin/patch - name: Checkout uses: actions/checkout@main + - uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Build wheels run: | python3 -m venv wheelbuilder_venv - export PIP_EXTRA_INDEX_URL=https://pypi.org/simple wheelbuilder_venv/bin/python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - name: Build wheels run: python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} diff --git a/.github/workflows/build-linux-amd64-wheels.yml b/.github/workflows/build-linux-amd64-wheels.yml index a8bffb78d2..6b20a4176d 100644 --- a/.github/workflows/build-linux-amd64-wheels.yml +++ b/.github/workflows/build-linux-amd64-wheels.yml @@ -27,10 +27,10 @@ jobs: dnf install -y /usr/bin/patch - name: Checkout uses: actions/checkout@main + - uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Build wheels run: | python3 -m venv wheelbuilder_venv - export PIP_EXTRA_INDEX_URL=https://pypi.org/simple wheelbuilder_venv/bin/python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - name: Store wheels uses: actions/upload-artifact@main diff --git a/.github/workflows/build-macos-aarch64-wheels.yml b/.github/workflows/build-macos-aarch64-wheels.yml index 08e8048541..d70f893991 100644 --- a/.github/workflows/build-macos-aarch64-wheels.yml +++ b/.github/workflows/build-macos-aarch64-wheels.yml @@ -18,13 +18,13 @@ jobs: steps: - name: Checkout uses: actions/checkout@main + - uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Setup Python uses: actions/setup-python@v5 with: python-version: 3.12 - name: Build wheels run: | - export PIP_EXTRA_INDEX_URL=https://pypi.org/simple python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - name: Store wheels uses: actions/upload-artifact@main diff --git a/.github/workflows/build-macos-amd64-wheels.yml b/.github/workflows/build-macos-amd64-wheels.yml index 3382d5f163..985b62fba2 100644 --- a/.github/workflows/build-macos-amd64-wheels.yml +++ b/.github/workflows/build-macos-amd64-wheels.yml @@ -18,13 +18,13 @@ jobs: steps: - name: Checkout uses: actions/checkout@main + - uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Setup Python uses: actions/setup-python@v5 with: python-version: 3.12 - name: Build wheels run: | - export PIP_EXTRA_INDEX_URL=https://pypi.org/simple python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - name: Store wheels uses: actions/upload-artifact@main diff --git a/.github/workflows/build-windows-amd64-wheels.yml b/.github/workflows/build-windows-amd64-wheels.yml index 8563dcb7de..aaf59986b3 100644 --- a/.github/workflows/build-windows-amd64-wheels.yml +++ b/.github/workflows/build-windows-amd64-wheels.yml @@ -19,6 +19,7 @@ jobs: - uses: ilammy/msvc-dev-cmd@v1 - name: Checkout uses: actions/checkout@main + - uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Setup Python uses: actions/setup-python@v5 with: @@ -26,7 +27,6 @@ jobs: - name: Build wheels run: | $env:PATH+=";C:\Program Files\Git\usr\bin" - $env:PIP_EXTRA_INDEX_URL="https://pypi.org/simple" python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - name: Store wheels uses: actions/upload-artifact@main diff --git a/scripts/wheelbuilder/darwin/cryptography.sh b/scripts/wheelbuilder/darwin/cryptography.sh new file mode 100755 index 0000000000..aa224941ef --- /dev/null +++ b/scripts/wheelbuilder/darwin/cryptography.sh @@ -0,0 +1,48 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + brew install libffi openssl +fi +export PKG_CONFIG_PATH=/opt/homebrew/opt/openssl/lib/pkgconfig +if [ -n "$1" ]; then + pip wheel "cryptography==$1" +else + pip wheel cryptography +fi diff --git a/scripts/wheelbuilder/darwin/pandas.sh b/scripts/wheelbuilder/darwin/pandas.sh new file mode 100755 index 0000000000..9217005d7b --- /dev/null +++ b/scripts/wheelbuilder/darwin/pandas.sh @@ -0,0 +1,49 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + brew install gcc openblas pkg-config + export PKG_CONFIG_PATH=/opt/homebrew/opt/openblas/lib/pkgconfig +fi +export FFLAGS=-fallow-argument-mismatch +if [ -n "$1" ]; then + pip wheel "pandas==$1" +else + pip wheel pandas +fi diff --git a/scripts/wheelbuilder/linux/cryptography.sh b/scripts/wheelbuilder/linux/cryptography.sh new file mode 100755 index 0000000000..ef0138706f --- /dev/null +++ b/scripts/wheelbuilder/linux/cryptography.sh @@ -0,0 +1,42 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + dnf install -y libffi-devel openssl-devel +fi diff --git a/scripts/wheelbuilder/packages.txt b/scripts/wheelbuilder/packages.txt index aa4de2a2f1..890c395feb 100644 --- a/scripts/wheelbuilder/packages.txt +++ b/scripts/wheelbuilder/packages.txt @@ -1,6 +1,133 @@ -numpy==1.26.4 +aiohappyeyeballs==2.4.6 +aiohttp==3.11.13 +aiosignal==1.3.2 +alembic==1.14.1 +annotated-types==0.7.0 +anyio==4.8.0 +asyncer==0.0.8 +attrs==25.1.0 +backoff==2.2.1 +bcrypt==4.2.1 +certifi==2025.1.31 +cffi==1.17.1 +charset-normalizer==3.4.1 +click==8.1.8 +colorama==0.4.6 +colorlog==6.9.0 +cryptography==44.0.1 +datasets==3.3.2 +deprecated==1.2.18 +dill==0.3.8 +diskcache==5.6.3 +distro==1.9.0 +dspy-ai==2.5.41 +dspy==2.5.41 +fastapi==0.115.6 +filelock==3.17.0 +frozenlist==1.5.0 +fsspec==2024.12.0 +fsspec[http]==2024.12.0 +googleapis-common-protos==1.56.1 +greenlet==3.1.1 +grpcio==1.70.0 +h11==0.14.0 +httpcore==1.0.7 httptools==0.6.1 +httpx==0.28.1 +huggingface-hub==0.29.1 +idna==3.10 +importlib-metadata==8.5.0 +javalang==0.13.0 +jinja2==3.1.5 +jiter==0.9.0 +joblib==1.4.2 +json-repair==0.39.1 +json5==0.10.0 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2024.10.1 +jsonschema==4.23.0 kiwisolver==1.4.5 +langchain-core==0.3.41 +langgraph-checkpoint==2.0.17 +langgraph-sdk==0.1.55 +langgraph==0.2.76 +langsmith==0.3.12 +levenshtein==0.26.1 +litellm==1.51.0 +magicattr==0.1.6 +mako==1.3.9 +markupsafe==3.0.2 +msgpack==1.1.0 +multidict==6.1.0 +multiprocess==0.70.16 +nest-asyncio==1.6.0 +numpy==1.26.4 +numpy==2.2.3 +openai==1.64.0 +openinference-instrumentation-dspy==0.1.9 +openinference-instrumentation-litellm==0.1.6 +openinference-instrumentation==0.1.23 +openinference-semantic-conventions==0.1.14 +opentelemetry-api==1.30.0 +opentelemetry-exporter-jaeger-proto-grpc==1.21.0 +opentelemetry-exporter-jaeger-thrift==1.21.0 +opentelemetry-exporter-jaeger==1.21.0 +opentelemetry-exporter-otlp-proto-common==1.30.0 +opentelemetry-exporter-otlp-proto-grpc==1.30.0 +opentelemetry-exporter-otlp-proto-http==1.30.0 +opentelemetry-exporter-otlp==1.30.0 +opentelemetry-exporter-prometheus==0.51b0 +opentelemetry-instrumentation==0.51b0 +opentelemetry-propagator-b3==1.30.0 +opentelemetry-proto==1.30.0 +opentelemetry-sdk==1.30.0 +opentelemetry-semantic-conventions==0.51b0 +optuna==4.2.1 +oracledb==2.5.1 +orjson==3.10.15 +packaging==24.2 +pandas==2.2.3 +paramiko==3.5.1 +prometheus-client==0.21.1 +propcache==0.3.0 +protobuf==5.29.3 psutil==5.9.8 +pyarrow==19.0.1 +pycparser==2.22 +pydantic-core==2.27.1 +pydantic==2.10.1 +pynacl==1.5.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 +pytz==2025.1 +pyyaml==6.0.2 +rapidfuzz==3.12.1 +referencing==0.36.2 +regex==2024.11.6 +requests-toolbelt==1.0.0 +requests==2.32.3 +rpds-py==0.23.1 +setuptools==76.0.0 +six==1.17.0 +sniffio==1.3.1 +sqlalchemy==2.0.38 +sshtunnel==0.4.0 +starlette==0.41.3 +tenacity==9.0.0 +thrift==0.21.0 +tiktoken==0.9.0 +tokenizers==0.21.0 +tqdm==4.67.1 +typing-extensions==4.12.2 +tzdata==2025.1 +ujson==5.10.0 ujson==5.10.0 +urllib3==2.3.0 +uvicorn==0.34.0 +wrapt==1.17.2 xxhash==3.4.1 +xxhash==3.5.0 +yarl==1.18.3 +zipp==3.21.0 +zstandard==0.23.0 From 9411925e95751654de9a63e975ffcf9460411cb8 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 18 Mar 2025 11:12:05 +0100 Subject: [PATCH 02/28] Run wheel scripts in subshell --- scripts/wheelbuilder/build_wheels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 605032e50d..2c1a008d7b 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -136,7 +136,7 @@ def build_wheels(pip): if sys.platform == "win32": cmd = [script, version] # Python's subprocess.py does the quoting we need else: - cmd = f"{shlex.quote(script)} {version}" + cmd = f"{os.environ.get('SHELL', '/bin/sh')} {shlex.quote(script)} {version}" subprocess.check_call(cmd, shell=True, env=env) if not len(glob("*.whl")) > whl_count: print("Building wheel for", name, version, "after", script, "did not", flush=True) From 6ee47fe34480ab1904cfd0648d758ac538792dfd Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 18 Mar 2025 20:09:20 +0100 Subject: [PATCH 03/28] Iterate over wheels to build until none can be built anymore This helps with wheels depending on other wheels when the scripts for these dependencies have not run, yet. This way we will eventually either be able to build the dependencies, or we will reach a point where no more wheels can be built, and then we give up. --- scripts/wheelbuilder/build_wheels.py | 61 +++++++++++++++++----------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 2c1a008d7b..fdb4161a78 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -121,29 +121,42 @@ def build_wheels(pip): available_scripts = {s.lower(): s for s in os.listdir(scriptdir)} else: available_scripts = {} - for spec in packages_to_build: - name, version = spec.split("==") - whl_count = len(glob("*.whl")) - script = f"{name}.{version}.{script_ext}".lower() - if script not in available_scripts: - script = f"{name}.{script_ext}".lower() - if script in available_scripts: - script = join(scriptdir, available_scripts[script]) - env = os.environ.copy() - env["PATH"] = abspath(dirname(pip)) + os.pathsep + env["PATH"] - env["VIRTUAL_ENV"] = abspath(dirname(dirname(pip))) - print("Building", name, version, "with", script, flush=True) - if sys.platform == "win32": - cmd = [script, version] # Python's subprocess.py does the quoting we need - else: - cmd = f"{os.environ.get('SHELL', '/bin/sh')} {shlex.quote(script)} {version}" - subprocess.check_call(cmd, shell=True, env=env) - if not len(glob("*.whl")) > whl_count: - print("Building wheel for", name, version, "after", script, "did not", flush=True) - subprocess.check_call([pip, "wheel", spec]) - else: + remaining_packages = 0 + while remaining_packages != len(packages_to_build): + remaining_packages = len(packages_to_build) + for spec in packages_to_build.copy(): + name, version = spec.split("==") + whl_count = len(glob("*.whl")) + script = f"{name}.{version}.{script_ext}".lower() + if script not in available_scripts: + script = f"{name}.{script_ext}".lower() + if script in available_scripts: + script = join(scriptdir, available_scripts[script]) + env = os.environ.copy() + env["PATH"] = abspath(dirname(pip)) + os.pathsep + env["PATH"] + env["VIRTUAL_ENV"] = abspath(dirname(dirname(pip))) + print("Building", name, version, "with", script, flush=True) + if sys.platform == "win32": + cmd = [script, version] # Python's subprocess.py does the quoting we need + else: + cmd = f"{os.environ.get('SHELL', '/bin/sh')} {shlex.quote(script)} {version}" + p = subprocess.run(cmd, shell=True, env=env) + if p.returncode != 0: + continue + if len(glob("*.whl")) > whl_count: + packages_to_build.remove(spec) + continue + print(script, "did not build a wheel, we will do so now", flush=True) print("Building", name, version, flush=True) - subprocess.check_call([pip, "wheel", spec]) + p = subprocess.run([pip, "wheel", spec]) + if p.returncode == 0: + packages_to_build.remove(spec) + if packages_to_build: + print("Failed to build all packages, the following packages failed") + print(packages_to_build) + return False + else: + return True def repair_wheels(): @@ -188,5 +201,7 @@ def repair_wheels(): download(args.graalpy_url, outpath) extract(outpath) pip = create_venv() - build_wheels(pip) + success = build_wheels(pip) repair_wheels() + if not success: + sys.exit(1) From 7657ccb6eff30d22e4128cd9930ef4e0784239e1 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 18 Mar 2025 21:05:26 +0100 Subject: [PATCH 04/28] Only attempt to repair graalpy platform specific wheels --- scripts/wheelbuilder/build_wheels.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index fdb4161a78..44f9e14026 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -160,6 +160,7 @@ def build_wheels(pip): def repair_wheels(): + whls = glob("*graalpy*.whl") if sys.platform == "win32": ensure_installed("delvewheel") env = os.environ.copy() @@ -175,19 +176,19 @@ def repair_wheels(): "python-native.dll", "-w", "wheelhouse", - *glob("*.whl"), + *whls, ], env=env, ) elif sys.platform == "linux": ensure_installed("auditwheel") subprocess.check_call( - [join(dirname(sys.executable), "auditwheel"), "repair", "-w", "wheelhouse", *glob("*.whl")] + [join(dirname(sys.executable), "auditwheel"), "repair", "-w", "wheelhouse", *whls] ) elif sys.platform == "darwin": ensure_installed("delocate") subprocess.check_call( - [join(dirname(sys.executable), "delocate-wheel"), "-v", "-w", "wheelhouse", *glob("*.whl")] + [join(dirname(sys.executable), "delocate-wheel"), "-v", "-w", "wheelhouse", *whls] ) From 2d6f0a74793f3a7a6848891c54ce8dbc305f26dc Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Wed, 19 Mar 2025 09:17:10 +0100 Subject: [PATCH 05/28] Fix multiple wheel build recipes --- .../workflows/build-linux-aarch64-wheels.yml | 2 + .../workflows/build-linux-amd64-wheels.yml | 2 + .../workflows/build-macos-aarch64-wheels.yml | 2 + .../workflows/build-macos-amd64-wheels.yml | 2 + .../workflows/build-windows-amd64-wheels.yml | 2 + scripts/wheelbuilder/linux/lxml.sh | 42 ++++++++++++++++++ scripts/wheelbuilder/linux/numpy.1.23.2.sh | 43 +++++++++++++++++++ scripts/wheelbuilder/linux/pynacl.sh | 42 ++++++++++++++++++ scripts/wheelbuilder/linux/zstandard.sh | 42 ++++++++++++++++++ 9 files changed, 179 insertions(+) create mode 100644 scripts/wheelbuilder/linux/lxml.sh create mode 100644 scripts/wheelbuilder/linux/numpy.1.23.2.sh create mode 100644 scripts/wheelbuilder/linux/pynacl.sh create mode 100644 scripts/wheelbuilder/linux/zstandard.sh diff --git a/.github/workflows/build-linux-aarch64-wheels.yml b/.github/workflows/build-linux-aarch64-wheels.yml index 5100902ecd..06fc6a3b33 100644 --- a/.github/workflows/build-linux-aarch64-wheels.yml +++ b/.github/workflows/build-linux-aarch64-wheels.yml @@ -30,6 +30,8 @@ jobs: - name: Checkout uses: actions/checkout@main - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" - name: Build wheels run: | python3 -m venv wheelbuilder_venv diff --git a/.github/workflows/build-linux-amd64-wheels.yml b/.github/workflows/build-linux-amd64-wheels.yml index 6b20a4176d..117da70702 100644 --- a/.github/workflows/build-linux-amd64-wheels.yml +++ b/.github/workflows/build-linux-amd64-wheels.yml @@ -28,6 +28,8 @@ jobs: - name: Checkout uses: actions/checkout@main - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" - name: Build wheels run: | python3 -m venv wheelbuilder_venv diff --git a/.github/workflows/build-macos-aarch64-wheels.yml b/.github/workflows/build-macos-aarch64-wheels.yml index d70f893991..c727c720c9 100644 --- a/.github/workflows/build-macos-aarch64-wheels.yml +++ b/.github/workflows/build-macos-aarch64-wheels.yml @@ -19,6 +19,8 @@ jobs: - name: Checkout uses: actions/checkout@main - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" - name: Setup Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/build-macos-amd64-wheels.yml b/.github/workflows/build-macos-amd64-wheels.yml index 985b62fba2..5bff458ad1 100644 --- a/.github/workflows/build-macos-amd64-wheels.yml +++ b/.github/workflows/build-macos-amd64-wheels.yml @@ -19,6 +19,8 @@ jobs: - name: Checkout uses: actions/checkout@main - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" - name: Setup Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/build-windows-amd64-wheels.yml b/.github/workflows/build-windows-amd64-wheels.yml index aaf59986b3..ccda631e8f 100644 --- a/.github/workflows/build-windows-amd64-wheels.yml +++ b/.github/workflows/build-windows-amd64-wheels.yml @@ -20,6 +20,8 @@ jobs: - name: Checkout uses: actions/checkout@main - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" - name: Setup Python uses: actions/setup-python@v5 with: diff --git a/scripts/wheelbuilder/linux/lxml.sh b/scripts/wheelbuilder/linux/lxml.sh new file mode 100644 index 0000000000..5335db4335 --- /dev/null +++ b/scripts/wheelbuilder/linux/lxml.sh @@ -0,0 +1,42 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + dnf install -y libxml2-devel libxslt-devel +fi diff --git a/scripts/wheelbuilder/linux/numpy.1.23.2.sh b/scripts/wheelbuilder/linux/numpy.1.23.2.sh new file mode 100644 index 0000000000..a9378be59d --- /dev/null +++ b/scripts/wheelbuilder/linux/numpy.1.23.2.sh @@ -0,0 +1,43 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + dnf install -y gcc-toolset-9 gcc-toolset-9-gcc-gfortran openblas-devel + scl enable gcc-toolset-9 'pip wheel "numpy==1.23.2"' +fi diff --git a/scripts/wheelbuilder/linux/pynacl.sh b/scripts/wheelbuilder/linux/pynacl.sh new file mode 100644 index 0000000000..f4e5359ec6 --- /dev/null +++ b/scripts/wheelbuilder/linux/pynacl.sh @@ -0,0 +1,42 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + dnf install -y libffi-devel +fi diff --git a/scripts/wheelbuilder/linux/zstandard.sh b/scripts/wheelbuilder/linux/zstandard.sh new file mode 100644 index 0000000000..f4e5359ec6 --- /dev/null +++ b/scripts/wheelbuilder/linux/zstandard.sh @@ -0,0 +1,42 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + dnf install -y libffi-devel +fi From 2684553fe05da2af437c3946b2a67cbeb148b5b0 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Wed, 19 Mar 2025 15:26:17 +0100 Subject: [PATCH 06/28] Fix errors with rust toolchain and act, add 1.24.3 script for numpy --- .../workflows/build-linux-aarch64-wheels.yml | 1 + .../workflows/build-linux-amd64-wheels.yml | 1 + .../workflows/build-macos-aarch64-wheels.yml | 1 + .../workflows/build-macos-amd64-wheels.yml | 1 + .../workflows/build-windows-amd64-wheels.yml | 1 + scripts/wheelbuilder/linux/numpy.1.24.3.sh | 43 +++++++++++++++++++ 6 files changed, 48 insertions(+) create mode 100644 scripts/wheelbuilder/linux/numpy.1.24.3.sh diff --git a/.github/workflows/build-linux-aarch64-wheels.yml b/.github/workflows/build-linux-aarch64-wheels.yml index 06fc6a3b33..b8943f9a3d 100644 --- a/.github/workflows/build-linux-aarch64-wheels.yml +++ b/.github/workflows/build-linux-aarch64-wheels.yml @@ -32,6 +32,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" + cache: false - name: Build wheels run: | python3 -m venv wheelbuilder_venv diff --git a/.github/workflows/build-linux-amd64-wheels.yml b/.github/workflows/build-linux-amd64-wheels.yml index 117da70702..f4da5d1d15 100644 --- a/.github/workflows/build-linux-amd64-wheels.yml +++ b/.github/workflows/build-linux-amd64-wheels.yml @@ -30,6 +30,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" + cache: false - name: Build wheels run: | python3 -m venv wheelbuilder_venv diff --git a/.github/workflows/build-macos-aarch64-wheels.yml b/.github/workflows/build-macos-aarch64-wheels.yml index c727c720c9..15937f2313 100644 --- a/.github/workflows/build-macos-aarch64-wheels.yml +++ b/.github/workflows/build-macos-aarch64-wheels.yml @@ -21,6 +21,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" + cache: false - name: Setup Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/build-macos-amd64-wheels.yml b/.github/workflows/build-macos-amd64-wheels.yml index 5bff458ad1..56f1e397f6 100644 --- a/.github/workflows/build-macos-amd64-wheels.yml +++ b/.github/workflows/build-macos-amd64-wheels.yml @@ -21,6 +21,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" + cache: false - name: Setup Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/build-windows-amd64-wheels.yml b/.github/workflows/build-windows-amd64-wheels.yml index ccda631e8f..069070b982 100644 --- a/.github/workflows/build-windows-amd64-wheels.yml +++ b/.github/workflows/build-windows-amd64-wheels.yml @@ -22,6 +22,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" + cache: false - name: Setup Python uses: actions/setup-python@v5 with: diff --git a/scripts/wheelbuilder/linux/numpy.1.24.3.sh b/scripts/wheelbuilder/linux/numpy.1.24.3.sh new file mode 100644 index 0000000000..7d50924cea --- /dev/null +++ b/scripts/wheelbuilder/linux/numpy.1.24.3.sh @@ -0,0 +1,43 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + dnf install -y gcc-toolset-9 gcc-toolset-9-gcc-gfortran openblas-devel + scl enable gcc-toolset-9 'pip wheel "numpy==1.24.3"' +fi From 417e3510585d99ee2556b17b39b99f4ad1a86ff7 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Wed, 19 Mar 2025 16:55:53 +0100 Subject: [PATCH 07/28] Add patches for numpy 2.2.3 and extend pydantic patch application --- .../lib-graalpython/patches/metadata.toml | 12 ++ .../lib-graalpython/patches/numpy-2.2.3.patch | 119 ++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 graalpython/lib-graalpython/patches/numpy-2.2.3.patch diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index 55e95f5619..7aa759cc93 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -301,6 +301,13 @@ patch = 'numpy-2.0.0.patch' license = 'BSD-3-Clause' dist-type = 'sdist' +[[numpy.rules]] +version = '== 2.2.3' +patch = 'numpy-2.2.3.patch' +license = 'BSD-3-Clause' +dist-type = 'sdist' +install-priority = 0 + [[numpy.rules]] version = '== 1.26.4' patch = 'numpy-1.26.4.patch' @@ -484,6 +491,11 @@ version = '== 2.27.1' patch = 'pydantic-core-2.27.1.patch' license = 'MIT' +[[pydantic-core.rules]] +version = '== 2.27.2' +patch = 'pydantic-core-2.27.1.patch' +license = 'MIT' + [[pydantic-core.rules]] version = '== 2.26.0' patch = 'pydantic-core-2.26.0.patch' diff --git a/graalpython/lib-graalpython/patches/numpy-2.2.3.patch b/graalpython/lib-graalpython/patches/numpy-2.2.3.patch new file mode 100644 index 0000000000..2250708aee --- /dev/null +++ b/graalpython/lib-graalpython/patches/numpy-2.2.3.patch @@ -0,0 +1,119 @@ +diff --git a/numpy/_core/include/numpy/ndarrayobject.h b/numpy/_core/include/numpy/ndarrayobject.h +index f06bafe5b5..52063a7d16 100644 +--- a/numpy/_core/include/numpy/ndarrayobject.h ++++ b/numpy/_core/include/numpy/ndarrayobject.h +@@ -220,7 +220,7 @@ NPY_TITLE_KEY_check(PyObject *key, PyObject *value) + if (key == title) { + return 1; + } +-#ifdef PYPY_VERSION ++#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) + /* + * On PyPy, dictionary keys do not always preserve object identity. + * Fall back to comparison by value. +diff --git a/numpy/_core/src/multiarray/compiled_base.c b/numpy/_core/src/multiarray/compiled_base.c +index 48524aff4d..e55e2479e1 100644 +--- a/numpy/_core/src/multiarray/compiled_base.c ++++ b/numpy/_core/src/multiarray/compiled_base.c +@@ -1455,6 +1455,7 @@ fail: + NPY_NO_EXPORT PyObject * + arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *const *args, Py_ssize_t len_args) + { ++#if 0 // GraalPy change + PyObject *obj; + PyObject *str; + const char *docstr; +@@ -1559,6 +1560,7 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *const *args, Py_ssize_t + } + + #undef _ADDDOC ++#endif // GraalPy change + + Py_RETURN_NONE; + } +diff --git a/numpy/_core/src/multiarray/shape.c b/numpy/_core/src/multiarray/shape.c +index 340fe7289a..8f13674383 100644 +--- a/numpy/_core/src/multiarray/shape.c ++++ b/numpy/_core/src/multiarray/shape.c +@@ -97,6 +97,11 @@ PyArray_Resize(PyArrayObject *self, PyArray_Dims *newshape, int refcheck, + "cannot resize an array with refcheck=True on PyPy.\n" + "Use the np.resize function or refcheck=False"); + return NULL; ++#elif defined(GRAALVM_PYTHON) ++ PyErr_SetString(PyExc_ValueError, ++ "cannot resize an array with refcheck=True on GraalPy.\n" ++ "Use the np.resize function or refcheck=False"); ++ return NULL; + #else + refcnt = Py_REFCNT(self); + #endif /* PYPY_VERSION */ +diff --git a/numpy/_core/src/multiarray/stringdtype/dtype.c b/numpy/_core/src/multiarray/stringdtype/dtype.c +index cb8265dd3d..0dabb49346 100644 +--- a/numpy/_core/src/multiarray/stringdtype/dtype.c ++++ b/numpy/_core/src/multiarray/stringdtype/dtype.c +@@ -841,7 +841,7 @@ init_string_dtype(void) + }; + + /* Loaded dynamically, so needs to be set here: */ +- Py_TYPE(((PyObject *)&PyArray_StringDType)) = &PyArrayDTypeMeta_Type; ++ Py_SET_TYPE((PyObject *)&PyArray_StringDType, &PyArrayDTypeMeta_Type); + ((PyTypeObject *)&PyArray_StringDType)->tp_base = &PyArrayDescr_Type; + if (PyType_Ready((PyTypeObject *)&PyArray_StringDType) < 0) { + return -1; +diff --git a/numpy/_core/src/multiarray/temp_elide.c b/numpy/_core/src/multiarray/temp_elide.c +index 662a2fa52b..791ede8c1a 100644 +--- a/numpy/_core/src/multiarray/temp_elide.c ++++ b/numpy/_core/src/multiarray/temp_elide.c +@@ -58,7 +58,7 @@ + * supported too by using the appropriate Windows APIs. + */ + +-#if defined HAVE_BACKTRACE && defined HAVE_DLFCN_H && ! defined PYPY_VERSION ++#if defined HAVE_BACKTRACE && defined HAVE_DLFCN_H && ! defined PYPY_VERSION && !defined(GRAALVM_PYTHON) + + #include + +diff --git a/numpy/_core/src/npymath/ieee754.c.src b/numpy/_core/src/npymath/ieee754.c.src +index 8fccc9a69b..3bb9cf0d58 100644 +--- a/numpy/_core/src/npymath/ieee754.c.src ++++ b/numpy/_core/src/npymath/ieee754.c.src +@@ -362,6 +362,11 @@ int npy_get_floatstatus_barrier(char* param) + * By using a volatile, the compiler cannot reorder this call + */ + if (param != NULL) { ++ // GraalPy change: the pointer needs to be dereferenced to establish ++ // a data dependency to to ensure the compiler won't reorder the call ++ if (points_to_py_handle_space(param)) { ++ param = (char*)pointer_to_stub(param); ++ } + volatile char NPY_UNUSED(c) = *(char*)param; + } + +diff --git a/numpy/_core/src/npymath/ieee754.cpp b/numpy/_core/src/npymath/ieee754.cpp +index 1c59bf300b..519fabc113 100644 +--- a/numpy/_core/src/npymath/ieee754.cpp ++++ b/numpy/_core/src/npymath/ieee754.cpp +@@ -428,6 +428,11 @@ npy_get_floatstatus_barrier(char *param) + * By using a volatile, the compiler cannot reorder this call + */ + if (param != NULL) { ++ // GraalPy change: the pointer needs to be dereferenced to establish ++ // a data dependency to to ensure the compiler won't reorder the call ++ if (points_to_py_handle_space(param)) { ++ param = (char*)pointer_to_stub(param); ++ } + volatile char NPY_UNUSED(c) = *(char *)param; + } + +diff --git a/vendored-meson/meson/mesonbuild/utils/universal.py b/vendored-meson/meson/mesonbuild/utils/universal.py +index 6aee268ee..539be8bab 100644 +--- a/vendored-meson/meson/mesonbuild/utils/universal.py ++++ b/vendored-meson/meson/mesonbuild/utils/universal.py +@@ -728,6 +728,7 @@ def windows_detect_native_arch() -> str: + """ + if sys.platform != 'win32': + return '' ++ return 'amd64' # Workaround for GraalPy bug on Windows with kernel32.GetCurrentProcess() + try: + import ctypes + process_arch = ctypes.c_ushort() From f5ab8fbb3a6a445db03d8a51eb85d05696c47667 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 20 Mar 2025 09:45:10 +0100 Subject: [PATCH 08/28] Fix pyarrow 0.19.1 wheel building and build sequentially, parallel OOMs on a 16GB container --- .../workflows/build-linux-aarch64-wheels.yml | 1 + .../workflows/build-linux-amd64-wheels.yml | 1 + .../workflows/build-macos-aarch64-wheels.yml | 1 + .../workflows/build-macos-amd64-wheels.yml | 1 + .../workflows/build-windows-amd64-wheels.yml | 2 + .../lib-graalpython/patches/metadata.toml | 5 + .../patches/pyarrow-12.0.0.patch | 2 +- .../patches/pyarrow-15.0.0.patch | 2 +- .../patches/pyarrow-19.0.0.patch | 26 ++- .../patches/pyarrow-19.0.1.patch | 188 ++++++++++++++++++ scripts/wheelbuilder/linux/pyarrow.sh | 58 ++++++ 11 files changed, 284 insertions(+), 3 deletions(-) create mode 100644 graalpython/lib-graalpython/patches/pyarrow-19.0.1.patch create mode 100644 scripts/wheelbuilder/linux/pyarrow.sh diff --git a/.github/workflows/build-linux-aarch64-wheels.yml b/.github/workflows/build-linux-aarch64-wheels.yml index b8943f9a3d..3aa29dbe0b 100644 --- a/.github/workflows/build-linux-aarch64-wheels.yml +++ b/.github/workflows/build-linux-aarch64-wheels.yml @@ -35,6 +35,7 @@ jobs: cache: false - name: Build wheels run: | + export PIP_GRAALPY_PATCHES_URL="${GITHUB_WORKSPACE}/graalpython/lib-graalpython/patches" python3 -m venv wheelbuilder_venv wheelbuilder_venv/bin/python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - name: Build wheels diff --git a/.github/workflows/build-linux-amd64-wheels.yml b/.github/workflows/build-linux-amd64-wheels.yml index f4da5d1d15..03578196c4 100644 --- a/.github/workflows/build-linux-amd64-wheels.yml +++ b/.github/workflows/build-linux-amd64-wheels.yml @@ -33,6 +33,7 @@ jobs: cache: false - name: Build wheels run: | + export PIP_GRAALPY_PATCHES_URL="${GITHUB_WORKSPACE}/graalpython/lib-graalpython/patches" python3 -m venv wheelbuilder_venv wheelbuilder_venv/bin/python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - name: Store wheels diff --git a/.github/workflows/build-macos-aarch64-wheels.yml b/.github/workflows/build-macos-aarch64-wheels.yml index 15937f2313..0e74bcea44 100644 --- a/.github/workflows/build-macos-aarch64-wheels.yml +++ b/.github/workflows/build-macos-aarch64-wheels.yml @@ -28,6 +28,7 @@ jobs: python-version: 3.12 - name: Build wheels run: | + export PIP_GRAALPY_PATCHES_URL="${GITHUB_WORKSPACE}/graalpython/lib-graalpython/patches" python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - name: Store wheels uses: actions/upload-artifact@main diff --git a/.github/workflows/build-macos-amd64-wheels.yml b/.github/workflows/build-macos-amd64-wheels.yml index 56f1e397f6..b6bd9a4226 100644 --- a/.github/workflows/build-macos-amd64-wheels.yml +++ b/.github/workflows/build-macos-amd64-wheels.yml @@ -28,6 +28,7 @@ jobs: python-version: 3.12 - name: Build wheels run: | + export PIP_GRAALPY_PATCHES_URL="${GITHUB_WORKSPACE}/graalpython/lib-graalpython/patches" python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - name: Store wheels uses: actions/upload-artifact@main diff --git a/.github/workflows/build-windows-amd64-wheels.yml b/.github/workflows/build-windows-amd64-wheels.yml index 069070b982..782524ed7b 100644 --- a/.github/workflows/build-windows-amd64-wheels.yml +++ b/.github/workflows/build-windows-amd64-wheels.yml @@ -30,6 +30,8 @@ jobs: - name: Build wheels run: | $env:PATH+=";C:\Program Files\Git\usr\bin" + $env:PIP_GRAALPY_PATCHES_URL=$env:GITHUB_WORKSPACE + $env:PIP_GRAALPY_PATCHES_URL+="/graalpython/lib-graalpython/patches" python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - name: Store wheels uses: actions/upload-artifact@main diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index 7aa759cc93..7f6628904c 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -455,6 +455,11 @@ version = '== 19.0.0' patch = 'pyarrow-19.0.0.patch' license = 'Apache-2.0' +[[pyarrow.rules]] +version = '== 19.0.1' +patch = 'pyarrow-19.0.1.patch' +license = 'Apache-2.0' + [[pybind11.rules]] # Upstreamed install-priority = 0 diff --git a/graalpython/lib-graalpython/patches/pyarrow-12.0.0.patch b/graalpython/lib-graalpython/patches/pyarrow-12.0.0.patch index 0051000a1c..b82828a56d 100644 --- a/graalpython/lib-graalpython/patches/pyarrow-12.0.0.patch +++ b/graalpython/lib-graalpython/patches/pyarrow-12.0.0.patch @@ -115,7 +115,7 @@ index 0000000..c9826ce + '-DARROW_WITH_BROTLI=ON', + ]) + subprocess.check_call([ -+ 'cmake', '--build', str(build_dir), '--parallel', ++ 'cmake', '--build', str(build_dir), + ]) + subprocess.check_call([ + 'cmake', '--install', str(build_dir), diff --git a/graalpython/lib-graalpython/patches/pyarrow-15.0.0.patch b/graalpython/lib-graalpython/patches/pyarrow-15.0.0.patch index 727ca72f9a..607b6dd982 100644 --- a/graalpython/lib-graalpython/patches/pyarrow-15.0.0.patch +++ b/graalpython/lib-graalpython/patches/pyarrow-15.0.0.patch @@ -115,7 +115,7 @@ index 0000000..955ac8d + '-DARROW_WITH_BROTLI=ON', + ]) + subprocess.check_call([ -+ 'cmake', '--build', str(build_dir), '--parallel', ++ 'cmake', '--build', str(build_dir), + ]) + subprocess.check_call([ + 'cmake', '--install', str(build_dir), diff --git a/graalpython/lib-graalpython/patches/pyarrow-19.0.0.patch b/graalpython/lib-graalpython/patches/pyarrow-19.0.0.patch index 70adda0ede..26a329f3ad 100644 --- a/graalpython/lib-graalpython/patches/pyarrow-19.0.0.patch +++ b/graalpython/lib-graalpython/patches/pyarrow-19.0.0.patch @@ -115,7 +115,7 @@ index 0000000..0929c5f + '-DARROW_WITH_BROTLI=ON', + ]) + subprocess.check_call([ -+ 'cmake', '--build', str(build_dir), '--parallel', ++ 'cmake', '--build', str(build_dir), + ]) + subprocess.check_call([ + 'cmake', '--install', str(build_dir), @@ -162,3 +162,27 @@ index ef2043f..cb08a86 100644 [project] name = "pyarrow" +diff --git a/MANIFEST.in b/MANIFEST.in +index ef2043f..cb08a86 100644 +--- a/MANIFEST.in ++++ b/MANIFEST.in +@@ -1,6 +1,4 @@ + include README.md +-include ../LICENSE.txt +-include ../NOTICE.txt + + global-include CMakeLists.txt + graft pyarrow +diff --git a/setup.cfg b/setup.cfg +index ef2043f..cb08a86 100644 +--- a/setup.cfg ++++ b/setup.cfg +@@ -1,7 +1,6 @@ + [metadata] + license_files = +- ../LICENSE.txt +- ../NOTICE.txt ++ README.md + + [build_sphinx] + source-dir = doc/ diff --git a/graalpython/lib-graalpython/patches/pyarrow-19.0.1.patch b/graalpython/lib-graalpython/patches/pyarrow-19.0.1.patch new file mode 100644 index 0000000000..3659ac4d27 --- /dev/null +++ b/graalpython/lib-graalpython/patches/pyarrow-19.0.1.patch @@ -0,0 +1,188 @@ +diff --git a/pyarrow/error.pxi b/pyarrow/error.pxi +index cbe2552..8d0d9d9 100644 +--- a/pyarrow/error.pxi ++++ b/pyarrow/error.pxi +@@ -248,7 +248,7 @@ cdef class SignalStopHandler: + if exc_value.signum: + # Re-emit the exact same signal. We restored the Python signal + # handler above, so it should receive it. +- if os.name == 'nt': ++ if os.name == 'nt' or sys.implementation.name == 'graalpy': + SendSignal(exc_value.signum) + else: + SendSignalToThread(exc_value.signum, +diff --git a/pyarrow/memory.pxi b/pyarrow/memory.pxi +index 1ddcb01..6805e42 100644 +--- a/pyarrow/memory.pxi ++++ b/pyarrow/memory.pxi +@@ -20,6 +20,10 @@ + # cython: embedsignature = True + + ++cdef extern from "Python.h": ++ void Py_INCREF(object) ++ ++ + cdef class MemoryPool(_Weakrefable): + """ + Base class for memory allocation. +@@ -35,6 +39,13 @@ cdef class MemoryPool(_Weakrefable): + + cdef void init(self, CMemoryPool* pool): + self.pool = pool ++ # GraalPy change: pyarrow doesn't maintain python references from ++ # buffers to pools, but they dereference the pointer to the pool in the ++ # destructor. They just assume buffers will get GC'ed before their ++ # pools. You can easily get a segfault even on CPython if you make ++ # a buffer outlive its pool. Since we can't guarantee destruction ++ # order, we just leak the pool. ++ Py_INCREF(self) + + def release_unused(self): + """ +diff --git a/pyarrow_build_backend.py b/pyarrow_build_backend.py +new file mode 100644 +index 0000000..0929c5f +--- /dev/null ++++ b/pyarrow_build_backend.py +@@ -0,0 +1,93 @@ ++import os ++import re ++import sys ++import tarfile ++import subprocess ++import tempfile ++import shutil ++import tarfile ++import urllib.request ++from pathlib import Path ++ ++VERSION = '19.0.1' ++ ++ ++def build_sdist(sdist_directory, config_settings=None): ++ nv = f'pyarrow-{VERSION}' ++ srcdir = Path(__file__).parent ++ archive_path = Path(sdist_directory) / f'{nv}.tar.gz' ++ ++ def tarfilter(info): ++ if re.match(r'\./(?:.git|venv|[^-/]+-venv|dist)', info.name): ++ return None ++ info.name = f'./{nv}/{info.name}' ++ return info ++ ++ with tarfile.open(archive_path, 'w:gz') as tar: ++ tar.add('.', filter=tarfilter) ++ return archive_path.name ++ ++ ++def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): ++ wheel_directory = Path(wheel_directory).absolute() ++ with tempfile.TemporaryDirectory() as tmpdir: ++ tmpdir = Path(tmpdir).absolute() ++ tarname = f'apache-arrow-{VERSION}.tar.gz' ++ tarpath = tmpdir / tarname ++ urllib.request.urlretrieve(f"https://github.com/apache/arrow/archive/refs/tags/{tarname}", tarpath) ++ with tarfile.open(tarpath) as tar: ++ tar.extractall(tmpdir) ++ arrow_dir = tmpdir / f'arrow-apache-arrow-{VERSION}' ++ assert arrow_dir.is_dir() ++ arrow_dist = tmpdir / 'arrow-dist' ++ build_dir = tmpdir / 'arrow-build' ++ subprocess.check_call([ ++ 'cmake', '-S', str(arrow_dir / 'cpp'), '-B', str(build_dir), ++ '-DCMAKE_INSTALL_LIBDIR=lib', ++ f'-DCMAKE_INSTALL_PREFIX={arrow_dist}', ++ '-DCMAKE_BUILD_TYPE=Release', ++ '-DARROW_RPATH_ORIGIN=ON', ++ '-DARROW_BUILD_TESTS=OFF', ++ '-DARROW_BUILD_SHARED=ON', ++ '-DARROW_BUILD_STATIC=OFF', ++ # Features ++ '-DARROW_COMPUTE=ON', ++ '-DARROW_CSV=ON', ++ '-DARROW_JSON=ON', ++ '-DARROW_FILESYSTEM=ON', ++ '-DARROW_DATASET=ON', ++ '-DARROW_PARQUET=ON', ++ '-DPARQUET_REQUIRE_ENCRYPTION=ON', ++ '-DARROW_GANDIVA=ON', ++ '-DARROW_WITH_BZ2=ON', ++ '-DARROW_WITH_ZLIB=ON', ++ '-DARROW_WITH_ZSTD=ON', ++ '-DARROW_WITH_LZ4=ON', ++ '-DARROW_WITH_SNAPPY=ON', ++ '-DARROW_WITH_BROTLI=ON', ++ ]) ++ subprocess.check_call([ ++ 'cmake', '--build', str(build_dir), ++ ]) ++ subprocess.check_call([ ++ 'cmake', '--install', str(build_dir), ++ ]) ++ env = os.environ.copy() ++ env['ARROW_HOME'] = str(arrow_dist) ++ env['CMAKE_PREFIX_PATH'] = str(arrow_dist) ++ env['PYARROW_WITH_DATASET'] = '1' ++ env['PYARROW_WITH_PARQUET'] = '1' ++ env['PYARROW_WITH_PARQUET_ENCRYPTION'] = '1' ++ env['PYARROW_WITH_GANDIVA'] = '1' ++ env['PYARROW_BUNDLE_ARROW_CPP'] = '1' ++ env['PYARROW_BUNDLE_CYTHON_CPP'] = '1' ++ subprocess.run( ++ [sys.executable, 'setup.py', 'bdist_wheel'], ++ env=env, ++ check=True, ++ ) ++ wheels = list(Path('dist').glob('*.whl')) ++ assert len(wheels) == 1, f"Expected 1 wheel, found {len(wheels)}" ++ wheel = wheels[0] ++ shutil.copyfile(wheel, wheel_directory / wheel.name) ++ return str(wheel.name) +diff --git a/pyproject.toml b/pyproject.toml +index ef2043f..cb08a86 100644 +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -17,7 +17,7 @@ + + [build-system] + requires = [ +- "cython >= 0.29.31", ++ "cython >= 0.29.31, < 3", + # Starting with NumPy 1.25, NumPy is (by default) as far back compatible + # as oldest-support-numpy was (customizable with a NPY_TARGET_VERSION + # define). For older Python versions (where NumPy 1.25 is not yet available) +@@ -29,7 +29,8 @@ requires = [ + "setuptools_scm[toml]>=8", + "setuptools>=64", + ] +-build-backend = "setuptools.build_meta" ++build-backend = "pyarrow_build_backend" ++backend-path = ["."] + + [project] + name = "pyarrow" +diff --git a/MANIFEST.in b/MANIFEST.in +index ef2043f..cb08a86 100644 +--- a/MANIFEST.in ++++ b/MANIFEST.in +@@ -1,6 +1,4 @@ + include README.md +-include ../LICENSE.txt +-include ../NOTICE.txt + + global-include CMakeLists.txt + graft pyarrow +diff --git a/setup.cfg b/setup.cfg +index ef2043f..cb08a86 100644 +--- a/setup.cfg ++++ b/setup.cfg +@@ -1,7 +1,6 @@ + [metadata] + license_files = +- ../LICENSE.txt +- ../NOTICE.txt ++ README.md + + [build_sphinx] + source-dir = doc/ diff --git a/scripts/wheelbuilder/linux/pyarrow.sh b/scripts/wheelbuilder/linux/pyarrow.sh new file mode 100644 index 0000000000..ed554ae5c4 --- /dev/null +++ b/scripts/wheelbuilder/linux/pyarrow.sh @@ -0,0 +1,58 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + dnf install -y libffi-devel \ + boost-devel \ + snappy-devel \ + brotli-devel \ + openssl-devel \ + thrift-devel \ + jemalloc-devel \ + xsimd-devel \ + libzstd-devel \ + re2-devel \ + mimalloc-devel \ + lz4-devel \ + bzip2-devel \ + llvm llvm-libs llvm-devel \ + llvm-cmake-utils \ + lld lld-devel \ + clang clang-libs clang-devel +fi From 7638a6a8100ab092ae9fa9e42999914dbb608fc5 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 21 Mar 2025 09:53:16 +0100 Subject: [PATCH 09/28] Add patch to make scikit-build-core 0.11 more lenient --- graalpython/lib-graalpython/patches/metadata.toml | 5 +++++ .../lib-graalpython/patches/scikit-build-core.patch | 11 +++++++++++ 2 files changed, 16 insertions(+) create mode 100644 graalpython/lib-graalpython/patches/scikit-build-core.patch diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index 7f6628904c..e49f845e28 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -619,6 +619,11 @@ version = '== 0.3.3' patch = 'safetensors-0.3.3.patch' license = 'Apache-2.0' +[[scikit-build-core.rules]] +version = '~= 0.11.0' +patch = 'scikit-build-core.patch' +license = 'Apache-2.0' + [[scikit-learn.rules]] version = '== 1.5.2' patch = 'scikit-learn-1.5.2.patch' diff --git a/graalpython/lib-graalpython/patches/scikit-build-core.patch b/graalpython/lib-graalpython/patches/scikit-build-core.patch new file mode 100644 index 0000000000..809e3f9d0a --- /dev/null +++ b/graalpython/lib-graalpython/patches/scikit-build-core.patch @@ -0,0 +1,11 @@ +--- a/scikit_build_core/_vendor/pyproject_metadata/__init__.py ++++ b/scikit_build_core/_vendor/pyproject_metadata/__init__.py +@@ -308,7 +308,7 @@ + project = pyproject_table["project"] + project_dir = pathlib.Path(project_dir) + +- if not allow_extra_keys: ++ if False: + extra_keys = extras_project(data) + if extra_keys: + extra_keys_str = ", ".join(sorted(f"{k!r}" for k in extra_keys)) From f409b97d9edfe58b0908c98bf5a93c8472c34145 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 21 Mar 2025 10:53:39 +0100 Subject: [PATCH 10/28] Switch to an orjson version we can build --- scripts/wheelbuilder/packages.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/packages.txt b/scripts/wheelbuilder/packages.txt index 890c395feb..352261afe0 100644 --- a/scripts/wheelbuilder/packages.txt +++ b/scripts/wheelbuilder/packages.txt @@ -85,7 +85,7 @@ opentelemetry-sdk==1.30.0 opentelemetry-semantic-conventions==0.51b0 optuna==4.2.1 oracledb==2.5.1 -orjson==3.10.15 +orjson==3.10.5 packaging==24.2 pandas==2.2.3 paramiko==3.5.1 From 6d0572e937a7f61a8c982e9b78ec4757f44f7cbd Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 21 Mar 2025 15:06:36 +0100 Subject: [PATCH 11/28] Add argument to wheelbuilder to ignore failures in generating exit code --- scripts/wheelbuilder/build_wheels.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 44f9e14026..909d1f5c01 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -195,6 +195,7 @@ def repair_wheels(): if __name__ == "__main__": parser = ArgumentParser() parser.add_argument("graalpy_url") + parser.add_argument("--ignore-failures", action="store_true", default=False) args = parser.parse_args() ext = splitext(args.graalpy_url)[1] outpath = f"graalpy{ext}" @@ -204,5 +205,5 @@ def repair_wheels(): pip = create_venv() success = build_wheels(pip) repair_wheels() - if not success: + if not success and not args.ignore_failures: sys.exit(1) From 3c58796d61d6b722dd8fe1277670a689fd2bf1fd Mon Sep 17 00:00:00 2001 From: Ivo Horak Date: Fri, 21 Mar 2025 16:31:33 +0100 Subject: [PATCH 12/28] Add MACOSX_DEPLOYMENT_TARGET to sysconfig --- graalpython/lib-graalpython/_sysconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/graalpython/lib-graalpython/_sysconfig.py b/graalpython/lib-graalpython/_sysconfig.py index dbac1041b0..8d91349c4d 100644 --- a/graalpython/lib-graalpython/_sysconfig.py +++ b/graalpython/lib-graalpython/_sysconfig.py @@ -96,6 +96,7 @@ def _get_posix_vars(): g['CFLAGS'] = ' '.join(cflags_default + [gnu_source]) g['LDFLAGS'] = "" g['CCSHARED'] = fpic + g['MACOSX_DEPLOYMENT_TARGET'] = "" # for linux it should return empty string and for macOS we should figure out how to get that value if darwin_native: g['LDFLAGS'] = "-bundle -undefined dynamic_lookup" ldshared_common = g['LDFLAGS'] From a64cd368710e1600b6ba7b2b0a9bb71f7274e110 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 21 Mar 2025 22:41:34 +0100 Subject: [PATCH 13/28] If repairing a wheel fails, keep the original --- scripts/wheelbuilder/build_wheels.py | 65 +++++++++++++++------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 909d1f5c01..4297163c9d 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -161,35 +161,42 @@ def build_wheels(pip): def repair_wheels(): whls = glob("*graalpy*.whl") - if sys.platform == "win32": - ensure_installed("delvewheel") - env = os.environ.copy() - env["PYTHONUTF8"] = "1" - subprocess.check_call( - [ - sys.executable, - "-m", - "delvewheel", - "repair", - "-v", - "--exclude", - "python-native.dll", - "-w", - "wheelhouse", - *whls, - ], - env=env, - ) - elif sys.platform == "linux": - ensure_installed("auditwheel") - subprocess.check_call( - [join(dirname(sys.executable), "auditwheel"), "repair", "-w", "wheelhouse", *whls] - ) - elif sys.platform == "darwin": - ensure_installed("delocate") - subprocess.check_call( - [join(dirname(sys.executable), "delocate-wheel"), "-v", "-w", "wheelhouse", *whls] - ) + for whl in whls: + if sys.platform == "win32": + ensure_installed("delvewheel") + env = os.environ.copy() + env["PYTHONUTF8"] = "1" + p = subprocess.run( + [ + sys.executable, + "-m", + "delvewheel", + "repair", + "-v", + "--exclude", + "python-native.dll", + "-w", + "wheelhouse", + whl, + ], + env=env, + ) + elif sys.platform == "linux": + ensure_installed("auditwheel") + p = subprocess.run( + [join(dirname(sys.executable), "auditwheel"), "repair", "-w", "wheelhouse", whl] + ) + elif sys.platform == "darwin": + ensure_installed("delocate") + p = subprocess.run( + [join(dirname(sys.executable), "delocate-wheel"), "-v", "-w", "wheelhouse", whl] + ) + if p.returncode != 0: + print("Repairing", whl, "failed, copying as is.") + try: + shutil.copy(whl, "wheelhouse") + except: + pass if __name__ == "__main__": From 9fad0faeee0cf9a8694a0514c981eb83ebca9f2c Mon Sep 17 00:00:00 2001 From: Ivo Horak Date: Tue, 25 Mar 2025 08:56:29 +0100 Subject: [PATCH 14/28] Adding pynacl.sh --- scripts/wheelbuilder/darwin/pynacl.sh | 51 +++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 scripts/wheelbuilder/darwin/pynacl.sh diff --git a/scripts/wheelbuilder/darwin/pynacl.sh b/scripts/wheelbuilder/darwin/pynacl.sh new file mode 100644 index 0000000000..cb10d556a6 --- /dev/null +++ b/scripts/wheelbuilder/darwin/pynacl.sh @@ -0,0 +1,51 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + brew install libsodium pkg-config + export PKG_CONFIG_PATH=/opt/homebrew/opt/openblas/lib/pkgconfig +fi +export SODIUM_INSTALL=system +export CFLAGS="-arch arm64 $(pkg-config libsodium --cflags)" +export LDFLAGS="$(pkg-config libsodium --libs)" +if [ -n "$1" ]; then + pip wheel "pynacl==$1" +else + pip wheel pynacl +fi From 5870dead1f62d23a893b23216452a980d82b4894 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 24 Mar 2025 10:02:52 +0100 Subject: [PATCH 15/28] Make sure to install pkg-config when using it --- scripts/wheelbuilder/darwin/cryptography.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/darwin/cryptography.sh b/scripts/wheelbuilder/darwin/cryptography.sh index aa224941ef..500059fb52 100755 --- a/scripts/wheelbuilder/darwin/cryptography.sh +++ b/scripts/wheelbuilder/darwin/cryptography.sh @@ -38,7 +38,7 @@ # SOFTWARE. if [ -n "$GITHUB_RUN_ID" ]; then - brew install libffi openssl + brew install pkg-config libffi openssl fi export PKG_CONFIG_PATH=/opt/homebrew/opt/openssl/lib/pkgconfig if [ -n "$1" ]; then From 6739b52c606e5c15deaef0b48a3ba3763efe2f05 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 24 Mar 2025 17:42:54 +0100 Subject: [PATCH 16/28] Try our best with delocate on macos --- scripts/wheelbuilder/build_wheels.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 4297163c9d..036bbe1de4 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -50,6 +50,7 @@ import hashlib import importlib import os +import platform import re import shlex import shutil @@ -189,7 +190,16 @@ def repair_wheels(): elif sys.platform == "darwin": ensure_installed("delocate") p = subprocess.run( - [join(dirname(sys.executable), "delocate-wheel"), "-v", "-w", "wheelhouse", whl] + [ + join(dirname(sys.executable), "delocate-wheel"), + "-v", + "--ignore-missing-dependencies", + "--require-archs", + "arm64" if platform.processor() == "arm" else "x86_64", + "-w", + "wheelhouse", + whl, + ], ) if p.returncode != 0: print("Repairing", whl, "failed, copying as is.") From 436cee17fdfdd7e273b35899f5efe7771109d6a6 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 24 Mar 2025 20:56:13 +0100 Subject: [PATCH 17/28] Expand readme for wheelbuilder --- scripts/wheelbuilder/README.md | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/scripts/wheelbuilder/README.md b/scripts/wheelbuilder/README.md index c12d7c5643..8f314373fa 100644 --- a/scripts/wheelbuilder/README.md +++ b/scripts/wheelbuilder/README.md @@ -33,3 +33,51 @@ That can be useful to put things like package installations specific to GitHub A Just run the `build_wheels.py` script. It expects a URL to download the GraalPy release from. You can set the environment variable `PACKAGES_TO_BUILD` to a comma-separated list of package build scripts you want to consider. + +You can build wheels locally. +The way to go about it is to make sure to include the packages you care about in the packages.txt. + +For Linux/amd64, I use [act](https://github.com/nektos/act/releases) on a Linux machine: +```shell +git clone https://github.com/oracle/graalpython +cd graalpython +VERSION=24.2.0 +BINDIR=. curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/nektos/act/master/install.sh | bash +echo "graalpy_url=https://github.com/oracle/graalpython/releases/download/graal-$VERSION/graalpy-$VERSION-linux-amd64.tar.gz" > .input +podman system service -t 0 unix:///tmp/podman.sock & +export DOCKER_HOST=unix:///tmp/podman.sock +./act --env http_proxy=$http_proxy --env https_proxy=$https_proxy -W .github/workflows/build-linux-amd64-wheels.yml --artifact-server-path=$(pwd)/artifacts +``` + +For Linux/aarch64, I use act on a mac, those are usually beefy ARM machines that a developer has in front of them, if we make sure the podman VM has enough memory. +```shell +git clone https://github.com/oracle/graalpython +cd graalpython +VERSION=24.2.0 +brew install act +echo "graalpy_url=https://github.com/oracle/graalpython/releases/download/graal-$VERSION/graalpy-$VERSION-linux-aarch64.tar.gz" > .input +podman machine init -m 16384 --now +act --env http_proxy=$http_proxy --env https_proxy=$https_proxy -W .github/workflows/build-linux-aarch64-wheels.yml --artifact-server-path=$(pwd)/artifacts --container-architecture linux/aarch64 +``` + +For macOS/aarch64, you get no isolation from act, so I just run it directly. +```shell +git clone https://github.com/oracle/graalpython +VERSION=24.2.0 +export GITHUB_RUN_ID=doesntMatterJustTriggerBrewInstallScripts +export PIP_GRAALPY_PATCHES_URL="$(pwd)/graalpython/graalpython/lib-graalpython/patches" +python3 -m venv wheelbuilder-venv +. wheelbuilder-venv/bin/activate +python3 graalpython/scripts/wheelbuilder/build_wheels.py https://github.com/oracle/graalpython/releases/download/graal-$VERSION/graalpy-$VERSION-macos-aarch64.tar.gz +``` + +For Windows/amd64, you get no isolation from act, so I just run it directly in Visual Studio powershell. +```shell +git clone https://github.com/oracle/graalpython +$VERSION="24.2.0" +$env:GITHUB_RUN_ID="doesntMatterJustTriggerBrewInstallScripts" +$env:PIP_GRAALPY_PATCHES_URL="$PWD/graalpython/graalpython/lib-graalpython/patches" +python3 -m venv wheelbuilder-venv +wheelbuilder-venv/scripts/activate +python3 graalpython/scripts/wheelbuilder/build_wheels.py https://github.com/oracle/graalpython/releases/download/graal-$VERSION/graalpy-$VERSION-windows-amd64.zip +``` From 209f46a5590f22685f65714a7f416e05c577ec1a Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 25 Mar 2025 09:18:09 +0100 Subject: [PATCH 18/28] Ignore top-level venvs --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 35689c2822..7c5bf6aa33 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,4 @@ graalpython/com.oracle.graal.python.test/src/tests/cpyext/build/ pom-mx.xml .venv !graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.jar +/*-venv/ From 0b6e9cb8f1c3414287b9d90d78f581554bdcca7b Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 25 Mar 2025 12:25:23 +0100 Subject: [PATCH 19/28] Remove duplicated wheelbuilder step --- .github/workflows/build-linux-aarch64-wheels.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-linux-aarch64-wheels.yml b/.github/workflows/build-linux-aarch64-wheels.yml index 3aa29dbe0b..a38721dbc8 100644 --- a/.github/workflows/build-linux-aarch64-wheels.yml +++ b/.github/workflows/build-linux-aarch64-wheels.yml @@ -38,8 +38,6 @@ jobs: export PIP_GRAALPY_PATCHES_URL="${GITHUB_WORKSPACE}/graalpython/lib-graalpython/patches" python3 -m venv wheelbuilder_venv wheelbuilder_venv/bin/python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - - name: Build wheels - run: python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - name: Store wheels uses: actions/upload-artifact@main with: From 4eb5990e96175f28b2cb2dc9a2897bfe4261fbc5 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 25 Mar 2025 12:36:44 +0100 Subject: [PATCH 20/28] Get python-patch-ng to provide patch.exe on windows --- scripts/wheelbuilder/build_wheels.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 036bbe1de4..00e4fe13e5 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -101,6 +101,11 @@ def create_venv(): subprocess.check_call([binary, "-m", "venv", "graalpy"]) print("Installing wheel with", pip, flush=True) subprocess.check_call([pip, "install", "wheel"]) + if sys.platform == "win32": + print("Installing python-patch-ng to provide patch.exe", flush=True) + p = subprocess.run([pip, "install", "git+https://github.com/timfel/python-patch-ng.git"]) + if p.returncode != 0: + print("Installing python-patch-ng failed, assuming patch.exe is on PATH", flush=True) return pip From 5760984519e0fb3e714baefea80b1c347eec5a8c Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 25 Mar 2025 13:38:45 +0100 Subject: [PATCH 21/28] Expand range for Cython patch --- graalpython/lib-graalpython/patches/metadata.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index e49f845e28..d53571d84f 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -105,7 +105,7 @@ patch = 'cryptography-42.0.5.patch' license = 'Apache-2.0 OR BSD-3-Clause' [[Cython.rules]] -version = '>= 3.0.10, < 3.0.12' +version = '>= 3.0.10, <= 3.0.12' patch = 'Cython-3.0.10.patch' license = 'Apache-2.0' From 0112380dda60e3b6f9a8c609eb142a1f830ea965 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 25 Mar 2025 13:39:06 +0100 Subject: [PATCH 22/28] Avoid redownload in build_wheels.py --- scripts/wheelbuilder/build_wheels.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 00e4fe13e5..9b462d3196 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -75,8 +75,11 @@ def ensure_installed(name): def download(url, out): - print("Downloading", url, flush=True) - urlretrieve(url, out) + if not os.path.exists(out): + print("Downloading", url, flush=True) + urlretrieve(url, out) + else: + print("Using previously downloaded", out, flush=True) def extract(archive): From 19fed3377a1336da216a4396688451d9bef45059 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 25 Mar 2025 13:59:16 +0100 Subject: [PATCH 23/28] Make sure we recythonize rapidfuzz 3.10 - 3.12 --- .../lib-graalpython/patches/metadata.toml | 14 +++++ .../patches/rapidfuzz-3.10.patch | 28 +++++++++ .../wheelbuilder/darwin/levenshtein.0.26.1.sh | 62 +++++++++++++++++++ .../wheelbuilder/darwin/rapidfuzz.3.12.1.sh | 41 ++++++++++++ .../wheelbuilder/linux/levenshtein.0.26.1.sh | 62 +++++++++++++++++++ .../wheelbuilder/linux/rapidfuzz.3.12.1.sh | 41 ++++++++++++ 6 files changed, 248 insertions(+) create mode 100644 graalpython/lib-graalpython/patches/rapidfuzz-3.10.patch create mode 100644 scripts/wheelbuilder/darwin/levenshtein.0.26.1.sh create mode 100644 scripts/wheelbuilder/darwin/rapidfuzz.3.12.1.sh create mode 100644 scripts/wheelbuilder/linux/levenshtein.0.26.1.sh create mode 100644 scripts/wheelbuilder/linux/rapidfuzz.3.12.1.sh diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index d53571d84f..7ace4cfbc8 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -580,9 +580,23 @@ license = 'BSD-3-Clause' dist-type = 'sdist' [[rapidfuzz.rules]] +version = '< 3.10' patch = 'rapidfuzz.patch' license = 'MIT' +[[rapidfuzz.rules]] +version = '>= 3.10.0, <= 3.12.2' +patch = 'rapidfuzz-3.10.patch' +license = 'MIT' + +[[levenshtein.add-sources]] +version = '0.26.1' +url = 'https://github.com/rapidfuzz/Levenshtein/archive/refs/tags/v0.26.1.zip' + +[[levenshtein.add-sources]] +version = '0.27.1' +url = 'https://github.com/rapidfuzz/Levenshtein/archive/refs/tags/v0.27.1.zip' + [[ray.rules]] version = '== 2.3.0' patch = 'ray-2.3.0.patch' diff --git a/graalpython/lib-graalpython/patches/rapidfuzz-3.10.patch b/graalpython/lib-graalpython/patches/rapidfuzz-3.10.patch new file mode 100644 index 0000000000..4cb200aee8 --- /dev/null +++ b/graalpython/lib-graalpython/patches/rapidfuzz-3.10.patch @@ -0,0 +1,28 @@ +diff --git a/src/rapidfuzz/CMakeLists.txt b/src/rapidfuzz/CMakeLists.txt +index c8cdfd6..8c99251 100644 +--- a/src/rapidfuzz/CMakeLists.txt ++++ b/src/rapidfuzz/CMakeLists.txt +@@ -1,9 +1,9 @@ + function(create_cython_target _name) +- if(EXISTS ${CMAKE_CURRENT_LIST_DIR}/${_name}.cxx) +- set(${_name} +- ${CMAKE_CURRENT_LIST_DIR}/${_name}.cxx +- PARENT_SCOPE) +- else() ++ # if(EXISTS ${CMAKE_CURRENT_LIST_DIR}/${_name}.cxx) ++ # set(${_name} ++ # ${CMAKE_CURRENT_LIST_DIR}/${_name}.cxx ++ # PARENT_SCOPE) ++ # else() + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${_name}.cxx" + MAIN_DEPENDENCY "${CMAKE_CURRENT_LIST_DIR}/${_name}.pyx" +@@ -16,7 +16,7 @@ function(create_cython_target _name) + set(${_name} + ${CMAKE_CURRENT_BINARY_DIR}/${_name}.cxx + PARENT_SCOPE) +- endif() ++ # endif() + endfunction(create_cython_target) + + function(rf_add_library name) diff --git a/scripts/wheelbuilder/darwin/levenshtein.0.26.1.sh b/scripts/wheelbuilder/darwin/levenshtein.0.26.1.sh new file mode 100644 index 0000000000..04be0e91b7 --- /dev/null +++ b/scripts/wheelbuilder/darwin/levenshtein.0.26.1.sh @@ -0,0 +1,62 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + git clone --single-branch --branch v3.3.2 --depth 1 https://github.com/rapidfuzz/rapidfuzz-cpp.git rapidfuzz-cpp + cd rapidfuzz-cpp + mkdir build && cd build + cmake .. -DCMAKE_BUILD_TYPE=Release + cmake --build . + sudo cmake --build . --target install + cd .. + cd .. +fi + +pip install 'Cython==3.0.11' 'scikit_build_core==0.11.1' +pip install --no-build-isolation 'rapidfuzz==3.12.1' +pip wheel --no-build-isolation 'levenshtein==0.26.1' + +if [ -n "$GITHUB_RUN_ID" ]; then + cd rapidfuzz-cpp + cd build + xargs rm < install_manifest.txt + cd .. + cd .. + rm -rf rapidfuzz-cpp +fi diff --git a/scripts/wheelbuilder/darwin/rapidfuzz.3.12.1.sh b/scripts/wheelbuilder/darwin/rapidfuzz.3.12.1.sh new file mode 100644 index 0000000000..8c083b59b0 --- /dev/null +++ b/scripts/wheelbuilder/darwin/rapidfuzz.3.12.1.sh @@ -0,0 +1,41 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +pip install 'Cython==3.0.11' 'scikit_build_core==0.11.1' +pip wheel --no-build-isolation 'rapidfuzz==3.12.1' diff --git a/scripts/wheelbuilder/linux/levenshtein.0.26.1.sh b/scripts/wheelbuilder/linux/levenshtein.0.26.1.sh new file mode 100644 index 0000000000..9fcc8842de --- /dev/null +++ b/scripts/wheelbuilder/linux/levenshtein.0.26.1.sh @@ -0,0 +1,62 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + git clone --single-branch --branch v3.3.2 --depth 1 https://github.com/rapidfuzz/rapidfuzz-cpp.git rapidfuzz-cpp + cd rapidfuzz-cpp + mkdir build && cd build + cmake .. -DCMAKE_BUILD_TYPE=Release + cmake --build . + sudo cmake --build . --target install + cd .. + cd .. +fi + +pip install 'Cython==3.0.11' 'scikit_build_core==0.11.1' +pip install --no-build-isolation 'rapidfuzz==3.12.1' +pip wheel --no-build-isolation 'levenshtein==0.26.1' + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + cd rapidfuzz-cpp + cd build + xargs rm < install_manifest.txt + cd .. + cd .. + rm -rf rapidfuzz-cpp +fi diff --git a/scripts/wheelbuilder/linux/rapidfuzz.3.12.1.sh b/scripts/wheelbuilder/linux/rapidfuzz.3.12.1.sh new file mode 100644 index 0000000000..8c083b59b0 --- /dev/null +++ b/scripts/wheelbuilder/linux/rapidfuzz.3.12.1.sh @@ -0,0 +1,41 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +pip install 'Cython==3.0.11' 'scikit_build_core==0.11.1' +pip wheel --no-build-isolation 'rapidfuzz==3.12.1' From 51d79d1ec217e8601d6077f61a2508aae4d3c0e7 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 27 Mar 2025 13:03:51 +0100 Subject: [PATCH 24/28] auditwheel repair for OL8 --- scripts/wheelbuilder/build_wheels.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 9b462d3196..883478ac45 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -193,7 +193,15 @@ def repair_wheels(): elif sys.platform == "linux": ensure_installed("auditwheel") p = subprocess.run( - [join(dirname(sys.executable), "auditwheel"), "repair", "-w", "wheelhouse", whl] + [ + join(dirname(sys.executable), "auditwheel"), + "repair", + "--plat", + "manylinux_2_28_x86_64", + "-w", + "wheelhouse", + whl, + ] ) elif sys.platform == "darwin": ensure_installed("delocate") From 12cc3864bc1d44f72d2a846a8ee1a2b0d45ca6e3 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 27 Mar 2025 13:09:44 +0100 Subject: [PATCH 25/28] Install patchelf in addition to auditwheel into the venv --- scripts/wheelbuilder/build_wheels.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 883478ac45..24ae6628a7 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -66,11 +66,11 @@ from urllib.request import urlretrieve -def ensure_installed(name): +def ensure_installed(name, *extra): try: return importlib.import_module(name) except ImportError: - subprocess.check_call([sys.executable, "-m", "pip", "install", name]) + subprocess.check_call([sys.executable, "-m", "pip", "install", name, *extra]) return importlib.import_module(name) @@ -170,11 +170,12 @@ def build_wheels(pip): def repair_wheels(): whls = glob("*graalpy*.whl") + env = os.environ.copy() + env["PYTHONUTF8"] = "1" + env["PATH"] = abspath(dirname(sys.executable)) + os.pathsep + env["PATH"] for whl in whls: if sys.platform == "win32": ensure_installed("delvewheel") - env = os.environ.copy() - env["PYTHONUTF8"] = "1" p = subprocess.run( [ sys.executable, @@ -191,7 +192,7 @@ def repair_wheels(): env=env, ) elif sys.platform == "linux": - ensure_installed("auditwheel") + ensure_installed("auditwheel", "patchelf") p = subprocess.run( [ join(dirname(sys.executable), "auditwheel"), @@ -201,7 +202,8 @@ def repair_wheels(): "-w", "wheelhouse", whl, - ] + ], + env=env, ) elif sys.platform == "darwin": ensure_installed("delocate") @@ -216,6 +218,7 @@ def repair_wheels(): "wheelhouse", whl, ], + env=env, ) if p.returncode != 0: print("Repairing", whl, "failed, copying as is.") From 9569ec59ae49e29ce4a278f8068c21fcab6c6398 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 27 Mar 2025 14:10:14 +0100 Subject: [PATCH 26/28] Use GITHUB_RUN_ID envvar everywhere to decide if build wheels shell scripts should run --- scripts/wheelbuilder/linux/cffi.sh | 2 +- scripts/wheelbuilder/linux/cryptography.sh | 2 +- scripts/wheelbuilder/linux/h5py.sh | 2 +- scripts/wheelbuilder/linux/levenshtein.0.26.1.sh | 4 ++-- scripts/wheelbuilder/linux/lxml.sh | 2 +- scripts/wheelbuilder/linux/numpy.1.23.2.sh | 2 +- scripts/wheelbuilder/linux/numpy.1.24.3.sh | 2 +- scripts/wheelbuilder/linux/numpy.sh | 2 +- scripts/wheelbuilder/linux/pillow.sh | 2 +- scripts/wheelbuilder/linux/pyarrow.sh | 2 +- scripts/wheelbuilder/linux/pynacl.sh | 2 +- scripts/wheelbuilder/linux/scipy.sh | 2 +- scripts/wheelbuilder/linux/tensorflow.sh | 2 +- scripts/wheelbuilder/linux/torch.sh | 2 +- scripts/wheelbuilder/linux/zstandard.sh | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/wheelbuilder/linux/cffi.sh b/scripts/wheelbuilder/linux/cffi.sh index 6770b98e27..f883a887ae 100755 --- a/scripts/wheelbuilder/linux/cffi.sh +++ b/scripts/wheelbuilder/linux/cffi.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y libffi-devel fi diff --git a/scripts/wheelbuilder/linux/cryptography.sh b/scripts/wheelbuilder/linux/cryptography.sh index ef0138706f..eb9b402c81 100755 --- a/scripts/wheelbuilder/linux/cryptography.sh +++ b/scripts/wheelbuilder/linux/cryptography.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y libffi-devel openssl-devel fi diff --git a/scripts/wheelbuilder/linux/h5py.sh b/scripts/wheelbuilder/linux/h5py.sh index 82172d55f0..f680b5f69e 100755 --- a/scripts/wheelbuilder/linux/h5py.sh +++ b/scripts/wheelbuilder/linux/h5py.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y hdf5-devel fi diff --git a/scripts/wheelbuilder/linux/levenshtein.0.26.1.sh b/scripts/wheelbuilder/linux/levenshtein.0.26.1.sh index 9fcc8842de..04be0e91b7 100644 --- a/scripts/wheelbuilder/linux/levenshtein.0.26.1.sh +++ b/scripts/wheelbuilder/linux/levenshtein.0.26.1.sh @@ -37,7 +37,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then git clone --single-branch --branch v3.3.2 --depth 1 https://github.com/rapidfuzz/rapidfuzz-cpp.git rapidfuzz-cpp cd rapidfuzz-cpp mkdir build && cd build @@ -52,7 +52,7 @@ pip install 'Cython==3.0.11' 'scikit_build_core==0.11.1' pip install --no-build-isolation 'rapidfuzz==3.12.1' pip wheel --no-build-isolation 'levenshtein==0.26.1' -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then cd rapidfuzz-cpp cd build xargs rm < install_manifest.txt diff --git a/scripts/wheelbuilder/linux/lxml.sh b/scripts/wheelbuilder/linux/lxml.sh index 5335db4335..c22e81e04b 100644 --- a/scripts/wheelbuilder/linux/lxml.sh +++ b/scripts/wheelbuilder/linux/lxml.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y libxml2-devel libxslt-devel fi diff --git a/scripts/wheelbuilder/linux/numpy.1.23.2.sh b/scripts/wheelbuilder/linux/numpy.1.23.2.sh index a9378be59d..eeafa70a2a 100644 --- a/scripts/wheelbuilder/linux/numpy.1.23.2.sh +++ b/scripts/wheelbuilder/linux/numpy.1.23.2.sh @@ -37,7 +37,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y gcc-toolset-9 gcc-toolset-9-gcc-gfortran openblas-devel scl enable gcc-toolset-9 'pip wheel "numpy==1.23.2"' fi diff --git a/scripts/wheelbuilder/linux/numpy.1.24.3.sh b/scripts/wheelbuilder/linux/numpy.1.24.3.sh index 7d50924cea..5b6d9e0613 100644 --- a/scripts/wheelbuilder/linux/numpy.1.24.3.sh +++ b/scripts/wheelbuilder/linux/numpy.1.24.3.sh @@ -37,7 +37,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y gcc-toolset-9 gcc-toolset-9-gcc-gfortran openblas-devel scl enable gcc-toolset-9 'pip wheel "numpy==1.24.3"' fi diff --git a/scripts/wheelbuilder/linux/numpy.sh b/scripts/wheelbuilder/linux/numpy.sh index a24e6a2aa3..57e2076985 100755 --- a/scripts/wheelbuilder/linux/numpy.sh +++ b/scripts/wheelbuilder/linux/numpy.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y gcc-toolset-12-gcc-gfortran openblas-devel fi diff --git a/scripts/wheelbuilder/linux/pillow.sh b/scripts/wheelbuilder/linux/pillow.sh index e6d64d0863..668580ab54 100755 --- a/scripts/wheelbuilder/linux/pillow.sh +++ b/scripts/wheelbuilder/linux/pillow.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y libtiff-devel libjpeg-devel openjpeg2-devel zlib-devel freetype-devel lcms2-devel fi diff --git a/scripts/wheelbuilder/linux/pyarrow.sh b/scripts/wheelbuilder/linux/pyarrow.sh index ed554ae5c4..1f56c7bee9 100644 --- a/scripts/wheelbuilder/linux/pyarrow.sh +++ b/scripts/wheelbuilder/linux/pyarrow.sh @@ -37,7 +37,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y libffi-devel \ boost-devel \ snappy-devel \ diff --git a/scripts/wheelbuilder/linux/pynacl.sh b/scripts/wheelbuilder/linux/pynacl.sh index f4e5359ec6..8d488965e9 100644 --- a/scripts/wheelbuilder/linux/pynacl.sh +++ b/scripts/wheelbuilder/linux/pynacl.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y libffi-devel fi diff --git a/scripts/wheelbuilder/linux/scipy.sh b/scripts/wheelbuilder/linux/scipy.sh index cb127c5abc..53e2005eda 100755 --- a/scripts/wheelbuilder/linux/scipy.sh +++ b/scripts/wheelbuilder/linux/scipy.sh @@ -37,7 +37,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y gcc-toolset-12-gcc-gfortran openblas-devel fi export FFLAGS=-fallow-argument-mismatch diff --git a/scripts/wheelbuilder/linux/tensorflow.sh b/scripts/wheelbuilder/linux/tensorflow.sh index f67401d4bf..a7cfb98b4f 100755 --- a/scripts/wheelbuilder/linux/tensorflow.sh +++ b/scripts/wheelbuilder/linux/tensorflow.sh @@ -37,7 +37,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y openblas-devel /usr/bin/cmake /usr/bin/sudo /usr/bin/curl java-11-openjdk-devel fi pip install pip numpy wheel packaging requests opt_einsum diff --git a/scripts/wheelbuilder/linux/torch.sh b/scripts/wheelbuilder/linux/torch.sh index e931f26abd..7ae3d60110 100755 --- a/scripts/wheelbuilder/linux/torch.sh +++ b/scripts/wheelbuilder/linux/torch.sh @@ -37,7 +37,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y openblas-devel /usr/bin/cmake /usr/bin/sudo libffi-devel export USE_CUDA=0 fi diff --git a/scripts/wheelbuilder/linux/zstandard.sh b/scripts/wheelbuilder/linux/zstandard.sh index f4e5359ec6..8d488965e9 100644 --- a/scripts/wheelbuilder/linux/zstandard.sh +++ b/scripts/wheelbuilder/linux/zstandard.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y libffi-devel fi From 920ac11d704dbfd1fe6d84de5c7dede8abcf4ea1 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 27 Mar 2025 14:15:35 +0100 Subject: [PATCH 27/28] Select manylinux tag appropriate for processor architecture --- scripts/wheelbuilder/build_wheels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 24ae6628a7..4a5181adb0 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -198,7 +198,7 @@ def repair_wheels(): join(dirname(sys.executable), "auditwheel"), "repair", "--plat", - "manylinux_2_28_x86_64", + "manylinux_2_28_x86_64" if platform.processor() == "x86_64" else "manylinux_2_28_aarch64", "-w", "wheelhouse", whl, From 5ff716171a5a32bac5329ada12fda18b1f68d20b Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 28 Mar 2025 11:09:20 +0100 Subject: [PATCH 28/28] Update copyrights, fix MACOSX_DEPLOYMENT_TARGET to 11 --- graalpython/lib-graalpython/_sysconfig.py | 9 +++++++-- scripts/wheelbuilder/build_wheels.py | 2 +- scripts/wheelbuilder/linux/cffi.sh | 2 +- scripts/wheelbuilder/linux/h5py.sh | 2 +- scripts/wheelbuilder/linux/lxml.sh | 2 +- scripts/wheelbuilder/linux/numpy.1.23.2.sh | 2 +- scripts/wheelbuilder/linux/numpy.1.24.3.sh | 2 +- scripts/wheelbuilder/linux/numpy.sh | 2 +- scripts/wheelbuilder/linux/pillow.sh | 2 +- scripts/wheelbuilder/linux/scipy.sh | 2 +- scripts/wheelbuilder/linux/tensorflow.sh | 2 +- scripts/wheelbuilder/linux/torch.sh | 2 +- 12 files changed, 18 insertions(+), 13 deletions(-) diff --git a/graalpython/lib-graalpython/_sysconfig.py b/graalpython/lib-graalpython/_sysconfig.py index 8d91349c4d..814d4c0dcb 100644 --- a/graalpython/lib-graalpython/_sysconfig.py +++ b/graalpython/lib-graalpython/_sysconfig.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -96,7 +96,12 @@ def _get_posix_vars(): g['CFLAGS'] = ' '.join(cflags_default + [gnu_source]) g['LDFLAGS'] = "" g['CCSHARED'] = fpic - g['MACOSX_DEPLOYMENT_TARGET'] = "" # for linux it should return empty string and for macOS we should figure out how to get that value + if darwin_native: + # MACOSX_DEPLOYMENT_TARGET is taken from the minimum version we build + # GraalPy for, which is currently BigSur + g['MACOSX_DEPLOYMENT_TARGET'] = "11" + else: + g['MACOSX_DEPLOYMENT_TARGET'] = "" if darwin_native: g['LDFLAGS'] = "-bundle -undefined dynamic_lookup" ldshared_common = g['LDFLAGS'] diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 4a5181adb0..78756b84a8 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/cffi.sh b/scripts/wheelbuilder/linux/cffi.sh index f883a887ae..21c79d86b7 100755 --- a/scripts/wheelbuilder/linux/cffi.sh +++ b/scripts/wheelbuilder/linux/cffi.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/h5py.sh b/scripts/wheelbuilder/linux/h5py.sh index f680b5f69e..7c70c6eba1 100755 --- a/scripts/wheelbuilder/linux/h5py.sh +++ b/scripts/wheelbuilder/linux/h5py.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/lxml.sh b/scripts/wheelbuilder/linux/lxml.sh index c22e81e04b..40cf8d251e 100644 --- a/scripts/wheelbuilder/linux/lxml.sh +++ b/scripts/wheelbuilder/linux/lxml.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/numpy.1.23.2.sh b/scripts/wheelbuilder/linux/numpy.1.23.2.sh index eeafa70a2a..473b91359d 100644 --- a/scripts/wheelbuilder/linux/numpy.1.23.2.sh +++ b/scripts/wheelbuilder/linux/numpy.1.23.2.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/numpy.1.24.3.sh b/scripts/wheelbuilder/linux/numpy.1.24.3.sh index 5b6d9e0613..f9e524e3d4 100644 --- a/scripts/wheelbuilder/linux/numpy.1.24.3.sh +++ b/scripts/wheelbuilder/linux/numpy.1.24.3.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/numpy.sh b/scripts/wheelbuilder/linux/numpy.sh index 57e2076985..8cd8c11077 100755 --- a/scripts/wheelbuilder/linux/numpy.sh +++ b/scripts/wheelbuilder/linux/numpy.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/pillow.sh b/scripts/wheelbuilder/linux/pillow.sh index 668580ab54..4b535f89df 100755 --- a/scripts/wheelbuilder/linux/pillow.sh +++ b/scripts/wheelbuilder/linux/pillow.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/scipy.sh b/scripts/wheelbuilder/linux/scipy.sh index 53e2005eda..19cffbffc4 100755 --- a/scripts/wheelbuilder/linux/scipy.sh +++ b/scripts/wheelbuilder/linux/scipy.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/tensorflow.sh b/scripts/wheelbuilder/linux/tensorflow.sh index a7cfb98b4f..03259d33cd 100755 --- a/scripts/wheelbuilder/linux/tensorflow.sh +++ b/scripts/wheelbuilder/linux/tensorflow.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/torch.sh b/scripts/wheelbuilder/linux/torch.sh index 7ae3d60110..b2c54a9644 100755 --- a/scripts/wheelbuilder/linux/torch.sh +++ b/scripts/wheelbuilder/linux/torch.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0