Skip to content

Commit 3cd7cfd

Browse files
committed
[GR-63270] Add scripts to build many packages for AI workloads
PullRequest: graalpython/3730
2 parents 48a8296 + 5ff7161 commit 3cd7cfd

39 files changed

+1410
-84
lines changed

.github/workflows/build-linux-aarch64-wheels.yml

+5-3
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,15 @@ jobs:
2929
dnf install -y /usr/bin/patch
3030
- name: Checkout
3131
uses: actions/checkout@main
32+
- uses: actions-rust-lang/setup-rust-toolchain@v1
33+
with:
34+
rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports"
35+
cache: false
3236
- name: Build wheels
3337
run: |
38+
export PIP_GRAALPY_PATCHES_URL="${GITHUB_WORKSPACE}/graalpython/lib-graalpython/patches"
3439
python3 -m venv wheelbuilder_venv
35-
export PIP_EXTRA_INDEX_URL=https://pypi.org/simple
3640
wheelbuilder_venv/bin/python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }}
37-
- name: Build wheels
38-
run: python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }}
3941
- name: Store wheels
4042
uses: actions/upload-artifact@main
4143
with:

.github/workflows/build-linux-amd64-wheels.yml

+5-1
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,14 @@ jobs:
2727
dnf install -y /usr/bin/patch
2828
- name: Checkout
2929
uses: actions/checkout@main
30+
- uses: actions-rust-lang/setup-rust-toolchain@v1
31+
with:
32+
rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports"
33+
cache: false
3034
- name: Build wheels
3135
run: |
36+
export PIP_GRAALPY_PATCHES_URL="${GITHUB_WORKSPACE}/graalpython/lib-graalpython/patches"
3237
python3 -m venv wheelbuilder_venv
33-
export PIP_EXTRA_INDEX_URL=https://pypi.org/simple
3438
wheelbuilder_venv/bin/python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }}
3539
- name: Store wheels
3640
uses: actions/upload-artifact@main

.github/workflows/build-macos-aarch64-wheels.yml

+5-1
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,17 @@ jobs:
1818
steps:
1919
- name: Checkout
2020
uses: actions/checkout@main
21+
- uses: actions-rust-lang/setup-rust-toolchain@v1
22+
with:
23+
rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports"
24+
cache: false
2125
- name: Setup Python
2226
uses: actions/setup-python@v5
2327
with:
2428
python-version: 3.12
2529
- name: Build wheels
2630
run: |
27-
export PIP_EXTRA_INDEX_URL=https://pypi.org/simple
31+
export PIP_GRAALPY_PATCHES_URL="${GITHUB_WORKSPACE}/graalpython/lib-graalpython/patches"
2832
python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }}
2933
- name: Store wheels
3034
uses: actions/upload-artifact@main

.github/workflows/build-macos-amd64-wheels.yml

+5-1
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,17 @@ jobs:
1818
steps:
1919
- name: Checkout
2020
uses: actions/checkout@main
21+
- uses: actions-rust-lang/setup-rust-toolchain@v1
22+
with:
23+
rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports"
24+
cache: false
2125
- name: Setup Python
2226
uses: actions/setup-python@v5
2327
with:
2428
python-version: 3.12
2529
- name: Build wheels
2630
run: |
27-
export PIP_EXTRA_INDEX_URL=https://pypi.org/simple
31+
export PIP_GRAALPY_PATCHES_URL="${GITHUB_WORKSPACE}/graalpython/lib-graalpython/patches"
2832
python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }}
2933
- name: Store wheels
3034
uses: actions/upload-artifact@main

.github/workflows/build-windows-amd64-wheels.yml

+6-1
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,19 @@ jobs:
1919
- uses: ilammy/msvc-dev-cmd@v1
2020
- name: Checkout
2121
uses: actions/checkout@main
22+
- uses: actions-rust-lang/setup-rust-toolchain@v1
23+
with:
24+
rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports"
25+
cache: false
2226
- name: Setup Python
2327
uses: actions/setup-python@v5
2428
with:
2529
python-version: 3.12
2630
- name: Build wheels
2731
run: |
2832
$env:PATH+=";C:\Program Files\Git\usr\bin"
29-
$env:PIP_EXTRA_INDEX_URL="https://pypi.org/simple"
33+
$env:PIP_GRAALPY_PATCHES_URL=$env:GITHUB_WORKSPACE
34+
$env:PIP_GRAALPY_PATCHES_URL+="/graalpython/lib-graalpython/patches"
3035
python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }}
3136
- name: Store wheels
3237
uses: actions/upload-artifact@main

.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -95,3 +95,4 @@ graalpython/com.oracle.graal.python.test/src/tests/cpyext/build/
9595
pom-mx.xml
9696
.venv
9797
!graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.jar
98+
/*-venv/

graalpython/lib-graalpython/_sysconfig.py

+7-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -96,6 +96,12 @@ def _get_posix_vars():
9696
g['CFLAGS'] = ' '.join(cflags_default + [gnu_source])
9797
g['LDFLAGS'] = ""
9898
g['CCSHARED'] = fpic
99+
if darwin_native:
100+
# MACOSX_DEPLOYMENT_TARGET is taken from the minimum version we build
101+
# GraalPy for, which is currently BigSur
102+
g['MACOSX_DEPLOYMENT_TARGET'] = "11"
103+
else:
104+
g['MACOSX_DEPLOYMENT_TARGET'] = ""
99105
if darwin_native:
100106
g['LDFLAGS'] = "-bundle -undefined dynamic_lookup"
101107
ldshared_common = g['LDFLAGS']

graalpython/lib-graalpython/patches/metadata.toml

+37-1
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ patch = 'cryptography-42.0.5.patch'
105105
license = 'Apache-2.0 OR BSD-3-Clause'
106106

107107
[[Cython.rules]]
108-
version = '>= 3.0.10, < 3.0.12'
108+
version = '>= 3.0.10, <= 3.0.12'
109109
patch = 'Cython-3.0.10.patch'
110110
license = 'Apache-2.0'
111111

@@ -301,6 +301,13 @@ patch = 'numpy-2.0.0.patch'
301301
license = 'BSD-3-Clause'
302302
dist-type = 'sdist'
303303

304+
[[numpy.rules]]
305+
version = '== 2.2.3'
306+
patch = 'numpy-2.2.3.patch'
307+
license = 'BSD-3-Clause'
308+
dist-type = 'sdist'
309+
install-priority = 0
310+
304311
[[numpy.rules]]
305312
version = '== 1.26.4'
306313
patch = 'numpy-1.26.4.patch'
@@ -448,6 +455,11 @@ version = '== 19.0.0'
448455
patch = 'pyarrow-19.0.0.patch'
449456
license = 'Apache-2.0'
450457

458+
[[pyarrow.rules]]
459+
version = '== 19.0.1'
460+
patch = 'pyarrow-19.0.1.patch'
461+
license = 'Apache-2.0'
462+
451463
[[pybind11.rules]]
452464
# Upstreamed
453465
install-priority = 0
@@ -484,6 +496,11 @@ version = '== 2.27.1'
484496
patch = 'pydantic-core-2.27.1.patch'
485497
license = 'MIT'
486498

499+
[[pydantic-core.rules]]
500+
version = '== 2.27.2'
501+
patch = 'pydantic-core-2.27.1.patch'
502+
license = 'MIT'
503+
487504
[[pydantic-core.rules]]
488505
version = '== 2.26.0'
489506
patch = 'pydantic-core-2.26.0.patch'
@@ -572,9 +589,23 @@ license = 'BSD-3-Clause'
572589
dist-type = 'sdist'
573590

574591
[[rapidfuzz.rules]]
592+
version = '< 3.10'
575593
patch = 'rapidfuzz.patch'
576594
license = 'MIT'
577595

596+
[[rapidfuzz.rules]]
597+
version = '>= 3.10.0, <= 3.12.2'
598+
patch = 'rapidfuzz-3.10.patch'
599+
license = 'MIT'
600+
601+
[[levenshtein.add-sources]]
602+
version = '0.26.1'
603+
url = 'https://github.com/rapidfuzz/Levenshtein/archive/refs/tags/v0.26.1.zip'
604+
605+
[[levenshtein.add-sources]]
606+
version = '0.27.1'
607+
url = 'https://github.com/rapidfuzz/Levenshtein/archive/refs/tags/v0.27.1.zip'
608+
578609
[[ray.rules]]
579610
version = '== 2.3.0'
580611
patch = 'ray-2.3.0.patch'
@@ -611,6 +642,11 @@ version = '== 0.3.3'
611642
patch = 'safetensors-0.3.3.patch'
612643
license = 'Apache-2.0'
613644

645+
[[scikit-build-core.rules]]
646+
version = '~= 0.11.0'
647+
patch = 'scikit-build-core.patch'
648+
license = 'Apache-2.0'
649+
614650
[[scikit-learn.rules]]
615651
version = '== 1.5.2'
616652
patch = 'scikit-learn-1.5.2.patch'
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
diff --git a/numpy/_core/include/numpy/ndarrayobject.h b/numpy/_core/include/numpy/ndarrayobject.h
2+
index f06bafe5b5..52063a7d16 100644
3+
--- a/numpy/_core/include/numpy/ndarrayobject.h
4+
+++ b/numpy/_core/include/numpy/ndarrayobject.h
5+
@@ -220,7 +220,7 @@ NPY_TITLE_KEY_check(PyObject *key, PyObject *value)
6+
if (key == title) {
7+
return 1;
8+
}
9+
-#ifdef PYPY_VERSION
10+
+#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON)
11+
/*
12+
* On PyPy, dictionary keys do not always preserve object identity.
13+
* Fall back to comparison by value.
14+
diff --git a/numpy/_core/src/multiarray/compiled_base.c b/numpy/_core/src/multiarray/compiled_base.c
15+
index 48524aff4d..e55e2479e1 100644
16+
--- a/numpy/_core/src/multiarray/compiled_base.c
17+
+++ b/numpy/_core/src/multiarray/compiled_base.c
18+
@@ -1455,6 +1455,7 @@ fail:
19+
NPY_NO_EXPORT PyObject *
20+
arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *const *args, Py_ssize_t len_args)
21+
{
22+
+#if 0 // GraalPy change
23+
PyObject *obj;
24+
PyObject *str;
25+
const char *docstr;
26+
@@ -1559,6 +1560,7 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *const *args, Py_ssize_t
27+
}
28+
29+
#undef _ADDDOC
30+
+#endif // GraalPy change
31+
32+
Py_RETURN_NONE;
33+
}
34+
diff --git a/numpy/_core/src/multiarray/shape.c b/numpy/_core/src/multiarray/shape.c
35+
index 340fe7289a..8f13674383 100644
36+
--- a/numpy/_core/src/multiarray/shape.c
37+
+++ b/numpy/_core/src/multiarray/shape.c
38+
@@ -97,6 +97,11 @@ PyArray_Resize(PyArrayObject *self, PyArray_Dims *newshape, int refcheck,
39+
"cannot resize an array with refcheck=True on PyPy.\n"
40+
"Use the np.resize function or refcheck=False");
41+
return NULL;
42+
+#elif defined(GRAALVM_PYTHON)
43+
+ PyErr_SetString(PyExc_ValueError,
44+
+ "cannot resize an array with refcheck=True on GraalPy.\n"
45+
+ "Use the np.resize function or refcheck=False");
46+
+ return NULL;
47+
#else
48+
refcnt = Py_REFCNT(self);
49+
#endif /* PYPY_VERSION */
50+
diff --git a/numpy/_core/src/multiarray/stringdtype/dtype.c b/numpy/_core/src/multiarray/stringdtype/dtype.c
51+
index cb8265dd3d..0dabb49346 100644
52+
--- a/numpy/_core/src/multiarray/stringdtype/dtype.c
53+
+++ b/numpy/_core/src/multiarray/stringdtype/dtype.c
54+
@@ -841,7 +841,7 @@ init_string_dtype(void)
55+
};
56+
57+
/* Loaded dynamically, so needs to be set here: */
58+
- Py_TYPE(((PyObject *)&PyArray_StringDType)) = &PyArrayDTypeMeta_Type;
59+
+ Py_SET_TYPE((PyObject *)&PyArray_StringDType, &PyArrayDTypeMeta_Type);
60+
((PyTypeObject *)&PyArray_StringDType)->tp_base = &PyArrayDescr_Type;
61+
if (PyType_Ready((PyTypeObject *)&PyArray_StringDType) < 0) {
62+
return -1;
63+
diff --git a/numpy/_core/src/multiarray/temp_elide.c b/numpy/_core/src/multiarray/temp_elide.c
64+
index 662a2fa52b..791ede8c1a 100644
65+
--- a/numpy/_core/src/multiarray/temp_elide.c
66+
+++ b/numpy/_core/src/multiarray/temp_elide.c
67+
@@ -58,7 +58,7 @@
68+
* supported too by using the appropriate Windows APIs.
69+
*/
70+
71+
-#if defined HAVE_BACKTRACE && defined HAVE_DLFCN_H && ! defined PYPY_VERSION
72+
+#if defined HAVE_BACKTRACE && defined HAVE_DLFCN_H && ! defined PYPY_VERSION && !defined(GRAALVM_PYTHON)
73+
74+
#include <feature_detection_misc.h>
75+
76+
diff --git a/numpy/_core/src/npymath/ieee754.c.src b/numpy/_core/src/npymath/ieee754.c.src
77+
index 8fccc9a69b..3bb9cf0d58 100644
78+
--- a/numpy/_core/src/npymath/ieee754.c.src
79+
+++ b/numpy/_core/src/npymath/ieee754.c.src
80+
@@ -362,6 +362,11 @@ int npy_get_floatstatus_barrier(char* param)
81+
* By using a volatile, the compiler cannot reorder this call
82+
*/
83+
if (param != NULL) {
84+
+ // GraalPy change: the pointer needs to be dereferenced to establish
85+
+ // a data dependency to to ensure the compiler won't reorder the call
86+
+ if (points_to_py_handle_space(param)) {
87+
+ param = (char*)pointer_to_stub(param);
88+
+ }
89+
volatile char NPY_UNUSED(c) = *(char*)param;
90+
}
91+
92+
diff --git a/numpy/_core/src/npymath/ieee754.cpp b/numpy/_core/src/npymath/ieee754.cpp
93+
index 1c59bf300b..519fabc113 100644
94+
--- a/numpy/_core/src/npymath/ieee754.cpp
95+
+++ b/numpy/_core/src/npymath/ieee754.cpp
96+
@@ -428,6 +428,11 @@ npy_get_floatstatus_barrier(char *param)
97+
* By using a volatile, the compiler cannot reorder this call
98+
*/
99+
if (param != NULL) {
100+
+ // GraalPy change: the pointer needs to be dereferenced to establish
101+
+ // a data dependency to to ensure the compiler won't reorder the call
102+
+ if (points_to_py_handle_space(param)) {
103+
+ param = (char*)pointer_to_stub(param);
104+
+ }
105+
volatile char NPY_UNUSED(c) = *(char *)param;
106+
}
107+
108+
diff --git a/vendored-meson/meson/mesonbuild/utils/universal.py b/vendored-meson/meson/mesonbuild/utils/universal.py
109+
index 6aee268ee..539be8bab 100644
110+
--- a/vendored-meson/meson/mesonbuild/utils/universal.py
111+
+++ b/vendored-meson/meson/mesonbuild/utils/universal.py
112+
@@ -728,6 +728,7 @@ def windows_detect_native_arch() -> str:
113+
"""
114+
if sys.platform != 'win32':
115+
return ''
116+
+ return 'amd64' # Workaround for GraalPy bug on Windows with kernel32.GetCurrentProcess()
117+
try:
118+
import ctypes
119+
process_arch = ctypes.c_ushort()

graalpython/lib-graalpython/patches/pyarrow-12.0.0.patch

+1-1
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ index 0000000..c9826ce
115115
+ '-DARROW_WITH_BROTLI=ON',
116116
+ ])
117117
+ subprocess.check_call([
118-
+ 'cmake', '--build', str(build_dir), '--parallel',
118+
+ 'cmake', '--build', str(build_dir),
119119
+ ])
120120
+ subprocess.check_call([
121121
+ 'cmake', '--install', str(build_dir),

graalpython/lib-graalpython/patches/pyarrow-15.0.0.patch

+1-1
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ index 0000000..955ac8d
115115
+ '-DARROW_WITH_BROTLI=ON',
116116
+ ])
117117
+ subprocess.check_call([
118-
+ 'cmake', '--build', str(build_dir), '--parallel',
118+
+ 'cmake', '--build', str(build_dir),
119119
+ ])
120120
+ subprocess.check_call([
121121
+ 'cmake', '--install', str(build_dir),

graalpython/lib-graalpython/patches/pyarrow-19.0.0.patch

+25-1
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ index 0000000..0929c5f
115115
+ '-DARROW_WITH_BROTLI=ON',
116116
+ ])
117117
+ subprocess.check_call([
118-
+ 'cmake', '--build', str(build_dir), '--parallel',
118+
+ 'cmake', '--build', str(build_dir),
119119
+ ])
120120
+ subprocess.check_call([
121121
+ 'cmake', '--install', str(build_dir),
@@ -162,3 +162,27 @@ index ef2043f..cb08a86 100644
162162

163163
[project]
164164
name = "pyarrow"
165+
diff --git a/MANIFEST.in b/MANIFEST.in
166+
index ef2043f..cb08a86 100644
167+
--- a/MANIFEST.in
168+
+++ b/MANIFEST.in
169+
@@ -1,6 +1,4 @@
170+
include README.md
171+
-include ../LICENSE.txt
172+
-include ../NOTICE.txt
173+
174+
global-include CMakeLists.txt
175+
graft pyarrow
176+
diff --git a/setup.cfg b/setup.cfg
177+
index ef2043f..cb08a86 100644
178+
--- a/setup.cfg
179+
+++ b/setup.cfg
180+
@@ -1,7 +1,6 @@
181+
[metadata]
182+
license_files =
183+
- ../LICENSE.txt
184+
- ../NOTICE.txt
185+
+ README.md
186+
187+
[build_sphinx]
188+
source-dir = doc/

0 commit comments

Comments
 (0)