[GR-63270] Add scripts to build many packages for AI workloads

PullRequest: graalpython/3730

Author: timfel

.github/workflows/build-linux-aarch64-wheels.yml

@@ -29,13 +29,15 @@ jobs:
           dnf install -y /usr/bin/patch
       - name: Checkout
         uses: actions/checkout@main
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports"
+          cache: false
       - name: Build wheels
         run: |
+          export PIP_GRAALPY_PATCHES_URL="${GITHUB_WORKSPACE}/graalpython/lib-graalpython/patches"
           python3 -m venv wheelbuilder_venv
-          export PIP_EXTRA_INDEX_URL=https://pypi.org/simple
           wheelbuilder_venv/bin/python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }}
-      - name: Build wheels
-        run: python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }}
       - name: Store wheels
         uses: actions/upload-artifact@main
         with:

.github/workflows/build-linux-amd64-wheels.yml

@@ -27,10 +27,14 @@ jobs:
           dnf install -y /usr/bin/patch
       - name: Checkout
         uses: actions/checkout@main
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports"
+          cache: false
       - name: Build wheels
         run: |
+          export PIP_GRAALPY_PATCHES_URL="${GITHUB_WORKSPACE}/graalpython/lib-graalpython/patches"
           python3 -m venv wheelbuilder_venv
-          export PIP_EXTRA_INDEX_URL=https://pypi.org/simple
           wheelbuilder_venv/bin/python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }}
       - name: Store wheels
         uses: actions/upload-artifact@main

.github/workflows/build-macos-aarch64-wheels.yml

@@ -18,13 +18,17 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@main
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports"
+          cache: false
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
           python-version: 3.12
       - name: Build wheels
         run: |
-          export PIP_EXTRA_INDEX_URL=https://pypi.org/simple
+          export PIP_GRAALPY_PATCHES_URL="${GITHUB_WORKSPACE}/graalpython/lib-graalpython/patches"
           python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }}
       - name: Store wheels
         uses: actions/upload-artifact@main

.github/workflows/build-macos-amd64-wheels.yml

@@ -18,13 +18,17 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@main
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports"
+          cache: false
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
           python-version: 3.12
       - name: Build wheels
         run: |
-          export PIP_EXTRA_INDEX_URL=https://pypi.org/simple
+          export PIP_GRAALPY_PATCHES_URL="${GITHUB_WORKSPACE}/graalpython/lib-graalpython/patches"
           python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }}
       - name: Store wheels
         uses: actions/upload-artifact@main

.github/workflows/build-windows-amd64-wheels.yml

@@ -19,14 +19,19 @@ jobs:
       - uses: ilammy/msvc-dev-cmd@v1
       - name: Checkout
         uses: actions/checkout@main
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports"
+          cache: false
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
           python-version: 3.12
       - name: Build wheels
         run: |
           $env:PATH+=";C:\Program Files\Git\usr\bin"
-          $env:PIP_EXTRA_INDEX_URL="https://pypi.org/simple"
+          $env:PIP_GRAALPY_PATCHES_URL=$env:GITHUB_WORKSPACE
+          $env:PIP_GRAALPY_PATCHES_URL+="/graalpython/lib-graalpython/patches"
           python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }}
       - name: Store wheels
         uses: actions/upload-artifact@main

.gitignore

@@ -95,3 +95,4 @@ graalpython/com.oracle.graal.python.test/src/tests/cpyext/build/
 pom-mx.xml
 .venv
 !graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.jar
+/*-venv/

graalpython/lib-graalpython/_sysconfig.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -96,6 +96,12 @@ def _get_posix_vars():
     g['CFLAGS'] = ' '.join(cflags_default + [gnu_source])
     g['LDFLAGS'] = ""
     g['CCSHARED'] = fpic
+    if darwin_native:
+        # MACOSX_DEPLOYMENT_TARGET is taken from the minimum version we build
+        # GraalPy for, which is currently BigSur
+        g['MACOSX_DEPLOYMENT_TARGET'] = "11"
+    else:
+        g['MACOSX_DEPLOYMENT_TARGET'] = ""
     if darwin_native:
         g['LDFLAGS'] = "-bundle -undefined dynamic_lookup"
         ldshared_common = g['LDFLAGS']

graalpython/lib-graalpython/patches/metadata.toml

@@ -105,7 +105,7 @@ patch = 'cryptography-42.0.5.patch'
 license = 'Apache-2.0 OR BSD-3-Clause'
 
 [[Cython.rules]]
-version = '>= 3.0.10, < 3.0.12'
+version = '>= 3.0.10, <= 3.0.12'
 patch = 'Cython-3.0.10.patch'
 license = 'Apache-2.0'
 
@@ -301,6 +301,13 @@ patch = 'numpy-2.0.0.patch'
 license = 'BSD-3-Clause'
 dist-type = 'sdist'
 
+[[numpy.rules]]
+version = '== 2.2.3'
+patch = 'numpy-2.2.3.patch'
+license = 'BSD-3-Clause'
+dist-type = 'sdist'
+install-priority = 0
+
 [[numpy.rules]]
 version = '== 1.26.4'
 patch = 'numpy-1.26.4.patch'
@@ -448,6 +455,11 @@ version = '== 19.0.0'
 patch = 'pyarrow-19.0.0.patch'
 license = 'Apache-2.0'
 
+[[pyarrow.rules]]
+version = '== 19.0.1'
+patch = 'pyarrow-19.0.1.patch'
+license = 'Apache-2.0'
+
 [[pybind11.rules]]
 # Upstreamed
 install-priority = 0
@@ -484,6 +496,11 @@ version = '== 2.27.1'
 patch = 'pydantic-core-2.27.1.patch'
 license = 'MIT'
 
+[[pydantic-core.rules]]
+version = '== 2.27.2'
+patch = 'pydantic-core-2.27.1.patch'
+license = 'MIT'
+
 [[pydantic-core.rules]]
 version = '== 2.26.0'
 patch = 'pydantic-core-2.26.0.patch'
@@ -572,9 +589,23 @@ license = 'BSD-3-Clause'
 dist-type = 'sdist'
 
 [[rapidfuzz.rules]]
+version = '< 3.10'
 patch = 'rapidfuzz.patch'
 license = 'MIT'
 
+[[rapidfuzz.rules]]
+version = '>= 3.10.0, <= 3.12.2'
+patch = 'rapidfuzz-3.10.patch'
+license = 'MIT'
+
+[[levenshtein.add-sources]]
+version = '0.26.1'
+url = 'https://github.com/rapidfuzz/Levenshtein/archive/refs/tags/v0.26.1.zip'
+
+[[levenshtein.add-sources]]
+version = '0.27.1'
+url = 'https://github.com/rapidfuzz/Levenshtein/archive/refs/tags/v0.27.1.zip'
+
 [[ray.rules]]
 version = '== 2.3.0'
 patch = 'ray-2.3.0.patch'
@@ -611,6 +642,11 @@ version = '== 0.3.3'
 patch = 'safetensors-0.3.3.patch'
 license = 'Apache-2.0'
 
+[[scikit-build-core.rules]]
+version = '~= 0.11.0'
+patch = 'scikit-build-core.patch'
+license = 'Apache-2.0'
+
 [[scikit-learn.rules]]
 version = '== 1.5.2'
 patch = 'scikit-learn-1.5.2.patch'

graalpython/lib-graalpython/patches/numpy-2.2.3.patch

@@ -0,0 +1,119 @@
+diff --git a/numpy/_core/include/numpy/ndarrayobject.h b/numpy/_core/include/numpy/ndarrayobject.h
+index f06bafe5b5..52063a7d16 100644
+--- a/numpy/_core/include/numpy/ndarrayobject.h
++++ b/numpy/_core/include/numpy/ndarrayobject.h
+@@ -220,7 +220,7 @@ NPY_TITLE_KEY_check(PyObject *key, PyObject *value)
+     if (key == title) {
+         return 1;
+     }
+-#ifdef PYPY_VERSION
++#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON)
+     /*
+      * On PyPy, dictionary keys do not always preserve object identity.
+      * Fall back to comparison by value.
+diff --git a/numpy/_core/src/multiarray/compiled_base.c b/numpy/_core/src/multiarray/compiled_base.c
+index 48524aff4d..e55e2479e1 100644
+--- a/numpy/_core/src/multiarray/compiled_base.c
++++ b/numpy/_core/src/multiarray/compiled_base.c
+@@ -1455,6 +1455,7 @@ fail:
+ NPY_NO_EXPORT PyObject *
+ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *const *args, Py_ssize_t len_args)
+ {
++#if 0 // GraalPy change
+     PyObject *obj;
+     PyObject *str;
+     const char *docstr;
+@@ -1559,6 +1560,7 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *const *args, Py_ssize_t
+     }
+ 
+ #undef _ADDDOC
++#endif // GraalPy change
+ 
+     Py_RETURN_NONE;
+ }
+diff --git a/numpy/_core/src/multiarray/shape.c b/numpy/_core/src/multiarray/shape.c
+index 340fe7289a..8f13674383 100644
+--- a/numpy/_core/src/multiarray/shape.c
++++ b/numpy/_core/src/multiarray/shape.c
+@@ -97,6 +97,11 @@ PyArray_Resize(PyArrayObject *self, PyArray_Dims *newshape, int refcheck,
+                     "cannot resize an array with refcheck=True on PyPy.\n"
+                     "Use the np.resize function or refcheck=False");
+             return NULL;
++#elif defined(GRAALVM_PYTHON)
++            PyErr_SetString(PyExc_ValueError,
++                    "cannot resize an array with refcheck=True on GraalPy.\n"
++                    "Use the np.resize function or refcheck=False");
++            return NULL;
+ #else
+             refcnt = Py_REFCNT(self);
+ #endif /* PYPY_VERSION */
+diff --git a/numpy/_core/src/multiarray/stringdtype/dtype.c b/numpy/_core/src/multiarray/stringdtype/dtype.c
+index cb8265dd3d..0dabb49346 100644
+--- a/numpy/_core/src/multiarray/stringdtype/dtype.c
++++ b/numpy/_core/src/multiarray/stringdtype/dtype.c
+@@ -841,7 +841,7 @@ init_string_dtype(void)
+     };
+ 
+     /* Loaded dynamically, so needs to be set here: */
+-    Py_TYPE(((PyObject *)&PyArray_StringDType)) = &PyArrayDTypeMeta_Type;
++    Py_SET_TYPE((PyObject *)&PyArray_StringDType, &PyArrayDTypeMeta_Type);
+     ((PyTypeObject *)&PyArray_StringDType)->tp_base = &PyArrayDescr_Type;
+     if (PyType_Ready((PyTypeObject *)&PyArray_StringDType) < 0) {
+         return -1;
+diff --git a/numpy/_core/src/multiarray/temp_elide.c b/numpy/_core/src/multiarray/temp_elide.c
+index 662a2fa52b..791ede8c1a 100644
+--- a/numpy/_core/src/multiarray/temp_elide.c
++++ b/numpy/_core/src/multiarray/temp_elide.c
+@@ -58,7 +58,7 @@
+  * supported too by using the appropriate Windows APIs.
+  */
+ 
+-#if defined HAVE_BACKTRACE && defined HAVE_DLFCN_H && ! defined PYPY_VERSION
++#if defined HAVE_BACKTRACE && defined HAVE_DLFCN_H && ! defined PYPY_VERSION && !defined(GRAALVM_PYTHON)
+ 
+ #include <feature_detection_misc.h>
+ 
+diff --git a/numpy/_core/src/npymath/ieee754.c.src b/numpy/_core/src/npymath/ieee754.c.src
+index 8fccc9a69b..3bb9cf0d58 100644
+--- a/numpy/_core/src/npymath/ieee754.c.src
++++ b/numpy/_core/src/npymath/ieee754.c.src
+@@ -362,6 +362,11 @@ int npy_get_floatstatus_barrier(char* param)
+      * By using a volatile, the compiler cannot reorder this call
+      */
+     if (param != NULL) {
++        // GraalPy change: the pointer needs to be dereferenced to establish
++        // a data dependency to to ensure the compiler won't reorder the call
++        if (points_to_py_handle_space(param)) {
++            param = (char*)pointer_to_stub(param);
++        }
+         volatile char NPY_UNUSED(c) = *(char*)param;
+     }
+ 
+diff --git a/numpy/_core/src/npymath/ieee754.cpp b/numpy/_core/src/npymath/ieee754.cpp
+index 1c59bf300b..519fabc113 100644
+--- a/numpy/_core/src/npymath/ieee754.cpp
++++ b/numpy/_core/src/npymath/ieee754.cpp
+@@ -428,6 +428,11 @@ npy_get_floatstatus_barrier(char *param)
+      * By using a volatile, the compiler cannot reorder this call
+      */
+     if (param != NULL) {
++        // GraalPy change: the pointer needs to be dereferenced to establish
++        // a data dependency to to ensure the compiler won't reorder the call
++        if (points_to_py_handle_space(param)) {
++            param = (char*)pointer_to_stub(param);
++        }
+         volatile char NPY_UNUSED(c) = *(char *)param;
+     }
+ 
+diff --git a/vendored-meson/meson/mesonbuild/utils/universal.py b/vendored-meson/meson/mesonbuild/utils/universal.py
+index 6aee268ee..539be8bab 100644
+--- a/vendored-meson/meson/mesonbuild/utils/universal.py
++++ b/vendored-meson/meson/mesonbuild/utils/universal.py
+@@ -728,6 +728,7 @@ def windows_detect_native_arch() -> str:
+     """
+     if sys.platform != 'win32':
+         return ''
++    return 'amd64'  # Workaround for GraalPy bug on Windows with kernel32.GetCurrentProcess()
+     try:
+         import ctypes
+         process_arch = ctypes.c_ushort()

graalpython/lib-graalpython/patches/pyarrow-12.0.0.patch

@@ -115,7 +115,7 @@ index 0000000..c9826ce
 +            '-DARROW_WITH_BROTLI=ON',
 +        ])
 +        subprocess.check_call([
-+            'cmake', '--build', str(build_dir), '--parallel',
++            'cmake', '--build', str(build_dir),
 +        ])
 +        subprocess.check_call([
 +            'cmake', '--install', str(build_dir),

graalpython/lib-graalpython/patches/pyarrow-15.0.0.patch

@@ -115,7 +115,7 @@ index 0000000..955ac8d
 +            '-DARROW_WITH_BROTLI=ON',
 +        ])
 +        subprocess.check_call([
-+            'cmake', '--build', str(build_dir), '--parallel',
++            'cmake', '--build', str(build_dir),
 +        ])
 +        subprocess.check_call([
 +            'cmake', '--install', str(build_dir),

graalpython/lib-graalpython/patches/pyarrow-19.0.0.patch

@@ -115,7 +115,7 @@ index 0000000..0929c5f
 +            '-DARROW_WITH_BROTLI=ON',
 +        ])
 +        subprocess.check_call([
-+            'cmake', '--build', str(build_dir), '--parallel',
++            'cmake', '--build', str(build_dir),
 +        ])
 +        subprocess.check_call([
 +            'cmake', '--install', str(build_dir),
@@ -162,3 +162,27 @@ index ef2043f..cb08a86 100644
 
  [project]
  name = "pyarrow"
+diff --git a/MANIFEST.in b/MANIFEST.in
+index ef2043f..cb08a86 100644
+--- a/MANIFEST.in
++++ b/MANIFEST.in
+@@ -1,6 +1,4 @@
+ include README.md
+-include ../LICENSE.txt
+-include ../NOTICE.txt
+ 
+ global-include CMakeLists.txt
+ graft pyarrow
+diff --git a/setup.cfg b/setup.cfg
+index ef2043f..cb08a86 100644
+--- a/setup.cfg
++++ b/setup.cfg
+@@ -1,7 +1,6 @@
+ [metadata]
+ license_files = 
+-	../LICENSE.txt
+-	../NOTICE.txt
++	README.md
+ 
+ [build_sphinx]
+ source-dir = doc/