Backports for 0.23.x release

.github/workflows/benches.yml

@@ -15,8 +15,7 @@ concurrency:
 
 jobs:
   benchmarks:
-    # No support for 24.04, see https://github.com/CodSpeedHQ/runner/issues/42
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5

.github/workflows/ci.yml

@@ -244,6 +244,7 @@ jobs:
             "3.13",
             "pypy3.9",
             "pypy3.10",
+            "pypy3.11",
             "graalpy24.0",
           ]
         platform:

newsfragments/4760.packaging.md

@@ -0,0 +1 @@
+add support for PyPy3.11

newsfragments/4879.fixed.md

@@ -0,0 +1 @@
+ * fixed spurious `test_double` failures.

newsfragments/4902.fixed.md

@@ -0,0 +1 @@
+* Fixed thread-unsafe implementation of freelist pyclasses on the free-threaded build.

newsfragments/4921.fixed.md

@@ -0,0 +1 @@
+* Reenabled a workaround for situations where CPython incorrectly does not add `__builtins__` to `__globals__` in code executed by `Python::py_run`.

noxfile.py

@@ -32,7 +32,7 @@
 PYO3_GUIDE_TARGET = PYO3_TARGET / "guide"
 PYO3_DOCS_TARGET = PYO3_TARGET / "doc"
 PY_VERSIONS = ("3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13")
-PYPY_VERSIONS = ("3.9", "3.10")
+PYPY_VERSIONS = ("3.9", "3.10", "3.11")
 FREE_THREADED_BUILD = bool(sysconfig.get_config_var("Py_GIL_DISABLED"))
 
 
@@ -673,8 +673,8 @@ def test_version_limits(session: nox.Session):
         config_file.set("PyPy", "3.8")
         _run_cargo(session, "check", env=env, expect_error=True)
 
-        assert "3.11" not in PYPY_VERSIONS
-        config_file.set("PyPy", "3.11")
+        assert "3.12" not in PYPY_VERSIONS
+        config_file.set("PyPy", "3.12")
         _run_cargo(session, "check", env=env, expect_error=True)
 
 

pyo3-ffi/build.rs

@@ -25,7 +25,7 @@ const SUPPORTED_VERSIONS_PYPY: SupportedVersions = SupportedVersions {
     min: PythonVersion { major: 3, minor: 9 },
     max: PythonVersion {
         major: 3,
-        minor: 10,
+        minor: 11,
     },
 };
 

pyo3-ffi/src/abstract_.rs

@@ -3,13 +3,19 @@ use crate::pyport::Py_ssize_t;
 use std::os::raw::{c_char, c_int};
 
 #[inline]
-#[cfg(all(not(Py_3_13), not(PyPy)))] // CPython exposed as a function in 3.13, in object.h
+#[cfg(all(
+    not(Py_3_13), // CPython exposed as a function in 3.13, in object.h 
+    not(all(PyPy, not(Py_3_11))) // PyPy exposed as a function until PyPy 3.10, macro in 3.11+
+))]
 pub unsafe fn PyObject_DelAttrString(o: *mut PyObject, attr_name: *const c_char) -> c_int {
     PyObject_SetAttrString(o, attr_name, std::ptr::null_mut())
 }
 
 #[inline]
-#[cfg(all(not(Py_3_13), not(PyPy)))] // CPython exposed as a function in 3.13, in object.h
+#[cfg(all(
+    not(Py_3_13), // CPython exposed as a function in 3.13, in object.h 
+    not(all(PyPy, not(Py_3_11))) // PyPy exposed as a function until PyPy 3.10, macro in 3.11+
+))]
 pub unsafe fn PyObject_DelAttr(o: *mut PyObject, attr_name: *mut PyObject) -> c_int {
     PyObject_SetAttr(o, attr_name, std::ptr::null_mut())
 }

pyo3-ffi/src/cpython/abstract_.rs

@@ -1,5 +1,5 @@
 use crate::{PyObject, Py_ssize_t};
-#[cfg(not(all(Py_3_11, GraalPy)))]
+#[cfg(any(all(Py_3_8, not(any(PyPy, GraalPy))), not(Py_3_11)))]
 use std::os::raw::c_char;
 use std::os::raw::c_int;
 

pyo3-ffi/src/cpython/genobject.rs

@@ -2,7 +2,7 @@ use crate::object::*;
 use crate::PyFrameObject;
 #[cfg(not(any(PyPy, GraalPy)))]
 use crate::_PyErr_StackItem;
-#[cfg(all(Py_3_11, not(GraalPy)))]
+#[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))]
 use std::os::raw::c_char;
 use std::os::raw::c_int;
 use std::ptr::addr_of_mut;

pyo3-ffi/src/cpython/mod.rs

@@ -71,7 +71,7 @@ pub use self::object::*;
 pub use self::objimpl::*;
 pub use self::pydebug::*;
 pub use self::pyerrors::*;
-#[cfg(Py_3_11)]
+#[cfg(all(Py_3_11, not(PyPy)))]
 pub use self::pyframe::*;
 #[cfg(all(Py_3_8, not(PyPy)))]
 pub use self::pylifecycle::*;

pyo3-ffi/src/cpython/object.rs

@@ -310,9 +310,9 @@ pub struct PyHeapTypeObject {
     pub ht_cached_keys: *mut c_void,
     #[cfg(Py_3_9)]
     pub ht_module: *mut object::PyObject,
-    #[cfg(Py_3_11)]
+    #[cfg(all(Py_3_11, not(PyPy)))]
     _ht_tpname: *mut c_char,
-    #[cfg(Py_3_11)]
+    #[cfg(all(Py_3_11, not(PyPy)))]
     _spec_cache: _specialization_cache,
 }
 

pyo3-ffi/src/cpython/objimpl.rs

@@ -1,4 +1,4 @@
-#[cfg(not(all(Py_3_11, GraalPy)))]
+#[cfg(not(all(Py_3_11, any(PyPy, GraalPy))))]
 use libc::size_t;
 use std::os::raw::c_int;
 

pyo3-ffi/src/cpython/pyframe.rs

@@ -1,2 +1,2 @@
-#[cfg(Py_3_11)]
+#[cfg(all(Py_3_11, not(PyPy)))]
 opaque_struct!(_PyInterpreterFrame);

pyo3-ffi/src/cpython/pystate.rs

@@ -69,21 +69,21 @@ extern "C" {
     pub fn PyThreadState_DeleteCurrent();
 }
 
-#[cfg(all(Py_3_9, not(Py_3_11)))]
+#[cfg(all(Py_3_9, not(any(Py_3_11, PyPy))))]
 pub type _PyFrameEvalFunction = extern "C" fn(
     *mut crate::PyThreadState,
     *mut crate::PyFrameObject,
     c_int,
 ) -> *mut crate::object::PyObject;
 
-#[cfg(Py_3_11)]
+#[cfg(all(Py_3_11, not(PyPy)))]
 pub type _PyFrameEvalFunction = extern "C" fn(
     *mut crate::PyThreadState,
     *mut crate::_PyInterpreterFrame,
     c_int,
 ) -> *mut crate::object::PyObject;
 
-#[cfg(Py_3_9)]
+#[cfg(all(Py_3_9, not(PyPy)))]
 extern "C" {
     /// Get the frame evaluation function.
     pub fn _PyInterpreterState_GetEvalFrameFunc(

pyo3-ffi/src/cpython/unicodeobject.rs

@@ -1,4 +1,4 @@
-#[cfg(not(PyPy))]
+#[cfg(any(Py_3_11, not(PyPy)))]
 use crate::Py_hash_t;
 use crate::{PyObject, Py_UCS1, Py_UCS2, Py_UCS4, Py_ssize_t};
 use libc::wchar_t;
@@ -251,7 +251,7 @@ impl From<PyASCIIObjectState> for u32 {
 pub struct PyASCIIObject {
     pub ob_base: PyObject,
     pub length: Py_ssize_t,
-    #[cfg(not(PyPy))]
+    #[cfg(any(Py_3_11, not(PyPy)))]
     pub hash: Py_hash_t,
     /// A bit field with various properties.
     ///

pyo3-ffi/src/object.rs

@@ -436,7 +436,7 @@ extern "C" {
         arg2: *const c_char,
         arg3: *mut PyObject,
     ) -> c_int;
-    #[cfg(any(Py_3_13, PyPy))] // CPython defined in 3.12 as an inline function in abstract.h
+    #[cfg(any(Py_3_13, all(PyPy, not(Py_3_11))))] // CPython defined in 3.12 as an inline function in abstract.h
     #[cfg_attr(PyPy, link_name = "PyPyObject_DelAttrString")]
     pub fn PyObject_DelAttrString(arg1: *mut PyObject, arg2: *const c_char) -> c_int;
     #[cfg_attr(PyPy, link_name = "PyPyObject_HasAttrString")]
@@ -460,7 +460,7 @@ extern "C" {
     #[cfg_attr(PyPy, link_name = "PyPyObject_SetAttr")]
     pub fn PyObject_SetAttr(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject)
         -> c_int;
-    #[cfg(any(Py_3_13, PyPy))] // CPython defined in 3.12 as an inline function in abstract.h
+    #[cfg(any(Py_3_13, all(PyPy, not(Py_3_11))))] // CPython defined in 3.12 as an inline function in abstract.h
     #[cfg_attr(PyPy, link_name = "PyPyObject_DelAttr")]
     pub fn PyObject_DelAttr(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int;
     #[cfg_attr(PyPy, link_name = "PyPyObject_HasAttr")]

pyo3-ffi/src/pybuffer.rs

@@ -103,7 +103,11 @@ extern "C" {
 }
 
 /// Maximum number of dimensions
-pub const PyBUF_MAX_NDIM: c_int = if cfg!(PyPy) { 36 } else { 64 };
+pub const PyBUF_MAX_NDIM: usize = if cfg!(all(PyPy, not(Py_3_11))) {
+    36
+} else {
+    64
+};
 
 /* Flags for getting buffers */
 pub const PyBUF_SIMPLE: c_int = 0;

pyo3-ffi/src/pyerrors.rs

@@ -116,6 +116,7 @@ extern "C" {
     #[cfg_attr(PyPy, link_name = "PyPyExc_BaseException")]
     pub static mut PyExc_BaseException: *mut PyObject;
     #[cfg(Py_3_11)]
+    #[cfg_attr(PyPy, link_name = "PyPyExc_BaseExceptionGroup")]
     pub static mut PyExc_BaseExceptionGroup: *mut PyObject;
     #[cfg_attr(PyPy, link_name = "PyPyExc_Exception")]
     pub static mut PyExc_Exception: *mut PyObject;

pyo3-macros-backend/src/pyclass.rs

@@ -2395,15 +2395,13 @@ impl<'a> PyClassImplsBuilder<'a> {
             quote! {
                 impl #pyo3_path::impl_::pyclass::PyClassWithFreeList for #cls {
                     #[inline]
-                    fn get_free_list(py: #pyo3_path::Python<'_>) -> &mut #pyo3_path::impl_::freelist::FreeList<*mut #pyo3_path::ffi::PyObject> {
-                        static mut FREELIST: *mut #pyo3_path::impl_::freelist::FreeList<*mut #pyo3_path::ffi::PyObject> = 0 as *mut _;
-                        unsafe {
-                            if FREELIST.is_null() {
-                                FREELIST = ::std::boxed::Box::into_raw(::std::boxed::Box::new(
-                                    #pyo3_path::impl_::freelist::FreeList::with_capacity(#freelist)));
-                            }
-                            &mut *FREELIST
-                        }
+                    fn get_free_list(py: #pyo3_path::Python<'_>) -> &'static ::std::sync::Mutex<#pyo3_path::impl_::freelist::PyObjectFreeList> {
+                        static FREELIST: #pyo3_path::sync::GILOnceCell<::std::sync::Mutex<#pyo3_path::impl_::freelist::PyObjectFreeList>> = #pyo3_path::sync::GILOnceCell::new();
+                        // If there's a race to fill the cell, the object created
+                        // by the losing thread will be deallocated via RAII
+                        &FREELIST.get_or_init(py, || {
+                            ::std::sync::Mutex::new(#pyo3_path::impl_::freelist::PyObjectFreeList::with_capacity(#freelist))
+                        })
                     }
                 }
             }

pytests/tests/test_othermod.py

@@ -3,11 +3,16 @@
 
 from pyo3_pytests import othermod
 
-INTEGER32_ST = st.integers(min_value=(-(2**31)), max_value=(2**31 - 1))
+INTEGER31_ST = st.integers(min_value=(-(2**30)), max_value=(2**30 - 1))
 USIZE_ST = st.integers(min_value=othermod.USIZE_MIN, max_value=othermod.USIZE_MAX)
 
 
-@given(x=INTEGER32_ST)
+# If the full 32 bits are used here, then you can get failures that look like this:
+# hypothesis.errors.FailedHealthCheck: It looks like your strategy is filtering out a lot of data.
+# Health check found 50 filtered examples but only 7 good ones.
+#
+# Limit the range to 31 bits to avoid this problem.
+@given(x=INTEGER31_ST)
 def test_double(x):
     expected = x * 2
     assume(-(2**31) <= expected <= (2**31 - 1))

src/ffi/tests.rs

@@ -121,7 +121,7 @@ fn ascii_object_bitfield() {
     let mut o = PyASCIIObject {
         ob_base,
         length: 0,
-        #[cfg(not(PyPy))]
+        #[cfg(any(Py_3_11, not(PyPy)))]
         hash: 0,
         state: 0u32,
         #[cfg(not(Py_3_12))]

src/impl_/freelist.rs

@@ -8,58 +8,64 @@
 //!
 //! [1]: https://en.wikipedia.org/wiki/Free_list
 
+use crate::ffi;
 use std::mem;
 
-/// Represents a slot of a [`FreeList`].
-pub enum Slot<T> {
+/// Represents a slot of a [`PyObjectFreeList`].
+enum PyObjectSlot {
     /// A free slot.
     Empty,
     /// An allocated slot.
-    Filled(T),
+    Filled(*mut ffi::PyObject),
 }
 
-/// A free allocation list.
+// safety: access is guarded by a per-pyclass mutex
+unsafe impl Send for PyObjectSlot {}
+
+/// A free allocation list for PyObject ffi pointers.
 ///
 /// See [the parent module](crate::impl_::freelist) for more details.
-pub struct FreeList<T> {
-    entries: Vec<Slot<T>>,
+pub struct PyObjectFreeList {
+    entries: Box<[PyObjectSlot]>,
     split: usize,
     capacity: usize,
 }
 
-impl<T> FreeList<T> {
-    /// Creates a new `FreeList` instance with specified capacity.
-    pub fn with_capacity(capacity: usize) -> FreeList<T> {
-        let entries = (0..capacity).map(|_| Slot::Empty).collect::<Vec<_>>();
+impl PyObjectFreeList {
+    /// Creates a new `PyObjectFreeList` instance with specified capacity.
+    pub fn with_capacity(capacity: usize) -> PyObjectFreeList {
+        let entries = (0..capacity)
+            .map(|_| PyObjectSlot::Empty)
+            .collect::<Box<[_]>>();
 
-        FreeList {
+        PyObjectFreeList {
             entries,
             split: 0,
             capacity,
         }
     }
 
     /// Pops the first non empty item.
-    pub fn pop(&mut self) -> Option<T> {
+    pub fn pop(&mut self) -> Option<*mut ffi::PyObject> {
         let idx = self.split;
         if idx == 0 {
             None
         } else {
-            match mem::replace(&mut self.entries[idx - 1], Slot::Empty) {
-                Slot::Filled(v) => {
+            match mem::replace(&mut self.entries[idx - 1], PyObjectSlot::Empty) {
+                PyObjectSlot::Filled(v) => {
                     self.split = idx - 1;
                     Some(v)
                 }
-                _ => panic!("FreeList is corrupt"),
+                _ => panic!("PyObjectFreeList is corrupt"),
             }
         }
     }
 
-    /// Inserts a value into the list. Returns `Some(val)` if the `FreeList` is full.
-    pub fn insert(&mut self, val: T) -> Option<T> {
+    /// Inserts a value into the list. Returns `Some(val)` if the `PyObjectFreeList` is full.
+    pub fn insert(&mut self, val: *mut ffi::PyObject) -> Option<*mut ffi::PyObject> {
         let next = self.split + 1;
         if next < self.capacity {
-            self.entries[self.split] = Slot::Filled(val);
+            self.entries[self.split] = PyObjectSlot::Filled(val);
             self.split = next;
             None
         } else {

src/impl_/pyclass.rs

@@ -2,7 +2,7 @@ use crate::{
     exceptions::{PyAttributeError, PyNotImplementedError, PyRuntimeError, PyValueError},
     ffi,
     impl_::{
-        freelist::FreeList,
+        freelist::PyObjectFreeList,
         pycell::{GetBorrowChecker, PyClassMutability, PyClassObjectLayout},
         pyclass_init::PyObjectInit,
         pymethods::{PyGetterDef, PyMethodDefType},
@@ -20,6 +20,7 @@ use std::{
     marker::PhantomData,
     os::raw::{c_int, c_void},
     ptr::NonNull,
+    sync::Mutex,
     thread,
 };
 
@@ -910,7 +911,7 @@ use super::{pycell::PyClassObject, pymethods::BoundRef};
 /// Do not implement this trait manually. Instead, use `#[pyclass(freelist = N)]`
 /// on a Rust struct to implement it.
 pub trait PyClassWithFreeList: PyClass {
-    fn get_free_list(py: Python<'_>) -> &mut FreeList<*mut ffi::PyObject>;
+    fn get_free_list(py: Python<'_>) -> &'static Mutex<PyObjectFreeList>;
 }
 
 /// Implementation of tp_alloc for `freelist` classes.
@@ -931,7 +932,9 @@ pub unsafe extern "C" fn alloc_with_freelist<T: PyClassWithFreeList>(
     // If this type is a variable type or the subtype is not equal to this type, we cannot use the
     // freelist
     if nitems == 0 && subtype == self_type {
-        if let Some(obj) = T::get_free_list(py).pop() {
+        let mut free_list = T::get_free_list(py).lock().unwrap();
+        if let Some(obj) = free_list.pop() {
+            drop(free_list);
             ffi::PyObject_Init(obj, subtype);
             return obj as _;
         }
@@ -951,7 +954,11 @@ pub unsafe extern "C" fn free_with_freelist<T: PyClassWithFreeList>(obj: *mut c_
         T::type_object_raw(Python::assume_gil_acquired()),
         ffi::Py_TYPE(obj)
     );
-    if let Some(obj) = T::get_free_list(Python::assume_gil_acquired()).insert(obj) {
+    let mut free_list = T::get_free_list(Python::assume_gil_acquired())
+        .lock()
+        .unwrap();
+    if let Some(obj) = free_list.insert(obj) {
+        drop(free_list);
         let ty = ffi::Py_TYPE(obj);
 
         // Deduce appropriate inverse of PyType_GenericAlloc