Merge branch 'main' into enum-feature-variants

Author: jeff-k

.github/workflows/ci.yml

@@ -513,7 +513,7 @@ jobs:
         with:
           components: rust-src
       # TODO: replace with setup-python when there is support
-      - uses: deadsnakes/action@v3.1.0
+      - uses: deadsnakes/action@v3.2.0
         with:
           python-version: '3.13-dev'
           nogil: true

Cargo.toml

@@ -34,7 +34,7 @@ inventory = { version = "0.3.0", optional = true }
 # crate integrations that can be added using the eponymous features
 anyhow = { version = "1.0.1", optional = true }
 chrono = { version = "0.4.25", default-features = false, optional = true }
-chrono-tz = { version = ">= 0.6, < 0.10", default-features = false, optional = true }
+chrono-tz = { version = ">= 0.6, < 0.11", default-features = false, optional = true }
 either = { version = "1.9", optional = true }
 eyre = { version = ">= 0.4, < 0.7", optional = true }
 hashbrown = { version = ">= 0.9, < 0.15", optional = true }
@@ -52,7 +52,7 @@ portable-atomic = "1.0"
 [dev-dependencies]
 assert_approx_eq = "1.1.0"
 chrono = "0.4.25"
-chrono-tz = ">= 0.6, < 0.10"
+chrono-tz = ">= 0.6, < 0.11"
 # Required for "and $N others" normalization
 trybuild = ">=1.0.70"
 proptest = { version = "1.0", default-features = false, features = ["std"] }

Contributing.md

@@ -53,7 +53,7 @@ nox -s docs -- open
 #### Doctests
 
 We use lots of code blocks in our docs. Run `cargo test --doc` when making changes to check that
-the doctests still work, or `cargo test` to run all the tests including doctests. See
+the doctests still work, or `cargo test` to run all the Rust tests including doctests. See
 https://doc.rust-lang.org/rustdoc/documentation-tests.html for a guide on doctests.
 
 #### Building the guide
@@ -86,16 +86,15 @@ Everybody is welcome to submit comments on open PRs. Please help ensure new PyO3
 
 Here are a few things to note when you are writing PRs.
 
-### Continuous Integration
+### Testing and Continuous Integration
 
 The PyO3 repo uses GitHub Actions. PRs are blocked from merging if CI is not successful. Formatting, linting and tests are checked for all Rust and Python code. In addition, all warnings in Rust code are disallowed (using `RUSTFLAGS="-D warnings"`).
 
 Tests run with all supported Python versions with the latest stable Rust compiler, as well as for Python 3.9 with the minimum supported Rust version.
 
 If you are adding a new feature, you should add it to the `full` feature in our *Cargo.toml** so that it is tested in CI.
 
-You can run these tests yourself with
-`nox`. Use  `nox -l` to list the full set of subcommands you can run.
+You can run these checks yourself with `nox`. Use  `nox -l` to list the full set of subcommands you can run.
 
 #### Linting Python code
 `nox -s ruff`
@@ -110,7 +109,7 @@ You can run these tests yourself with
 `nox -s clippy-all`
 
 #### Tests
-`cargo test --features full`
+`nox -s test` or `cargo test` for Rust tests only, `nox -f pytests/noxfile.py -s test` for Python tests only
 
 #### Check all conditional compilation
 `nox -s check-feature-powerset`

LICENSE-APACHE

@@ -219,6 +219,7 @@ about this topic.
 - [river](https://github.com/online-ml/river) _Online machine learning in python, the computationally heavy statistics algorithms are implemented in Rust._
 - [robyn](https://github.com/sparckles/Robyn) A Super Fast Async Python Web Framework with a Rust runtime.
 - [rust-python-coverage](https://github.com/cjermain/rust-python-coverage) _Example PyO3 project with automated test coverage for Rust and Python._
+- [sail](https://github.com/lakehq/sail) _Unifying stream, batch, and AI workloads with Apache Spark compatibility._
 - [tiktoken](https://github.com/openai/tiktoken) _A fast BPE tokeniser for use with OpenAI's models._
 - [tokenizers](https://github.com/huggingface/tokenizers/tree/main/bindings/python) _Python bindings to the Hugging Face tokenizers (NLP) written in Rust._
 - [tzfpy](http://github.com/ringsaturn/tzfpy) _A fast package to convert longitude/latitude to timezone name._

README.md

@@ -51,7 +51,7 @@ impl PyCounter {
         println!("{} has been called {} time(s).", name, new_count);
 
         // After doing something, we finally forward the call to the wrapped function
-        let ret = self.wraps.call_bound(py, args, kwargs)?;
+        let ret = self.wraps.call(py, args, kwargs)?;
 
         // We could do something with the return value of
         // the function before returning it

examples/decorator/src/lib.rs

@@ -13,7 +13,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     //do useful work
     Python::with_gil(|py| {
         //add the current directory to import path of Python (do not use this in production!)
-        let syspath: &PyList = py.import("sys")?.getattr("path")?.extract()?;
+        let syspath: Bound<PyList> = py.import("sys")?.getattr("path")?.extract()?;
         syspath.insert(0, &path)?;
         println!("Import path is: {:?}", syspath);
 

examples/plugin/src/main.rs

@@ -92,8 +92,10 @@ If you're packaging your library for redistribution, you should indicated the Py
 
 To use PyO3 with bazel one needs to manually configure PyO3, PyO3-ffi and PyO3-macros. In particular, one needs to make sure that it is compiled with the right python flags for the version you intend to use.
 For example see:
-1. https://github.com/OliverFM/pytorch_with_gazelle -- for a minimal example of a repo that can use PyO3, PyTorch and Gazelle to generate python Build files.
-2. https://github.com/TheButlah/rules_pyo3 -- which has more extensive support, but is outdated.
+
+1. [github.com/abrisco/rules_pyo3](https://github.com/abrisco/rules_pyo3) -- General rules for building extension modules.
+2. [github.com/OliverFM/pytorch_with_gazelle](https://github.com/OliverFM/pytorch_with_gazelle) -- for a minimal example of a repo that can use PyO3, PyTorch and Gazelle to generate python Build files.
+3. [github.com/TheButlah/rules_pyo3](https://github.com/TheButlah/rules_pyo3) -- is somewhat dated.
 
 #### Platform tags
 

guide/src/building-and-distribution.md

@@ -330,7 +330,7 @@ or [`PyRefMut`] instead of `&mut self`.
 Then you can access a parent class by `self_.as_super()` as `&PyRef<Self::BaseClass>`,
 or by `self_.into_super()` as `PyRef<Self::BaseClass>` (and similar for the `PyRefMut`
 case). For convenience, `self_.as_ref()` can also be used to get `&Self::BaseClass`
-directly; however, this approach does not let you access base clases higher in the
+directly; however, this approach does not let you access base classes higher in the
 inheritance hierarchy, for which you would need to chain multiple `as_super` or
 `into_super` calls.
 

guide/src/class.md

@@ -139,7 +139,7 @@ This is purely additional and should just extend the possible return types.
 
 PyO3 0.23 introduces a new unified `IntoPyObject` trait to convert Rust types into Python objects.
 Notable features of this new trait:
-- conversions can now return an error 
+- conversions can now return an error
 - compared to `IntoPy<T>` the generic `T` moved into an associated type, so
   - there is now only one way to convert a given type
   - the output type is stronger typed and may return any Python type instead of just `PyAny`
@@ -150,7 +150,7 @@ All PyO3 provided types as well as `#[pyclass]`es already implement `IntoPyObjec
 need to adapt an implementation of `IntoPyObject` to stay compatible with the Python APIs.
 
 
-Before: 
+Before:
 ```rust
 # use pyo3::prelude::*;
 # #[allow(dead_code)]
@@ -198,6 +198,80 @@ impl<'a, 'py> IntoPyObject<'py> for &'a MyPyObjectWrapper {
 ```
 </details>
 
+### Free-threaded Python Support
+<details open>
+<summary><small>Click to expand</small></summary>
+
+PyO3 0.23 introduces preliminary support for the new free-threaded build of
+CPython 3.13. PyO3 features that implicitly assumed the existence of the GIL
+are not exposed in the free-threaded build, since they are no longer safe.
+
+If you make use of these features then you will need to account for the
+unavailability of this API in the free-threaded build. One way to handle it is
+via conditional compilation -- extensions built for the free-threaded build will
+have the `Py_GIL_DISABLED` attribute defined.
+
+### `GILProtected`
+
+`GILProtected` allows mutable access to static data by leveraging the GIL to
+lock concurrent access from other threads. In free-threaded python there is no
+GIL, so you will need to replace this type with some other form of locking. In
+many cases, `std::sync::Atomic` or `std::sync::Mutex` will be sufficient. If the
+locks do not guard the execution of arbitrary Python code or use of the CPython
+C API then conditional compilation is likely unnecessary since `GILProtected`
+was not needed in the first place.
+
+Before:
+
+```rust
+# fn main() {
+# #[cfg(not(Py_GIL_DISABLED))] {
+# use pyo3::prelude::*;
+use pyo3::sync::GILProtected;
+use pyo3::types::{PyDict, PyNone};
+use std::cell::RefCell;
+
+static OBJECTS: GILProtected<RefCell<Vec<Py<PyDict>>>> =
+    GILProtected::new(RefCell::new(Vec::new()));
+
+Python::with_gil(|py| {
+    // stand-in for something that executes arbitrary python code
+    let d = PyDict::new(py);
+    d.set_item(PyNone::get(py), PyNone::get(py)).unwrap();
+    OBJECTS.get(py).borrow_mut().push(d.unbind());
+});
+# }}
+```
+
+After:
+
+```rust
+# use pyo3::prelude::*;
+# fn main() {
+use pyo3::types::{PyDict, PyNone};
+use std::sync::Mutex;
+
+static OBJECTS: Mutex<Vec<Py<PyDict>>> = Mutex::new(Vec::new());
+
+Python::with_gil(|py| {
+    // stand-in for something that executes arbitrary python code
+    let d = PyDict::new(py);
+    d.set_item(PyNone::get(py), PyNone::get(py)).unwrap();
+    // we're not executing python code while holding the lock, so GILProtected
+    // was never needed
+    OBJECTS.lock().unwrap().push(d.unbind());
+});
+# }
+```
+
+If you are executing arbitrary Python code while holding the lock, then you will
+need to use conditional compilation to use `GILProtected` on GIL-enabled python
+builds and mutexes otherwise. Python 3.13 introduces `PyMutex`, which releases
+the GIL while the lock is held, so that is another option if you only need to
+support newer Python versions.
+
+</details>
+
 ## from 0.21.* to 0.22
 
 ### Deprecation of `gil-refs` feature continues
@@ -248,6 +322,8 @@ If you rely on `impl<T> Clone for Py<T>` to fulfil trait requirements imposed by
 
 However, take care to note that the behaviour is different from previous versions. If `Clone` was called without the GIL being held, we tried to delay the application of these reference count increments until PyO3-based code would re-acquire it. This turned out to be impossible to implement in a sound manner and hence was removed. Now, if `Clone` is called without the GIL being held, we panic instead for which calling code might not be prepared.
 
+It is advised to migrate off the `py-clone` feature. The simplest way to remove dependency on `impl<T> Clone for Py<T>` is to wrap `Py<T>` as `Arc<Py<T>>` and use cloning of the arc.
+
 Related to this, we also added a `pyo3_disable_reference_pool` conditional compilation flag which removes the infrastructure necessary to apply delayed reference count decrements implied by `impl<T> Drop for Py<T>`. They do not appear to be a soundness hazard as they should lead to memory leaks in the worst case. However, the global synchronization adds significant overhead to cross the Python-Rust boundary. Enabling this feature will remove these costs and make the `Drop` implementation abort the process if called without the GIL being held instead.
 </details>
 

guide/src/migration.md

@@ -27,7 +27,7 @@ fn main() -> PyResult<()> {
 ## Want to run just an expression? Then use `eval_bound`.
 
 [`Python::eval_bound`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.eval_bound) is
-a method to execute a [Python expression](https://docs.python.org/3.7/reference/expressions.html)
+a method to execute a [Python expression](https://docs.python.org/3/reference/expressions.html)
 and return the evaluated value as a `Bound<'py, PyAny>` object.
 
 ```rust
@@ -51,7 +51,7 @@ Python::with_gil(|py| {
 ## Want to run statements? Then use `run_bound`.
 
 [`Python::run_bound`] is a method to execute one or more
-[Python statements](https://docs.python.org/3.7/reference/simple_stmts.html).
+[Python statements](https://docs.python.org/3/reference/simple_stmts.html).
 This method returns nothing (like any Python statement), but you can get
 access to manipulated objects via the `locals` dict.
 

guide/src/python-from-rust/calling-existing-code.md

@@ -91,26 +91,18 @@ fn main() -> PyResult<()> {
 
         // call object with PyDict
         let kwargs = [(key1, val1)].into_py_dict(py);
-        fun.call_bound(py, (), Some(&kwargs))?;
+        fun.call(py, (), Some(&kwargs))?;
 
         // pass arguments as Vec
         let kwargs = vec![(key1, val1), (key2, val2)];
-        fun.call_bound(py, (), Some(&kwargs.into_py_dict(py)))?;
+        fun.call(py, (), Some(&kwargs.into_py_dict(py)))?;
 
         // pass arguments as HashMap
         let mut kwargs = HashMap::<&str, i32>::new();
         kwargs.insert(key1, 1);
-        fun.call_bound(py, (), Some(&kwargs.into_py_dict(py)))?;
+        fun.call(py, (), Some(&kwargs.into_py_dict(py)))?;
 
         Ok(())
     })
 }
-```
-
-<div class="warning">
-
-During PyO3's [migration from "GIL Refs" to the `Bound<T>` smart pointer](../migration.md#migrating-from-the-gil-refs-api-to-boundt), `Py<T>::call` is temporarily named [`Py<T>::call_bound`]({{#PYO3_DOCS_URL}}/pyo3/struct.Py.html#method.call_bound) (and `call_method` is temporarily `call_method_bound`).
-
-(This temporary naming is only the case for the `Py<T>` smart pointer. The methods on the `&PyAny` GIL Ref such as `call` have not been given replacements, and the methods on the `Bound<PyAny>` smart pointer such as [`Bound<PyAny>::call`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call) already use follow the newest API conventions.)
-
-</div>
+```

guide/src/python-from-rust/function-calls.md

@@ -0,0 +1 @@
+Make subclassing a class that doesn't allow that a compile-time error instead of runtime

newsfragments/4453.changed.md

@@ -0,0 +1,2 @@
+* The `GILProtected` struct is not available on the free-threaded build of
+  Python 3.13.

newsfragments/4504.changed.md

@@ -0,0 +1 @@
+Add Python-ref cloning `clone_ref` for `GILOnceCell<Py<T>>`

newsfragments/4511.added.md

@@ -0,0 +1 @@
+Add `Borrowed::as_ptr`.

newsfragments/4520.added.md

@@ -0,0 +1 @@
+Remove private FFI definitions `_Py_c_sum`, `_Py_c_diff`, `_Py_c_neg`, `_Py_c_prod`, `_Py_c_quot`, `_Py_c_pow`, `_Py_c_abs`.

newsfragments/4521.removed.md

@@ -0,0 +1 @@
+* Added FFI bindings for `PyImport_AddModuleRef`.

newsfragments/4529.added.md

@@ -0,0 +1,3 @@
+* Updated the FFI bindings for functions and struct fields that have been
+  deprecated or removed. You may see new deprecation warnings if you are using
+  functions or fields exposed by the C API that are deprecated.

newsfragments/4534.changed.md

@@ -0,0 +1 @@
+* Removed the bindings for the private function `_PyErr_ChainExceptions`.

newsfragments/4534.removed.md

@@ -1059,8 +1059,9 @@ impl FromStr for BuildFlag {
     }
 }
 
-/// A list of python interpreter compile-time preprocessor defines that
-/// we will pick up and pass to rustc via `--cfg=py_sys_config={varname}`;
+/// A list of python interpreter compile-time preprocessor defines.
+///
+/// PyO3 will pick these up and pass to rustc via `--cfg=py_sys_config={varname}`;
 /// this allows using them conditional cfg attributes in the .rs files, so
 ///
 /// ```rust

pyo3-build-config/src/impl_.rs

@@ -48,7 +48,8 @@ fn main() {
 
     macro_rules! check_field {
         ($struct_name:ident, $field:ident, $bindgen_field:ident) => {{
-            #[allow(clippy::used_underscore_binding)]
+            // some struct fields are deprecated but still present in the ABI
+            #[allow(clippy::used_underscore_binding, deprecated)]
             let pyo3_ffi_offset = memoffset::offset_of!(pyo3_ffi::$struct_name, $field);
             #[allow(clippy::used_underscore_binding)]
             let bindgen_offset = memoffset::offset_of!(bindings::$struct_name, $bindgen_field);

pyo3-ffi-check/src/main.rs

@@ -24,6 +24,7 @@ extern "C" {
         closure: *mut PyObject,
     ) -> *mut PyObject;
 
+    #[cfg(not(Py_3_13))]
     #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))]
     #[cfg_attr(PyPy, link_name = "PyPyEval_CallObjectWithKeywords")]
     pub fn PyEval_CallObjectWithKeywords(
@@ -33,6 +34,7 @@ extern "C" {
     ) -> *mut PyObject;
 }
 
+#[cfg(not(Py_3_13))]
 #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))]
 #[inline]
 pub unsafe fn PyEval_CallObject(func: *mut PyObject, arg: *mut PyObject) -> *mut PyObject {
@@ -41,9 +43,11 @@ pub unsafe fn PyEval_CallObject(func: *mut PyObject, arg: *mut PyObject) -> *mut
 }
 
 extern "C" {
+    #[cfg(not(Py_3_13))]
     #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))]
     #[cfg_attr(PyPy, link_name = "PyPyEval_CallFunction")]
     pub fn PyEval_CallFunction(obj: *mut PyObject, format: *const c_char, ...) -> *mut PyObject;
+    #[cfg(not(Py_3_13))]
     #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))]
     #[cfg_attr(PyPy, link_name = "PyPyEval_CallMethod")]
     pub fn PyEval_CallMethod(
@@ -95,9 +99,22 @@ extern "C" {
 }
 
 extern "C" {
+    #[cfg(not(Py_3_13))]
     #[cfg_attr(PyPy, link_name = "PyPyEval_ThreadsInitialized")]
+    #[cfg_attr(
+        Py_3_9,
+        deprecated(
+            note = "Deprecated in Python 3.9, this function always returns true in Python 3.7 or newer."
+        )
+    )]
     pub fn PyEval_ThreadsInitialized() -> c_int;
     #[cfg_attr(PyPy, link_name = "PyPyEval_InitThreads")]
+    #[cfg_attr(
+        Py_3_9,
+        deprecated(
+            note = "Deprecated in Python 3.9, this function does nothing in Python 3.7 or newer."
+        )
+    )]
     pub fn PyEval_InitThreads();
     pub fn PyEval_AcquireLock();
     pub fn PyEval_ReleaseLock();

pyo3-ffi/src/ceval.rs

@@ -37,3 +37,16 @@ compat_function!(
         item
     }
 );
+
+compat_function!(
+    originally_defined_for(Py_3_13);
+
+    #[inline]
+    pub unsafe fn PyImport_AddModuleRef(
+        name: *const std::os::raw::c_char,
+    ) -> *mut crate::PyObject {
+        use crate::{compat::Py_XNewRef, PyImport_AddModule};
+
+        Py_XNewRef(PyImport_AddModule(name))
+    }
+);