Skip to content

Commit e7e5d6e

Browse files
rwgkrwgk
authored
Make wrapped C++ functions pickleable (#5580)
* Backport of google/pybind11clif#30099 * Add back `PYBIND11_WARNING_DISABLE_CLANG("-Wcast-function-type-mismatch")` macos-13 • brew install llvm ``` /Users/runner/work/pybind11/pybind11/include/pybind11/detail/function_record_pyobject.h:40:26: error: cast from 'PyObject *(*)(PyObject *, PyObject *, PyObject *)' (aka '_object *(*)(_object *, _object *, _object *)') to 'PyCFunction' (aka '_object *(*)(_object *, _object *)') converts to incompatible function type [-Werror,-Wcast-function-type-mismatch] 40 | = {{"__reduce_ex__", (PyCFunction) reduce_ex_impl, METH_VARARGS | METH_KEYWORDS, nullptr}, | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``` * Remove obsolete `PY_VERSION_HEX >= 0x03080000` (discovered by gh-henryiii)
1 parent 8f00d1e commit e7e5d6e

File tree

9 files changed

+294
-88
lines changed

9 files changed

+294
-88
lines changed

‎CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ set(PYBIND11_HEADERS
134134
include/pybind11/detail/cpp_conduit.h
135135
include/pybind11/detail/descr.h
136136
include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h
137+
include/pybind11/detail/function_record_pyobject.h
137138
include/pybind11/detail/init.h
138139
include/pybind11/detail/internals.h
139140
include/pybind11/detail/native_enum_data.h

‎include/pybind11/attr.h

+1
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,7 @@ struct argument_record {
192192

193193
/// Internal data structure which holds metadata about a bound function (signature, overloads,
194194
/// etc.)
195+
#define PYBIND11_DETAIL_FUNCTION_RECORD_ABI_ID "v1" // PLEASE UPDATE if the struct is changed.
195196
struct function_record {
196197
function_record()
197198
: is_constructor(false), is_new_style_constructor(false), is_stateless(false),

‎include/pybind11/detail/function_record_pyobject.h

+208
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
// Copyright (c) 2024-2025 The Pybind Development Team.
2+
// All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
// For background see the description of PR google/pybind11clif#30099.
6+
7+
#pragma once
8+
9+
#include <pybind11/attr.h>
10+
#include <pybind11/conduit/pybind11_platform_abi_id.h>
11+
#include <pybind11/pytypes.h>
12+
13+
#include "common.h"
14+
15+
#include <cstring>
16+
17+
PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
18+
PYBIND11_NAMESPACE_BEGIN(detail)
19+
20+
struct function_record_PyObject {
21+
PyObject_HEAD
22+
function_record *cpp_func_rec;
23+
};
24+
25+
PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods)
26+
27+
PyObject *tp_new_impl(PyTypeObject *type, PyObject *args, PyObject *kwds);
28+
PyObject *tp_alloc_impl(PyTypeObject *type, Py_ssize_t nitems);
29+
int tp_init_impl(PyObject *self, PyObject *args, PyObject *kwds);
30+
void tp_dealloc_impl(PyObject *self);
31+
void tp_free_impl(void *self);
32+
33+
static PyObject *reduce_ex_impl(PyObject *self, PyObject *, PyObject *);
34+
35+
PYBIND11_WARNING_PUSH
36+
#if defined(__GNUC__) && __GNUC__ >= 8
37+
PYBIND11_WARNING_DISABLE_GCC("-Wcast-function-type")
38+
#endif
39+
#if defined(__clang__) && !defined(__apple_build_version__) && __clang_major__ >= 19
40+
PYBIND11_WARNING_DISABLE_CLANG("-Wcast-function-type-mismatch")
41+
#endif
42+
static PyMethodDef tp_methods_impl[]
43+
= {{"__reduce_ex__", (PyCFunction) reduce_ex_impl, METH_VARARGS | METH_KEYWORDS, nullptr},
44+
{nullptr, nullptr, 0, nullptr}};
45+
PYBIND11_WARNING_POP
46+
47+
// Note that this name is versioned.
48+
constexpr char tp_name_impl[]
49+
= "pybind11_detail_function_record_" PYBIND11_DETAIL_FUNCTION_RECORD_ABI_ID
50+
"_" PYBIND11_PLATFORM_ABI_ID;
51+
52+
PYBIND11_NAMESPACE_END(function_record_PyTypeObject_methods)
53+
54+
// Designated initializers are a C++20 feature:
55+
// https://en.cppreference.com/w/cpp/language/aggregate_initialization#Designated_initializers
56+
// MSVC rejects them unless /std:c++20 is used (error code C7555).
57+
PYBIND11_WARNING_PUSH
58+
PYBIND11_WARNING_DISABLE_CLANG("-Wmissing-field-initializers")
59+
#if defined(__GNUC__) && __GNUC__ >= 8
60+
PYBIND11_WARNING_DISABLE_GCC("-Wmissing-field-initializers")
61+
#endif
62+
static PyTypeObject function_record_PyTypeObject = {
63+
PyVarObject_HEAD_INIT(nullptr, 0)
64+
/* const char *tp_name */ function_record_PyTypeObject_methods::tp_name_impl,
65+
/* Py_ssize_t tp_basicsize */ sizeof(function_record_PyObject),
66+
/* Py_ssize_t tp_itemsize */ 0,
67+
/* destructor tp_dealloc */ function_record_PyTypeObject_methods::tp_dealloc_impl,
68+
/* Py_ssize_t tp_vectorcall_offset */ 0,
69+
/* getattrfunc tp_getattr */ nullptr,
70+
/* setattrfunc tp_setattr */ nullptr,
71+
/* PyAsyncMethods *tp_as_async */ nullptr,
72+
/* reprfunc tp_repr */ nullptr,
73+
/* PyNumberMethods *tp_as_number */ nullptr,
74+
/* PySequenceMethods *tp_as_sequence */ nullptr,
75+
/* PyMappingMethods *tp_as_mapping */ nullptr,
76+
/* hashfunc tp_hash */ nullptr,
77+
/* ternaryfunc tp_call */ nullptr,
78+
/* reprfunc tp_str */ nullptr,
79+
/* getattrofunc tp_getattro */ nullptr,
80+
/* setattrofunc tp_setattro */ nullptr,
81+
/* PyBufferProcs *tp_as_buffer */ nullptr,
82+
/* unsigned long tp_flags */ Py_TPFLAGS_DEFAULT,
83+
/* const char *tp_doc */ nullptr,
84+
/* traverseproc tp_traverse */ nullptr,
85+
/* inquiry tp_clear */ nullptr,
86+
/* richcmpfunc tp_richcompare */ nullptr,
87+
/* Py_ssize_t tp_weaklistoffset */ 0,
88+
/* getiterfunc tp_iter */ nullptr,
89+
/* iternextfunc tp_iternext */ nullptr,
90+
/* struct PyMethodDef *tp_methods */ function_record_PyTypeObject_methods::tp_methods_impl,
91+
/* struct PyMemberDef *tp_members */ nullptr,
92+
/* struct PyGetSetDef *tp_getset */ nullptr,
93+
/* struct _typeobject *tp_base */ nullptr,
94+
/* PyObject *tp_dict */ nullptr,
95+
/* descrgetfunc tp_descr_get */ nullptr,
96+
/* descrsetfunc tp_descr_set */ nullptr,
97+
/* Py_ssize_t tp_dictoffset */ 0,
98+
/* initproc tp_init */ function_record_PyTypeObject_methods::tp_init_impl,
99+
/* allocfunc tp_alloc */ function_record_PyTypeObject_methods::tp_alloc_impl,
100+
/* newfunc tp_new */ function_record_PyTypeObject_methods::tp_new_impl,
101+
/* freefunc tp_free */ function_record_PyTypeObject_methods::tp_free_impl,
102+
/* inquiry tp_is_gc */ nullptr,
103+
/* PyObject *tp_bases */ nullptr,
104+
/* PyObject *tp_mro */ nullptr,
105+
/* PyObject *tp_cache */ nullptr,
106+
/* PyObject *tp_subclasses */ nullptr,
107+
/* PyObject *tp_weaklist */ nullptr,
108+
/* destructor tp_del */ nullptr,
109+
/* unsigned int tp_version_tag */ 0,
110+
/* destructor tp_finalize */ nullptr,
111+
/* vectorcallfunc tp_vectorcall */ nullptr,
112+
};
113+
PYBIND11_WARNING_POP
114+
115+
static bool function_record_PyTypeObject_PyType_Ready_first_call = true;
116+
117+
inline void function_record_PyTypeObject_PyType_Ready() {
118+
if (function_record_PyTypeObject_PyType_Ready_first_call) {
119+
if (PyType_Ready(&function_record_PyTypeObject) < 0) {
120+
throw error_already_set();
121+
}
122+
function_record_PyTypeObject_PyType_Ready_first_call = false;
123+
}
124+
}
125+
126+
inline bool is_function_record_PyObject(PyObject *obj) {
127+
if (PyType_Check(obj) != 0) {
128+
return false;
129+
}
130+
PyTypeObject *obj_type = Py_TYPE(obj);
131+
// Fast path (pointer comparison).
132+
if (obj_type == &function_record_PyTypeObject) {
133+
return true;
134+
}
135+
// This works across extension modules. Note that tp_name is versioned.
136+
if (strcmp(obj_type->tp_name, function_record_PyTypeObject.tp_name) == 0) {
137+
return true;
138+
}
139+
return false;
140+
}
141+
142+
inline function_record *function_record_ptr_from_PyObject(PyObject *obj) {
143+
if (is_function_record_PyObject(obj)) {
144+
return ((detail::function_record_PyObject *) obj)->cpp_func_rec;
145+
}
146+
return nullptr;
147+
}
148+
149+
inline object function_record_PyObject_New() {
150+
auto *py_func_rec = PyObject_New(function_record_PyObject, &function_record_PyTypeObject);
151+
if (py_func_rec == nullptr) {
152+
throw error_already_set();
153+
}
154+
py_func_rec->cpp_func_rec = nullptr; // For clarity/purity. Redundant in practice.
155+
return reinterpret_steal<object>((PyObject *) py_func_rec);
156+
}
157+
158+
PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods)
159+
160+
// Guard against accidents & oversights, in particular when porting to future Python versions.
161+
inline PyObject *tp_new_impl(PyTypeObject *, PyObject *, PyObject *) {
162+
pybind11_fail("UNEXPECTED CALL OF function_record_PyTypeObject_methods::tp_new_impl");
163+
// return nullptr; // Unreachable.
164+
}
165+
166+
inline PyObject *tp_alloc_impl(PyTypeObject *, Py_ssize_t) {
167+
pybind11_fail("UNEXPECTED CALL OF function_record_PyTypeObject_methods::tp_alloc_impl");
168+
// return nullptr; // Unreachable.
169+
}
170+
171+
inline int tp_init_impl(PyObject *, PyObject *, PyObject *) {
172+
pybind11_fail("UNEXPECTED CALL OF function_record_PyTypeObject_methods::tp_init_impl");
173+
// return -1; // Unreachable.
174+
}
175+
176+
// The implementation needs the definition of `class cpp_function`.
177+
void tp_dealloc_impl(PyObject *self);
178+
179+
inline void tp_free_impl(void *) {
180+
pybind11_fail("UNEXPECTED CALL OF function_record_PyTypeObject_methods::tp_free_impl");
181+
}
182+
183+
inline PyObject *reduce_ex_impl(PyObject *self, PyObject *, PyObject *) {
184+
// Deliberately ignoring the arguments for simplicity (expected is `protocol: int`).
185+
const function_record *rec = function_record_ptr_from_PyObject(self);
186+
if (rec == nullptr) {
187+
pybind11_fail(
188+
"FATAL: function_record_PyTypeObject reduce_ex_impl(): cannot obtain cpp_func_rec.");
189+
}
190+
if (rec->name != nullptr && rec->name[0] != '\0' && rec->scope
191+
&& PyModule_Check(rec->scope.ptr()) != 0) {
192+
object scope_module = get_scope_module(rec->scope);
193+
if (scope_module) {
194+
return make_tuple(reinterpret_borrow<object>(PyEval_GetBuiltins())["eval"],
195+
make_tuple(str("__import__('importlib').import_module('")
196+
+ scope_module + str("')")))
197+
.release()
198+
.ptr();
199+
}
200+
}
201+
set_error(PyExc_RuntimeError, repr(self) + str(" is not pickleable."));
202+
return nullptr;
203+
}
204+
205+
PYBIND11_NAMESPACE_END(function_record_PyTypeObject_methods)
206+
207+
PYBIND11_NAMESPACE_END(detail)
208+
PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

‎include/pybind11/detail/internals.h

+3-27
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,11 @@
3737
/// further ABI-incompatible changes may be made before the ABI is officially
3838
/// changed to the new version.
3939
#ifndef PYBIND11_INTERNALS_VERSION
40-
# define PYBIND11_INTERNALS_VERSION 8
40+
# define PYBIND11_INTERNALS_VERSION 9
4141
#endif
4242

43-
#if PYBIND11_INTERNALS_VERSION < 8
44-
# error "PYBIND11_INTERNALS_VERSION 8 is the minimum for all platforms for pybind11v3."
43+
#if PYBIND11_INTERNALS_VERSION < 9
44+
# error "PYBIND11_INTERNALS_VERSION 9 is the minimum for all platforms for pybind11v3."
4545
#endif
4646

4747
PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
@@ -190,10 +190,6 @@ struct internals {
190190
// Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined:
191191
PyInterpreterState *istate = nullptr;
192192

193-
// Note that we have to use a std::string to allocate memory to ensure a unique address
194-
// We want unique addresses since we use pointer equality to compare function records
195-
std::string function_record_capsule_name = internals_function_record_capsule_name;
196-
197193
type_map<PyObject *> native_enum_type_map;
198194

199195
internals() = default;
@@ -612,26 +608,6 @@ const char *c_str(Args &&...args) {
612608
return strings.front().c_str();
613609
}
614610

615-
inline const char *get_function_record_capsule_name() {
616-
// On GraalPy, pointer equality of the names is currently not guaranteed
617-
#if !defined(GRAALVM_PYTHON)
618-
return get_internals().function_record_capsule_name.c_str();
619-
#else
620-
return nullptr;
621-
#endif
622-
}
623-
624-
// Determine whether or not the following capsule contains a pybind11 function record.
625-
// Note that we use `internals` to make sure that only ABI compatible records are touched.
626-
//
627-
// This check is currently used in two places:
628-
// - An important optimization in functional.h to avoid overhead in C++ -> Python -> C++
629-
// - The sibling feature of cpp_function to allow overloads
630-
inline bool is_function_record_capsule(const capsule &cap) {
631-
// Pointer equality as we rely on internals() to ensure unique pointers
632-
return cap.name() == get_function_record_capsule_name();
633-
}
634-
635611
PYBIND11_NAMESPACE_END(detail)
636612

637613
/// Returns a named pointer that is shared among all extension modules (using the same

‎include/pybind11/functional.h

+2-9
Original file line numberDiff line numberDiff line change
@@ -93,15 +93,8 @@ struct type_caster<std::function<Return(Args...)>> {
9393
auto *cfunc_self = PyCFunction_GET_SELF(cfunc.ptr());
9494
if (cfunc_self == nullptr) {
9595
PyErr_Clear();
96-
} else if (isinstance<capsule>(cfunc_self)) {
97-
auto c = reinterpret_borrow<capsule>(cfunc_self);
98-
99-
function_record *rec = nullptr;
100-
// Check that we can safely reinterpret the capsule into a function_record
101-
if (detail::is_function_record_capsule(c)) {
102-
rec = c.get_pointer<function_record>();
103-
}
104-
96+
} else {
97+
function_record *rec = function_record_ptr_from_PyObject(cfunc_self);
10598
while (rec != nullptr) {
10699
if (rec->is_stateless
107100
&& same_type(typeid(function_type),

0 commit comments

Comments
 (0)