Allow per-interpreter internals/local_internals

Author: b-pass

Diff for: CMakeLists.txt

@@ -92,6 +92,7 @@ option(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION
        "To enforce that a handle_type_name<> specialization exists" OFF)
 option(PYBIND11_SIMPLE_GIL_MANAGEMENT
        "Use simpler GIL management logic that does not support disassociation" OFF)
+option(PYBIND11_SUBINTERPRETER_SUPPORT "Enable support for sub-interpreters" OFF)
 option(PYBIND11_NUMPY_1_ONLY
        "Disable NumPy 2 support to avoid changes to previous pybind11 versions." OFF)
 set(PYBIND11_INTERNALS_VERSION
@@ -105,6 +106,9 @@ endif()
 if(PYBIND11_SIMPLE_GIL_MANAGEMENT)
   add_compile_definitions(PYBIND11_SIMPLE_GIL_MANAGEMENT)
 endif()
+if(PYBIND11_SUBINTERPRETER_SUPPORT)
+  add_compile_definitions(PYBIND11_SUBINTERPRETER_SUPPORT)
+endif()
 if(PYBIND11_NUMPY_1_ONLY)
   add_compile_definitions(PYBIND11_NUMPY_1_ONLY)
 endif()

Diff for: include/pybind11/detail/internals.h

@@ -260,7 +260,25 @@ struct type_info {
 /// Each module locally stores a pointer to the `internals` data. The data
 /// itself is shared among modules with the same `PYBIND11_INTERNALS_ID`.
 inline internals **&get_internals_pp() {
+#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) || PY_VERSION_HEX < 0x030C0000               \
+    || !defined(PYBIND11_SUBINTERPRETER_SUPPORT)
     static internals **internals_pp = nullptr;
+#else
+    static thread_local internals **internals_pp = nullptr;
+    // This is one per interpreter, we cache it but if the thread changed
+    // then we need to invalidate our cache
+    // the caller will find the right value and set it if its null
+    static thread_local PyThreadState *tstate_cached = nullptr;
+#    if PY_VERSION_HEX < 0x030D0000
+    PyThreadState *tstate = _PyThreadState_UncheckedGet();
+#    else
+    PyThreadState *tstate = PyThreadState_UncheckedGet();
+#    endif
+    if (tstate != tstate_cached) {
+        tstate_cached = tstate;
+        internals_pp = nullptr;
+    }
+#endif
     return internals_pp;
 }
 
@@ -427,9 +445,6 @@ PYBIND11_NOINLINE internals &get_internals() {
         return **internals_pp;
     }
 
-#if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
-    gil_scoped_acquire gil;
-#else
     // Ensure that the GIL is held since we will need to make Python calls.
     // Cannot use py::gil_scoped_acquire here since that constructor calls get_internals.
     struct gil_scoped_acquire_local {
@@ -439,7 +454,7 @@ PYBIND11_NOINLINE internals &get_internals() {
         ~gil_scoped_acquire_local() { PyGILState_Release(state); }
         const PyGILState_STATE state;
     } gil;
-#endif
+
     error_scope err_scope;
 
     dict state_dict = get_python_state_dict();
@@ -455,7 +470,12 @@ PYBIND11_NOINLINE internals &get_internals() {
         // libc++ with CPython doesn't require this (types are explicitly exported)
         // libc++ with PyPy still need it, awaiting further investigation
 #if !defined(__GLIBCXX__)
-        (*internals_pp)->registered_exception_translators.push_front(&translate_local_exception);
+        if ((*internals_pp)->registered_exception_translators.empty()
+            || (*internals_pp)->registered_exception_translators.front()
+                   != &translate_local_exception) {
+            (*internals_pp)
+                ->registered_exception_translators.push_front(&translate_local_exception);
+        }
 #endif
     } else {
         if (!internals_pp) {
@@ -515,7 +535,61 @@ inline local_internals &get_local_internals() {
     // static deinitialization fiasco. In order to avoid it we avoid destruction of the
     // local_internals static. One can read more about the problem and current solution here:
     // https://google.github.io/styleguide/cppguide.html#Static_and_Global_Variables
+
+#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) || PY_VERSION_HEX < 0x030C0000               \
+    || !defined(PYBIND11_SUBINTERPRETER_SUPPORT)
     static auto *locals = new local_internals();
+#else
+    static thread_local local_internals *locals = nullptr;
+    // This is one per interpreter, we cache it but if the interpreter changed
+    // then we need to invalidate our cache and re-fetch from the state dict
+    static thread_local PyThreadState *tstate_cached = nullptr;
+#    if PY_VERSION_HEX < 0x030D0000
+    PyThreadState *tstate = _PyThreadState_UncheckedGet();
+#    else
+    PyThreadState *tstate = PyThreadState_UncheckedGet();
+#    endif
+    if (!tstate) {
+        pybind11_fail(
+            "pybind11::detail::get_local_internals() called without a current python thread");
+    }
+    if (tstate != tstate_cached) {
+        // we create a unique value at first run which is based on a pointer to
+        // a (non-thread_local) static value in this function, then multiple
+        // loaded modules using this code will still each have a unique key.
+        static const std::string this_module_idstr
+            = PYBIND11_MODULE_LOCAL_ID
+              + std::to_string(reinterpret_cast<uintptr_t>(&this_module_idstr));
+
+        // Ensure that the GIL is held since we will need to make Python calls.
+        // Cannot use py::gil_scoped_acquire here since that constructor calls get_internals.
+        struct gil_scoped_acquire_local {
+            gil_scoped_acquire_local() : state(PyGILState_Ensure()) {}
+            gil_scoped_acquire_local(const gil_scoped_acquire_local &) = delete;
+            gil_scoped_acquire_local &operator=(const gil_scoped_acquire_local &) = delete;
+            ~gil_scoped_acquire_local() { PyGILState_Release(state); }
+            const PyGILState_STATE state;
+        } gil;
+
+        error_scope err_scope;
+        dict state_dict = get_python_state_dict();
+        object local_capsule = reinterpret_steal<object>(
+            dict_getitemstringref(state_dict.ptr(), this_module_idstr.c_str()));
+        if (!local_capsule) {
+            locals = new local_internals();
+            state_dict[this_module_idstr.c_str()] = capsule(reinterpret_cast<void *>(locals));
+        } else {
+            void *ptr = PyCapsule_GetPointer(local_capsule.ptr(), nullptr);
+            if (!ptr) {
+                raise_from(PyExc_SystemError, "pybind11::detail::get_local_internals() FAILED");
+                throw error_already_set();
+            }
+            locals = reinterpret_cast<local_internals *>(ptr);
+        }
+        tstate_cached = tstate;
+    }
+#endif
+
     return *locals;
 }
 

Diff for: tests/test_embed/external_module.cpp

@@ -6,7 +6,7 @@ namespace py = pybind11;
  * modules aren't preserved over a finalize/initialize.
  */
 
-PYBIND11_MODULE(external_module, m, py::mod_gil_not_used()) {
+PYBIND11_MODULE(external_module, m, py::mod_gil_not_used(), py::mod_multi_interpreter_one_gil()) {
     class A {
     public:
         explicit A(int value) : v{value} {};

Diff for: tests/test_embed/test_interpreter.cpp

@@ -338,11 +338,16 @@ TEST_CASE("Subinterpreter") {
     auto *main_tstate = PyThreadState_Get();
     auto *sub_tstate = Py_NewInterpreter();
 
-    // Subinterpreters get their own copy of builtins. detail::get_internals() still
-    // works by returning from the static variable, i.e. all interpreters share a single
-    // global pybind11::internals;
+    // Subinterpreters get their own copy of builtins.
     REQUIRE_FALSE(has_state_dict_internals_obj());
+
+#ifdef PYBIND11_SUBINTERPRETER_SUPPORT
+    // internals hasn't been populated yet, but will be different for the subinterpreter
+    REQUIRE_FALSE(has_pybind11_internals_static());
+#else
+    // This static is still defined
     REQUIRE(has_pybind11_internals_static());
+#endif
 
     // Modules tags should be gone.
     REQUIRE_FALSE(py::hasattr(py::module_::import("__main__"), "tag"));
@@ -354,12 +359,16 @@ TEST_CASE("Subinterpreter") {
         REQUIRE(m.attr("add")(1, 2).cast<int>() == 3);
     }
 
+    // The subinterpreter now has internals populated since we imported a pybind11 module
+    REQUIRE(has_pybind11_internals_static());
+
     // Restore main interpreter.
     Py_EndInterpreter(sub_tstate);
     PyThreadState_Swap(main_tstate);
 
     REQUIRE(py::hasattr(py::module_::import("__main__"), "main_tag"));
     REQUIRE(py::hasattr(py::module_::import("widget_module"), "extension_module_tag"));
+    REQUIRE(has_state_dict_internals_obj());
 }
 
 TEST_CASE("Execution frame") {