Skip to content

Commit 2f3f1ba

Browse files
committed
fix: serialize cross-extension Go calls via a process-wide pthread mutex
Loading two gopy extensions in the same Python process embeds two independent Go runtimes. On macOS x86_64 / Go ≥1.24 this causes "fatal error: bad sweepgen in refill" (issue #370) when both runtimes run Go code concurrently. Add a process-wide pthread_mutex_t stored as a Python capsule in builtins._gopy_global_mu so every gopy extension in the same interpreter shares the same lock. The generated CGo wrappers: 1. Call gopy_ensure_mu() (lazy init, Python GIL must be held) before releasing the GIL. 2. Release the GIL via PyEval_SaveThread. 3. Acquire the mutex via gopy_lock() — blocking until any other extension's Go call finishes. 4. Release the mutex (gopy_unlock()) before restoring the GIL (PyEval_RestoreThread), avoiding the GIL/mutex deadlock. On Windows the lock/unlock are compiled as no-ops. Fixes #370 / #385.
1 parent 0299311 commit 2f3f1ba

2 files changed

Lines changed: 48 additions & 3 deletions

File tree

bind/gen.go

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,42 @@ static inline void gopy_err_handle() {
8787
PyErr_Print();
8888
}
8989
}
90+
// gopy_lock / gopy_unlock serialize all gopy extension calls to prevent
91+
// concurrent Go runtimes in the same process from corrupting each other's
92+
// GC sweep-generation counters (issue #370 / #385).
93+
// On non-Windows, the mutex lives in builtins._gopy_global_mu so every
94+
// extension loaded in the same Python process shares the same instance.
95+
// gopy_ensure_mu() must be called while the Python GIL is held (before
96+
// each PyEval_SaveThread), after which gopy_lock/unlock need no GIL.
97+
#ifndef _WIN32
98+
#include <pthread.h>
99+
#include <stdlib.h>
100+
static pthread_mutex_t *_gopy_mu = NULL;
101+
static void gopy_ensure_mu(void) {
102+
if (_gopy_mu) return;
103+
PyObject *bi = PyImport_ImportModule("builtins");
104+
if (!bi) { PyErr_Clear(); return; }
105+
PyObject *cap = PyObject_GetAttrString(bi, "_gopy_global_mu");
106+
if (cap && PyCapsule_CheckExact(cap)) {
107+
_gopy_mu = (pthread_mutex_t *)PyCapsule_GetPointer(cap, "gopy.global_mu");
108+
Py_DECREF(cap);
109+
} else {
110+
PyErr_Clear();
111+
pthread_mutex_t *mu = (pthread_mutex_t *)malloc(sizeof(pthread_mutex_t));
112+
pthread_mutex_init(mu, NULL);
113+
PyObject *nc = PyCapsule_New(mu, "gopy.global_mu", NULL);
114+
if (nc) { PyObject_SetAttrString(bi, "_gopy_global_mu", nc); Py_DECREF(nc); }
115+
_gopy_mu = mu;
116+
}
117+
Py_DECREF(bi);
118+
}
119+
static inline void gopy_lock(void) { if (_gopy_mu) pthread_mutex_lock(_gopy_mu); }
120+
static inline void gopy_unlock(void) { if (_gopy_mu) pthread_mutex_unlock(_gopy_mu); }
121+
#else
122+
static void gopy_ensure_mu(void) {}
123+
static inline void gopy_lock(void) {}
124+
static inline void gopy_unlock(void) {}
125+
#endif
90126
%[8]s
91127
*/
92128
import "C"

bind/gen_func.go

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -305,11 +305,19 @@ func (g *pyGen) genFuncBody(sym *symbol, fsym *Func) {
305305
g.gofile.Printf("C.PyGILState_Release(_gstate)\n")
306306
}
307307

308-
// release GIL
308+
// Ensure the process-wide mutex is initialized (lazy, GIL must be held).
309+
g.gofile.Printf("C.gopy_ensure_mu()\n")
310+
// release GIL then acquire the process-wide runtime mutex.
311+
// Ordering: GIL released first so that a second Python thread blocked on
312+
// the mutex does not hold the GIL while we try to reclaim it, which would
313+
// deadlock. gopy_unlock must run before PyEval_RestoreThread (LIFO defer).
309314
g.gofile.Printf("_saved_thread := C.PyEval_SaveThread()\n")
315+
g.gofile.Printf("C.gopy_lock()\n")
310316
if !rvIsErr && nres != 2 {
311-
// reacquire GIL after return
317+
// PyEval_RestoreThread deferred first → runs last (LIFO).
312318
g.gofile.Printf("defer C.PyEval_RestoreThread(_saved_thread)\n")
319+
// gopy_unlock deferred second → runs first (LIFO): release mutex before reclaiming GIL.
320+
g.gofile.Printf("defer C.gopy_unlock()\n")
313321
}
314322

315323
if isMethod {
@@ -462,7 +470,8 @@ if __err != nil {
462470

463471
if rvIsErr || nres == 2 {
464472
g.gofile.Printf("\n")
465-
// reacquire GIL
473+
// release mutex then reacquire GIL
474+
g.gofile.Printf("C.gopy_unlock()\n")
466475
g.gofile.Printf("C.PyEval_RestoreThread(_saved_thread)\n")
467476

468477
g.gofile.Printf("if __err != nil {\n")

0 commit comments

Comments
 (0)