Fix sidecar crashes in nts builds

Signed-off-by: Bob Weinand <[email protected]>

Author: bwoebi

ext/ddtrace.c

@@ -459,16 +459,31 @@ static PHP_GINIT_FUNCTION(ddtrace) {
     zai_hook_ginit();
 }
 
+// Rust code will call __cxa_thread_atexit_impl. This is a weak symbol; it's defined by glibc.
+// The problem is that calls to __cxa_thread_atexit_impl cause shared libraries to remain referenced until the calling thread terminates.
+// However in NTS builds the calling thread is the main thread and thus the shared object (i.e. ddtrace.so) WILL remain loaded.
+// This is problematic with e.g. apache which, upon reloading will unload all PHP modules including PHP itself, then reload.
+// This prevents us from a) having the weak symbols updated to the new locations and b) having ddtrace updates going live without hard restart.
+// Thus, we need to intercept it: define it ourselves so that the linker will force the rust code to call our code here.
+// Then we can collect the callbacks and invoke them ourselves right at thread shutdown, i.e. GSHUTDOWN.
 #if defined(COMPILE_DL_DDTRACE) && defined(__GLIBC__) && __GLIBC_MINOR__
 #define CXA_THREAD_ATEXIT_WRAPPER 1
 #endif
 #ifdef CXA_THREAD_ATEXIT_WRAPPER
+#define CXA_THREAD_ATEXIT_PHP ((void *)0)
+#define CXA_THREAD_ATEXIT_UNINITIALIZED ((void *)1)
+#define CXA_THREAD_ATEXIT_UNAVAILABLE ((void *)2)
+
+static int (*glibc__cxa_thread_atexit_impl)(void (*func)(void *), void *obj, void *dso_symbol) = CXA_THREAD_ATEXIT_UNINITIALIZED;
+static pthread_key_t dd_cxa_thread_atexit_key; // fallback for sidecar
+
 struct dd_rust_thread_destructor {
     void (*dtor)(void *);
     void *obj;
     struct dd_rust_thread_destructor *next;
 };
-ZEND_TLS struct dd_rust_thread_destructor *dd_rust_thread_destructors = NULL;
+// Use __thread explicitly: ZEND_TLS is empty on NTS builds.
+static __thread struct dd_rust_thread_destructor *dd_rust_thread_destructors = NULL;
 ZEND_TLS bool dd_is_main_thread = false;
 
 static void dd_run_rust_thread_destructors(void *unused) {
@@ -482,40 +497,12 @@ static void dd_run_rust_thread_destructors(void *unused) {
         free(cur);
     }
 }
-#endif
-
-static PHP_GSHUTDOWN_FUNCTION(ddtrace) {
-    if (ddtrace_globals->remote_config_reader) {
-        ddog_agent_remote_config_reader_drop(ddtrace_globals->remote_config_reader);
-    }
-    zai_hook_gshutdown();
-
-#ifdef CXA_THREAD_ATEXIT_WRAPPER
-    if (!dd_is_main_thread) {
-        dd_run_rust_thread_destructors(NULL);
-    }
-#endif
-}
-
-// Rust code will call __cxa_thread_atexit_impl. This is a weak symbol; it's defined by glibc.
-// The problem is that calls to __cxa_thread_atexit_impl cause shared libraries to remain referenced until the calling thread terminates.
-// However in NTS builds the calling thread is the main thread and thus the shared object (i.e. ddtrace.so) WILL remain loaded.
-// This is problematic with e.g. apache which, upon reloading will unload all PHP modules including PHP itself, then reload.
-// This prevents us from a) having the weak symbols updated to the new locations and b) having ddtrace updates going live without hard restart.
-// Thus, we need to intercept it: define it ourselves so that the linker will force the rust code to call our code here.
-// Then we can collect the callbacks and invoke them ourselves right at thread shutdown, i.e. GSHUTDOWN.
-#ifdef CXA_THREAD_ATEXIT_WRAPPER
-#define CXA_THREAD_ATEXIT_PHP ((void *)0)
-#define CXA_THREAD_ATEXIT_UNINITIALIZED ((void *)1)
-#define CXA_THREAD_ATEXIT_UNAVAILABLE ((void *)2)
-
-static int (*glibc__cxa_thread_atexit_impl)(void (*func)(void *), void *obj, void *dso_symbol) = CXA_THREAD_ATEXIT_UNINITIALIZED;
-static pthread_key_t dd_cxa_thread_atexit_key; // fallback for sidecar
 
 // Note: this symbol is not public
 int __cxa_thread_atexit_impl(void (*func)(void *), void *obj, void *dso_symbol) {
+    if (!f) f = open("/tmp/log.log", O_CREAT | O_WRONLY | O_APPEND);
     if (glibc__cxa_thread_atexit_impl == CXA_THREAD_ATEXIT_UNINITIALIZED) {
-        glibc__cxa_thread_atexit_impl = DL_FETCH_SYMBOL(RTLD_DEFAULT, "__cxa_thread_atexit_impl");
+        glibc__cxa_thread_atexit_impl = NULL; // DL_FETCH_SYMBOL(RTLD_DEFAULT, "__cxa_thread_atexit_impl");
         if (glibc__cxa_thread_atexit_impl == NULL) {
             // no race condition here: logging is initialized in MINIT, at which point only a single thread lives
             glibc__cxa_thread_atexit_impl = CXA_THREAD_ATEXIT_UNAVAILABLE;
@@ -532,6 +519,7 @@ int __cxa_thread_atexit_impl(void (*func)(void *), void *obj, void *dso_symbol)
     }
 
     struct dd_rust_thread_destructor *entry = malloc(sizeof(struct dd_rust_thread_destructor));
+    dprintf(f, "%d: reg %p, %p\n", gettid(), func, obj);
     entry->dtor = func;
     entry->obj = obj;
     entry->next = dd_rust_thread_destructors;
@@ -544,6 +532,19 @@ static void dd_clean_main_thread_locals() {
 }
 #endif
 
+static PHP_GSHUTDOWN_FUNCTION(ddtrace) {
+    if (ddtrace_globals->remote_config_reader) {
+        ddog_agent_remote_config_reader_drop(ddtrace_globals->remote_config_reader);
+    }
+    zai_hook_gshutdown();
+
+#ifdef CXA_THREAD_ATEXIT_WRAPPER
+    if (!dd_is_main_thread) {
+        dd_run_rust_thread_destructors(NULL);
+    }
+#endif
+}
+
 /* DDTrace\SpanLink */
 zend_class_entry *ddtrace_ce_span_link;