std: print a backtrace on stackoverflow

Since `backtrace` requires locking and memory allocation, it cannot be used from inside a signal handler. Instead, this uses `libunwind` and `dladdr`, even though both of them are not guaranteed to be async-signal-safe, strictly speaking. However, at least LLVM's libunwind (used by macOS) has a [test] for unwinding in signal handlers, and `dladdr` is used by `backtrace_symbols_fd`
in glibc, which it [documents] as async-signal-safe.

In practice, this hack works well enough on GNU/Linux and macOS (and perhaps some other platforms in the future). Realistically, the worst thing that can happen is that the stack overflow occurred inside the dynamic loaded while it holds some sort of lock, which could result in a deadlock if that happens in just the right moment. That's unlikely enough and not the *worst* thing to happen considering that a stack overflow is already an unrecoverable error and most likely
indicates a bug.

Fixes #51405

[test]: https://github.com/llvm/llvm-project/blob/a6385a3fc8a88f092d07672210a1e773481c2919/libunwind/test/signal_unwind.pass.cpp
[documents]: https://www.gnu.org/software/libc/manual/html_node/Backtraces.html#index-backtrace_005fsymbols_005ffd

Author: joboet

Diff for: library/std/src/sys/pal/unix/stack_overflow.rs

@@ -3,6 +3,9 @@
 pub use self::imp::{cleanup, init};
 use self::imp::{drop_handler, make_handler};
 
+#[cfg(any(all(target_os = "linux", target_env = "gnu"), target_os = "macos",))]
+mod backtrace;
+
 pub struct Handler {
     data: *mut libc::c_void,
 }
@@ -104,6 +107,13 @@ mod imp {
                 "\nthread '{}' has overflowed its stack\n",
                 thread::current().name().unwrap_or("<unknown>")
             );
+
+            #[cfg(any(all(target_os = "linux", target_env = "gnu"), target_os = "macos",))]
+            {
+                rtprintpanic!("backtrace:\n\n");
+                super::backtrace::print();
+            }
+
             rtabort!("stack overflow");
         } else {
             // Unregister ourselves by reverting back to the default behavior.

Diff for: library/std/src/sys/pal/unix/stack_overflow/backtrace.rs

@@ -0,0 +1,61 @@
+use crate::ffi::{CStr, c_void};
+use crate::mem::MaybeUninit;
+use crate::ptr;
+
+/// Prints the current backtrace, even in a signal handlers.
+///
+/// Since `backtrace` requires locking and memory allocation, it cannot be used
+/// from inside a signal handler. Instead, this uses `libunwind` and `dladdr`,
+/// even though both of them are not guaranteed to be async-signal-safe, strictly
+/// speaking. However, at least LLVM's libunwind (used by macOS) has a [test] for
+/// unwinding in signal handlers, and `dladdr` is used by `backtrace_symbols_fd`
+/// in glibc, which it [documents] as async-signal-safe.
+///
+/// In practice, this hack works well enough on GNU/Linux and macOS (and perhaps
+/// some other platforms). Realistically, the worst thing that can happen is that
+/// the stack overflow occurred inside the dynamic loaded while it holds some sort
+/// of lock, which could result in a deadlock if that happens in just the right
+/// moment. That's unlikely enough and not the *worst* thing to happen considering
+/// that a stack overflow is already an unrecoverable error and most likely
+/// indicates a bug.
+///
+/// [test]: https://github.com/llvm/llvm-project/blob/a6385a3fc8a88f092d07672210a1e773481c2919/libunwind/test/signal_unwind.pass.cpp
+/// [documents]: https://www.gnu.org/software/libc/manual/html_node/Backtraces.html#index-backtrace_005fsymbols_005ffd
+pub fn print() {
+    extern "C" fn frame(
+        ctx: *mut unwind::_Unwind_Context,
+        arg: *mut c_void,
+    ) -> unwind::_Unwind_Reason_Code {
+        let count = unsafe { &mut *(arg as *mut usize) };
+        let depth = *count;
+        *count += 1;
+        if depth > 128 {
+            return unwind::_URC_NO_REASON;
+        }
+
+        let ip = unsafe { unwind::_Unwind_GetIP(ctx) };
+        let mut info = MaybeUninit::uninit();
+        let r = unsafe { libc::dladdr(ip.cast(), info.as_mut_ptr()) };
+        if r != 0 {
+            let info = unsafe { info.assume_init() };
+            let name = unsafe { CStr::from_ptr(info.dli_sname) };
+            if let Ok(name) = name.to_str() {
+                rtprintpanic!("{depth}: {}\n", rustc_demangle::demangle(name));
+                return unwind::_URC_NO_REASON;
+            }
+        }
+
+        rtprintpanic!("{depth}: {ip:p}\n");
+        unwind::_URC_NO_REASON
+    }
+
+    let mut count = 0usize;
+    unsafe { unwind::_Unwind_Backtrace(frame, ptr::from_mut(&mut count).cast()) };
+    if count > 128 {
+        rtprintpanic!(
+            "[... omitted {} frame{} ...]\n",
+            count - 128,
+            if count - 128 > 1 { "s" } else { "" }
+        );
+    }
+}