From b23f9434dd7163b00c655a3c8edaaae1349193a0 Mon Sep 17 00:00:00 2001
From: Mike Pall <mike>
Date: Sun, 27 Nov 2022 15:25:32 +0100
Subject: [PATCH] Avoid assertion in case of stack overflow from stitched
 trace.

Reported by Sergey Bronnikov. Fixed by Sergey Kaplun.

(cherry-picked from commit 1d75cd4d7be638babe6d4e47bf73ea05fc65d81c)

When we exit from a stitched trace due to the Lua stack overflow
error, the Lua and JIT stacks are not synchronized, and they
won't be as long as the mentioned error is raised. Because of
that, we get the incorrect bytecode instruction in
`debug_framepc`. This patch fixes this behavior, so the
`debug_framepc` now returns `NO_BCPOS` for this case.

Maxim Kokryashkin:
* added the description and the test for the problem

Part of tarantool/tarantool#9145
---
 src/lj_debug.c                                |  9 +++++---
 ...-913-stackoverflow-stitched-trace.test.lua | 23 +++++++++++++++++++
 2 files changed, 29 insertions(+), 3 deletions(-)
 create mode 100644 test/tarantool-tests/lj-913-stackoverflow-stitched-trace.test.lua

diff --git a/src/lj_debug.c b/src/lj_debug.c
index 46c442c6e7..107f464c83 100644
--- a/src/lj_debug.c
+++ b/src/lj_debug.c
@@ -101,9 +101,12 @@ static BCPos debug_framepc(lua_State *L, GCfunc *fn, cTValue *nextframe)
   pos = proto_bcpos(pt, ins) - 1;
 #if LJ_HASJIT
   if (pos > pt->sizebc) {  /* Undo the effects of lj_trace_exit for JLOOP. */
-    GCtrace *T = (GCtrace *)((char *)(ins-1) - offsetof(GCtrace, startins));
-    lj_assertL(bc_isret(bc_op(ins[-1])), "return bytecode expected");
-    pos = proto_bcpos(pt, mref(T->startpc, const BCIns));
+    if (bc_isret(bc_op(ins[-1]))) {
+      GCtrace *T = (GCtrace *)((char *)(ins-1) - offsetof(GCtrace, startins));
+      pos = proto_bcpos(pt, mref(T->startpc, const BCIns));
+    } else {
+      pos = NO_BCPOS;  /* Punt in case of stack overflow for stitched trace. */
+    }
   }
 #endif
   return pos;
diff --git a/test/tarantool-tests/lj-913-stackoverflow-stitched-trace.test.lua b/test/tarantool-tests/lj-913-stackoverflow-stitched-trace.test.lua
new file mode 100644
index 0000000000..3c12f0d914
--- /dev/null
+++ b/test/tarantool-tests/lj-913-stackoverflow-stitched-trace.test.lua
@@ -0,0 +1,23 @@
+local tap = require('tap')
+-- Test to demonstrate the incorrect LuaJIT behavior when exiting
+-- from a snapshot for stitched trace.
+local test = tap.test('lj-913-stackoverflow-stitched-trace'):skipcond({
+  ['Test requires JIT enabled'] = not jit.status(),
+})
+
+test:plan(3)
+
+-- Recursion to cause stack overflow.
+local function callee()
+  -- `math.fmod()` is NYI, so trace will be stitched here.
+  local _ = math.fmod(42, 42)
+  callee()
+end
+
+local st, err = pcall(callee)
+
+test:ok(true, 'assertion is not triggered')
+test:ok(not st, 'error happened')
+test:like(err, 'stack overflow', 'stack overflow happened')
+
+test:done(true)