Avoid assertion in case of stack overflow from stitched trace.

Reported by Sergey Bronnikov. Fixed by Sergey Kaplun.

(cherry-picked from commit 1d75cd4)

When we exit from a stitched trace due to the Lua stack overflow
error, the Lua and JIT stacks are not synchronized, and they
won't be as long as the mentioned error is raised. Because of
that, we get the incorrect bytecode instruction in
`debug_framepc`. This patch fixes this behavior, so the
`debug_framepc` now returns `NO_BCPOS` for this case.

Maxim Kokryashkin:
* added the description and the test for the problem

Part of tarantool/tarantool#9145

src/lj_debug.c

@@ -101,9 +101,12 @@ static BCPos debug_framepc(lua_State *L, GCfunc *fn, cTValue *nextframe)
   pos = proto_bcpos(pt, ins) - 1;
 #if LJ_HASJIT
   if (pos > pt->sizebc) {  /* Undo the effects of lj_trace_exit for JLOOP. */
-    GCtrace *T = (GCtrace *)((char *)(ins-1) - offsetof(GCtrace, startins));
-    lj_assertL(bc_isret(bc_op(ins[-1])), "return bytecode expected");
-    pos = proto_bcpos(pt, mref(T->startpc, const BCIns));
+    if (bc_isret(bc_op(ins[-1]))) {
+      GCtrace *T = (GCtrace *)((char *)(ins-1) - offsetof(GCtrace, startins));
+      pos = proto_bcpos(pt, mref(T->startpc, const BCIns));
+    } else {
+      pos = NO_BCPOS;  /* Punt in case of stack overflow for stitched trace. */
+    }
   }
 #endif
   return pos;

test/tarantool-tests/lj-913-stackoverflow-stitched-trace.test.lua

@@ -0,0 +1,23 @@
+local tap = require('tap')
+-- Test to demonstrate the incorrect LuaJIT behavior when exiting
+-- from a snapshot for stitched trace.
+local test = tap.test('lj-913-stackoverflow-stitched-trace'):skipcond({
+  ['Test requires JIT enabled'] = not jit.status(),
+})
+
+test:plan(3)
+
+-- Recursion to cause stack overflow.
+local function callee()
+  -- `math.fmod()` is NYI, so trace will be stitched here.
+  local _ = math.fmod(42, 42)
+  callee()
+end
+
+local st, err = pcall(callee)
+
+test:ok(true, 'assertion is not triggered')
+test:ok(not st, 'error happened')
+test:like(err, 'stack overflow', 'stack overflow happened')
+
+test:done(true)