Fix bit op coercion in DUALNUM builds.

Thanks to Sergey Kaplun.

(cherry picked from commit f5fd222)

The `lj_carith_check64()` function coerces the given number value to the
32-bit wide value. In this case, the 64-bit-wide operands will lose
upper bits.

This patch removes the excess coercion for the DUALNUM mode.

Sergey Kaplun:
* added the description and the test for the problem

Part of tarantool/tarantool#10199

Reviewed-by: Sergey Bronnikov <[email protected]>
Reviewed-by: Maxim Kokryashkin <[email protected]>
Signed-off-by: Sergey Kaplun <[email protected]>
(cherry picked from commit 8cd79d1)

Author: Mike Pall

src/lj_carith.c

@@ -349,9 +349,7 @@ uint64_t lj_carith_check64(lua_State *L, int narg, CTypeID *id)
   if (LJ_LIKELY(tvisint(o))) {
     return (uint32_t)intV(o);
   } else {
-    int32_t i = lj_num2bit(numV(o));
-    if (LJ_DUALNUM) setintV(o, i);
-    return (uint32_t)i;
+    return (uint32_t)lj_num2bit(numV(o));
   }
 }
 

test/tarantool-tests/lj-1273-dualnum-bit-coercion.test.lua

@@ -0,0 +1,19 @@
+local tap = require('tap')
+
+-- Test file to demonstrate LuaJIT misbehaviour on operating
+-- for 64-bit operands in built-in `bit` library in DUALNUM mode.
+-- See also, https://github.com/LuaJIT/LuaJIT/issues/1273.
+
+local test = tap.test('lj-1273-dualnum-bit-coercion')
+test:plan(1)
+
+local bit = require('bit')
+
+-- The cdata value for 2 ^ 33.
+local EXPECTED = 2 ^ 33 + 0LL
+-- Same value is used as mask for `bit.band()`.
+local MASK = EXPECTED
+
+test:is(bit.band(2 ^ 33, MASK), EXPECTED, 'correct bit.band result')
+
+test:done(true)