-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathluajit-ARM64-Fix-exit-stub-patching.patch
238 lines (224 loc) · 8.55 KB
/
luajit-ARM64-Fix-exit-stub-patching.patch
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
From 9da06535092d6d9dec442641a26c64bce5574322 Mon Sep 17 00:00:00 2001
From: Mike Pall <mike>
Date: Sun, 24 Jun 2018 14:08:59 +0200
Subject: [PATCH 47/72] ARM64: Fix exit stub patching.
Contributed by Javier Guerra Giraldez.
---
src/lj_asm_arm64.h | 64 +++++++++++++++++++++++++------------------
src/lj_emit_arm64.h | 18 ++++++------
src/lj_target_arm64.h | 7 +++--
3 files changed, 51 insertions(+), 38 deletions(-)
diff --git a/src/lj_asm_arm64.h b/src/lj_asm_arm64.h
index cbb186d..baafa21 100644
--- a/src/lj_asm_arm64.h
+++ b/src/lj_asm_arm64.h
@@ -56,11 +56,11 @@ static void asm_exitstub_setup(ASMState *as, ExitNo nexits)
asm_mclimit(as);
/* 1: str lr,[sp]; bl ->vm_exit_handler; movz w0,traceno; bl <1; bl <1; ... */
for (i = nexits-1; (int32_t)i >= 0; i--)
- *--mxp = A64I_LE(A64I_BL|((-3-i)&0x03ffffffu));
- *--mxp = A64I_LE(A64I_MOVZw|A64F_U16(as->T->traceno));
+ *--mxp = A64I_LE(A64I_BL | A64F_S26(-3-i));
+ *--mxp = A64I_LE(A64I_MOVZw | A64F_U16(as->T->traceno));
mxp--;
- *mxp = A64I_LE(A64I_BL|(((MCode *)(void *)lj_vm_exit_handler-mxp)&0x03ffffffu));
- *--mxp = A64I_LE(A64I_STRx|A64F_D(RID_LR)|A64F_N(RID_SP));
+ *mxp = A64I_LE(A64I_BL | A64F_S26(((MCode *)(void *)lj_vm_exit_handler-mxp)));
+ *--mxp = A64I_LE(A64I_STRx | A64F_D(RID_LR) | A64F_N(RID_SP));
as->mctop = mxp;
}
@@ -77,7 +77,7 @@ static void asm_guardcc(ASMState *as, A64CC cc)
MCode *p = as->mcp;
if (LJ_UNLIKELY(p == as->invmcp)) {
as->loopinv = 1;
- *p = A64I_B | ((target-p) & 0x03ffffffu);
+ *p = A64I_B | A64F_S26(target-p);
emit_cond_branch(as, cc^1, p-1);
return;
}
@@ -91,7 +91,7 @@ static void asm_guardtnb(ASMState *as, A64Ins ai, Reg r, uint32_t bit)
MCode *p = as->mcp;
if (LJ_UNLIKELY(p == as->invmcp)) {
as->loopinv = 1;
- *p = A64I_B | ((target-p) & 0x03ffffffu);
+ *p = A64I_B | A64F_S26(target-p);
emit_tnb(as, ai^0x01000000u, r, bit, p-1);
return;
}
@@ -105,7 +105,7 @@ static void asm_guardcnb(ASMState *as, A64Ins ai, Reg r)
MCode *p = as->mcp;
if (LJ_UNLIKELY(p == as->invmcp)) {
as->loopinv = 1;
- *p = A64I_B | ((target-p) & 0x03ffffffu);
+ *p = A64I_B | A64F_S26(target-p);
emit_cnb(as, ai^0x01000000u, r, p-1);
return;
}
@@ -1850,7 +1850,7 @@ static void asm_loop_fixup(ASMState *as)
p[-2] |= ((uint32_t)delta & mask) << 5;
} else {
ptrdiff_t delta = target - (p - 1);
- p[-1] = A64I_B | ((uint32_t)(delta) & 0x03ffffffu);
+ p[-1] = A64I_B | A64F_S26(delta);
}
}
@@ -1919,7 +1919,7 @@ static void asm_tail_fixup(ASMState *as, TraceNo lnk)
}
/* Patch exit branch. */
target = lnk ? traceref(as->J, lnk)->mcode : (MCode *)lj_vm_exit_interp;
- p[-1] = A64I_B | (((target-p)+1)&0x03ffffffu);
+ p[-1] = A64I_B | A64F_S26((target-p)+1);
}
/* Prepare tail of code. */
@@ -1982,40 +1982,50 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
{
MCode *p = T->mcode;
MCode *pe = (MCode *)((char *)p + T->szmcode);
- MCode *cstart = NULL, *cend = p;
+ MCode *cstart = NULL;
MCode *mcarea = lj_mcode_patch(J, p, 0);
MCode *px = exitstub_trace_addr(T, exitno);
+ /* Note: this assumes a trace exit is only ever patched once. */
for (; p < pe; p++) {
/* Look for exitstub branch, replace with branch to target. */
+ ptrdiff_t delta = target - p;
MCode ins = A64I_LE(*p);
if ((ins & 0xff000000u) == 0x54000000u &&
((ins ^ ((px-p)<<5)) & 0x00ffffe0u) == 0) {
- /* Patch bcc exitstub. */
- *p = A64I_LE((ins & 0xff00001fu) | (((target-p)<<5) & 0x00ffffe0u));
- cend = p+1;
- if (!cstart) cstart = p;
+ /* Patch bcc, if within range. */
+ if (A64F_S_OK(delta, 19)) {
+ *p = A64I_LE((ins & 0xff00001fu) | A64F_S19(delta));
+ if (!cstart) cstart = p;
+ }
} else if ((ins & 0xfc000000u) == 0x14000000u &&
((ins ^ (px-p)) & 0x03ffffffu) == 0) {
- /* Patch b exitstub. */
- *p = A64I_LE((ins & 0xfc000000u) | ((target-p) & 0x03ffffffu));
- cend = p+1;
+ /* Patch b. */
+ lua_assert(A64F_S_OK(delta, 26));
+ *p = A64I_LE((ins & 0xfc000000u) | A64F_S26(delta));
if (!cstart) cstart = p;
} else if ((ins & 0x7e000000u) == 0x34000000u &&
((ins ^ ((px-p)<<5)) & 0x00ffffe0u) == 0) {
- /* Patch cbz/cbnz exitstub. */
- *p = A64I_LE((ins & 0xff00001f) | (((target-p)<<5) & 0x00ffffe0u));
- cend = p+1;
- if (!cstart) cstart = p;
+ /* Patch cbz/cbnz, if within range. */
+ if (A64F_S_OK(delta, 19)) {
+ *p = A64I_LE((ins & 0xff00001fu) | A64F_S19(delta));
+ if (!cstart) cstart = p;
+ }
} else if ((ins & 0x7e000000u) == 0x36000000u &&
((ins ^ ((px-p)<<5)) & 0x0007ffe0u) == 0) {
- /* Patch tbz/tbnz exitstub. */
- *p = A64I_LE((ins & 0xfff8001fu) | (((target-p)<<5) & 0x0007ffe0u));
- cend = p+1;
- if (!cstart) cstart = p;
+ /* Patch tbz/tbnz, if within range. */
+ if (A64F_S_OK(delta, 14)) {
+ *p = A64I_LE((ins & 0xfff8001fu) | A64F_S14(delta));
+ if (!cstart) cstart = p;
+ }
}
}
- lua_assert(cstart != NULL);
- lj_mcode_sync(cstart, cend);
+ { /* Always patch long-range branch in exit stub itself. */
+ ptrdiff_t delta = target - px;
+ lua_assert(A64F_S_OK(delta, 26));
+ *px = A64I_B | A64F_S26(delta);
+ if (!cstart) cstart = px;
+ }
+ lj_mcode_sync(cstart, px+1);
lj_mcode_patch(J, mcarea, 1);
}
diff --git a/src/lj_emit_arm64.h b/src/lj_emit_arm64.h
index 6da4c7d..1001b1d 100644
--- a/src/lj_emit_arm64.h
+++ b/src/lj_emit_arm64.h
@@ -241,7 +241,7 @@ static void emit_loadk(ASMState *as, Reg rd, uint64_t u64, int is64)
#define mcpofs(as, k) \
((intptr_t)((uintptr_t)(k) - (uintptr_t)(as->mcp - 1)))
#define checkmcpofs(as, k) \
- ((((mcpofs(as, k)>>2) + 0x00040000) >> 19) == 0)
+ (A64F_S_OK(mcpofs(as, k)>>2, 19))
static Reg ra_allock(ASMState *as, intptr_t k, RegSet allow);
@@ -312,7 +312,7 @@ static void emit_cond_branch(ASMState *as, A64CC cond, MCode *target)
{
MCode *p = --as->mcp;
ptrdiff_t delta = target - p;
- lua_assert(((delta + 0x40000) >> 19) == 0);
+ lua_assert(A64F_S_OK(delta, 19));
*p = A64I_BCC | A64F_S19(delta) | cond;
}
@@ -320,24 +320,24 @@ static void emit_branch(ASMState *as, A64Ins ai, MCode *target)
{
MCode *p = --as->mcp;
ptrdiff_t delta = target - p;
- lua_assert(((delta + 0x02000000) >> 26) == 0);
- *p = ai | ((uint32_t)delta & 0x03ffffffu);
+ lua_assert(A64F_S_OK(delta, 26));
+ *p = ai | A64F_S26(delta);
}
static void emit_tnb(ASMState *as, A64Ins ai, Reg r, uint32_t bit, MCode *target)
{
MCode *p = --as->mcp;
ptrdiff_t delta = target - p;
- lua_assert(bit < 63 && ((delta + 0x2000) >> 14) == 0);
+ lua_assert(bit < 63 && A64F_S_OK(delta, 14));
if (bit > 31) ai |= A64I_X;
- *p = ai | A64F_BIT(bit & 31) | A64F_S14((uint32_t)delta & 0x3fffu) | r;
+ *p = ai | A64F_BIT(bit & 31) | A64F_S14(delta) | r;
}
static void emit_cnb(ASMState *as, A64Ins ai, Reg r, MCode *target)
{
MCode *p = --as->mcp;
ptrdiff_t delta = target - p;
- lua_assert(((delta + 0x40000) >> 19) == 0);
+ lua_assert(A64F_S_OK(delta, 19));
*p = ai | A64F_S19(delta) | r;
}
@@ -347,8 +347,8 @@ static void emit_call(ASMState *as, void *target)
{
MCode *p = --as->mcp;
ptrdiff_t delta = (char *)target - (char *)p;
- if ((((delta>>2) + 0x02000000) >> 26) == 0) {
- *p = A64I_BL | ((uint32_t)(delta>>2) & 0x03ffffffu);
+ if (A64F_S_OK(delta>>2, 26)) {
+ *p = A64I_BL | A64F_S26(delta>>2);
} else { /* Target out of range: need indirect call. But don't use R0-R7. */
Reg r = ra_allock(as, i64ptr(target),
RSET_RANGE(RID_X8, RID_MAX_GPR)-RSET_FIXED);
diff --git a/src/lj_target_arm64.h b/src/lj_target_arm64.h
index 520023a..a207a2b 100644
--- a/src/lj_target_arm64.h
+++ b/src/lj_target_arm64.h
@@ -132,9 +132,9 @@ static LJ_AINLINE uint32_t *exitstub_trace_addr_(uint32_t *p, uint32_t exitno)
#define A64F_IMMR(x) ((x) << 16)
#define A64F_U16(x) ((x) << 5)
#define A64F_U12(x) ((x) << 10)
-#define A64F_S26(x) (x)
+#define A64F_S26(x) (((uint32_t)(x) & 0x03ffffffu))
#define A64F_S19(x) (((uint32_t)(x) & 0x7ffffu) << 5)
-#define A64F_S14(x) ((x) << 5)
+#define A64F_S14(x) (((uint32_t)(x) & 0x3fffu) << 5)
#define A64F_S9(x) ((x) << 12)
#define A64F_BIT(x) ((x) << 19)
#define A64F_SH(sh, x) (((sh) << 22) | ((x) << 10))
@@ -145,6 +145,9 @@ static LJ_AINLINE uint32_t *exitstub_trace_addr_(uint32_t *p, uint32_t exitno)
#define A64F_LSL16(x) (((x) / 16) << 21)
#define A64F_BSH(sh) ((sh) << 10)
+/* Check for valid field range. */
+#define A64F_S_OK(x, b) ((((x) + (1 << (b-1))) >> (b)) == 0)
+
typedef enum A64Ins {
A64I_S = 0x20000000,
A64I_X = 0x80000000,
--
2.20.1