Skip to content

Commit c225589

Browse files
obeisobeis
committed
arm32: extend atomics to i8, i16, and i64
Threads an access size through the exclusive instructions so they encode the byte and halfword forms. Sub-word rmw needs no operand extension for the plain ops (the store truncates), but min/max and cas extend their inputs before comparing. 64-bit is trickier: ldrexd/strexd need consecutive even/odd register pairs, which regalloc2 can't express, so the paired atomics pin every operand to a fixed register the way aarch64 does. rmw uses the adds/adc/subs/sbc carry chains, with min/max selecting both halves on a Z-independent condition. Signed-off-by: Obei Sideg <obei.sideg@gmail.com>
1 parent 0d5dfed commit c225589

9 files changed

Lines changed: 1260 additions & 156 deletions

File tree

‎cranelift/codegen/src/isa/arm32/inst.isle‎

Lines changed: 109 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,16 @@
133133
(Isb)
134134
(Clrex)))
135135

136+
;; Access width for exclusive / acquire-release memory operations. `DWord`
137+
;; (64-bit) uses an even/odd register pair and is only valid on the paired
138+
;; exclusive instructions.
139+
(type AtomicSize
140+
(enum
141+
(Byte)
142+
(Half)
143+
(Word)
144+
(DWord)))
145+
136146
;; The precision of a VFP floating-point operation.
137147
(type FpuSize
138148
(enum
@@ -297,28 +307,50 @@
297307
;; Load/store multiple, increment-after (`ldmia`/`stmia rn{!}, {list}`).
298308
(LdmStm (load bool) (rn Reg) (writeback bool) (reg_list u32))
299309

300-
;; Load-exclusive (`ldrex`) or load-acquire-exclusive (`ldaex`).
301-
(LoadEx (acquire bool) (rt WritableReg) (rn Reg))
302-
;; Store-exclusive (`strex`) or store-release-exclusive (`stlex`); `rd`
303-
;; receives the success flag.
304-
(StoreEx (acquire bool) (rd WritableReg) (rt Reg) (rn Reg))
305-
;; Load-acquire (`lda rt, [rn]`).
306-
(LoadAcq (rt WritableReg) (rn Reg))
307-
;; Store-release (`stl rt, [rn]`).
308-
(StoreRel (rt Reg) (rn Reg))
310+
;; Load-exclusive (`ldrex{b,h}`) or load-acquire-exclusive (`ldaex{b,h}`).
311+
(LoadEx (acquire bool) (size AtomicSize) (rt WritableReg) (rn Reg))
312+
;; Store-exclusive (`strex{b,h}`) or store-release-exclusive
313+
;; (`stlex{b,h}`); `rd` receives the success flag.
314+
(StoreEx (acquire bool) (size AtomicSize) (rd WritableReg) (rt Reg) (rn Reg))
315+
;; Load-acquire (`lda{b,h} rt, [rn]`).
316+
(LoadAcq (size AtomicSize) (rt WritableReg) (rn Reg))
317+
;; Store-release (`stl{b,h} rt, [rn]`).
318+
(StoreRel (size AtomicSize) (rt Reg) (rn Reg))
309319

310320
;; A memory barrier / clear-exclusive.
311321
(Barrier (op BarrierOp))
312322

313-
;; An atomic read-modify-write loop over a 32-bit word:
314-
;; loop: ldaex rd, [addr]; <op> tmp1, rd, operand;
315-
;; stlex tmp2, tmp1, [addr]; cmp tmp2, #0; bne loop
316-
;; `rd` receives the old value.
317-
(AtomicRmw (op AtomicRmwOp) (rd WritableReg) (addr Reg) (operand Reg)
318-
(tmp1 WritableReg) (tmp2 WritableReg))
319-
;; An atomic compare-and-swap loop over a 32-bit word; `rd` receives the
320-
;; old value.
321-
(AtomicCas (rd WritableReg) (addr Reg) (expected Reg) (new Reg) (tmp WritableReg))
323+
;; An atomic read-modify-write loop over a `size`-wide location:
324+
;; loop: ldaex{,b,h} rd, [addr]; <op> tmp1, rd, operand;
325+
;; stlex{,b,h} tmp2, tmp1, [addr]; cmp tmp2, #0; bne loop
326+
;; `rd` receives the (zero-extended) old value.
327+
(AtomicRmw (op AtomicRmwOp) (size AtomicSize) (rd WritableReg) (addr Reg)
328+
(operand Reg) (tmp1 WritableReg) (tmp2 WritableReg))
329+
;; An atomic compare-and-swap loop over a `size`-wide location; `rd`
330+
;; receives the (zero-extended) old value.
331+
(AtomicCas (size AtomicSize) (rd WritableReg) (addr Reg) (expected Reg)
332+
(new Reg) (tmp WritableReg))
333+
334+
;; A 64-bit atomic read-modify-write loop using the paired exclusive
335+
;; instructions (`ldaexd`/`stlexd`). The old value is returned in
336+
;; (rd_lo, rd_hi). All registers are pinned to a fixed even/odd layout in
337+
;; the operand collector because A32 exclusive-pair instructions require
338+
;; consecutive even/odd register pairs, which regalloc2 cannot model.
339+
(AtomicRmw64 (op AtomicRmwOp) (rd_lo WritableReg) (rd_hi WritableReg)
340+
(addr Reg) (operand_lo Reg) (operand_hi Reg)
341+
(tmp_new_lo WritableReg) (tmp_new_hi WritableReg)
342+
(tmp_res WritableReg))
343+
;; A 64-bit atomic compare-and-swap loop; the old value is in (rd_lo, rd_hi).
344+
(AtomicCas64 (rd_lo WritableReg) (rd_hi WritableReg) (addr Reg)
345+
(expected_lo Reg) (expected_hi Reg) (new_lo Reg) (new_hi Reg)
346+
(tmp_res WritableReg))
347+
;; A 64-bit sequentially-consistent atomic load (`ldaexd` + `clrex`).
348+
(AtomicLoad64 (rd_lo WritableReg) (rd_hi WritableReg) (addr Reg))
349+
;; A 64-bit sequentially-consistent atomic store via an `ldaexd`/`stlexd`
350+
;; retry loop; the loaded value is discarded into (scratch_lo, scratch_hi).
351+
(AtomicStore64 (addr Reg) (src_lo Reg) (src_hi Reg)
352+
(scratch_lo WritableReg) (scratch_hi WritableReg)
353+
(tmp_res WritableReg))
322354

323355
;; A three-register VFP op (`vadd`/`vsub`/`vmul`/`vdiv`).
324356
(FpuRRR (op FpuOp3) (size FpuSize) (rd WritableReg) (rn Reg) (rm Reg))
@@ -550,33 +582,77 @@
550582
(rule (dmb)
551583
(SideEffectNoResult.Inst (MInst.Barrier (BarrierOp.Dmb))))
552584

553-
;;;; Atomics (32-bit) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
585+
;;;; Atomics (8/16/32-bit) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
554586

555-
(decl atomic_rmw_i32 (AtomicRmwOp Reg Reg) Reg)
556-
(rule (atomic_rmw_i32 op addr operand)
587+
;; Map a scalar integer type to the exclusive-access width.
588+
(decl atomic_size (Type) AtomicSize)
589+
(rule (atomic_size $I8) (AtomicSize.Byte))
590+
(rule (atomic_size $I16) (AtomicSize.Half))
591+
(rule (atomic_size $I32) (AtomicSize.Word))
592+
593+
(decl atomic_rmw_size (Type AtomicRmwOp Reg Reg) Reg)
594+
(rule (atomic_rmw_size ty op addr operand)
557595
(let ((rd WritableReg (temp_writable_reg $I32))
558596
(tmp1 WritableReg (temp_writable_reg $I32))
559597
(tmp2 WritableReg (temp_writable_reg $I32))
560-
(_ Unit (emit (MInst.AtomicRmw op rd addr operand tmp1 tmp2))))
598+
(_ Unit (emit (MInst.AtomicRmw op (atomic_size ty) rd addr operand tmp1 tmp2))))
561599
rd))
562600

563-
(decl atomic_cas_i32 (Reg Reg Reg) Reg)
564-
(rule (atomic_cas_i32 addr expected new)
601+
(decl atomic_cas_size (Type Reg Reg Reg) Reg)
602+
(rule (atomic_cas_size ty addr expected new)
565603
(let ((rd WritableReg (temp_writable_reg $I32))
566604
(tmp WritableReg (temp_writable_reg $I32))
567-
(_ Unit (emit (MInst.AtomicCas rd addr expected new tmp))))
605+
(_ Unit (emit (MInst.AtomicCas (atomic_size ty) rd addr expected new tmp))))
568606
rd))
569607

570608
;; Sequentially-consistent load/store via load-acquire / store-release.
571-
(decl atomic_load_i32 (Reg) Reg)
572-
(rule (atomic_load_i32 addr)
609+
(decl atomic_load_size (Type Reg) Reg)
610+
(rule (atomic_load_size ty addr)
573611
(let ((rd WritableReg (temp_writable_reg $I32))
574-
(_ Unit (emit (MInst.LoadAcq rd addr))))
575-
rd))
576-
577-
(decl atomic_store_i32 (Reg Reg) InstOutput)
578-
(rule (atomic_store_i32 src addr)
579-
(side_effect (SideEffectNoResult.Inst (MInst.StoreRel src addr))))
612+
(_ Unit (emit (MInst.LoadAcq (atomic_size ty) rd addr))))
613+
rd))
614+
615+
(decl atomic_store_size (Type Reg Reg) InstOutput)
616+
(rule (atomic_store_size ty src addr)
617+
(side_effect (SideEffectNoResult.Inst (MInst.StoreRel (atomic_size ty) src addr))))
618+
619+
;;;; Atomics (64-bit) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
620+
621+
(decl atomic_rmw_i64 (AtomicRmwOp Reg ValueRegs) ValueRegs)
622+
(rule (atomic_rmw_i64 op addr operand)
623+
(let ((rd_lo WritableReg (temp_writable_reg $I32))
624+
(rd_hi WritableReg (temp_writable_reg $I32))
625+
(nl WritableReg (temp_writable_reg $I32))
626+
(nh WritableReg (temp_writable_reg $I32))
627+
(res WritableReg (temp_writable_reg $I32))
628+
(_ Unit (emit (MInst.AtomicRmw64 op rd_lo rd_hi addr
629+
(vr_lo operand) (vr_hi operand) nl nh res))))
630+
(value_regs (writable_reg_to_reg rd_lo) (writable_reg_to_reg rd_hi))))
631+
632+
(decl atomic_cas_i64 (Reg ValueRegs ValueRegs) ValueRegs)
633+
(rule (atomic_cas_i64 addr expected new)
634+
(let ((rd_lo WritableReg (temp_writable_reg $I32))
635+
(rd_hi WritableReg (temp_writable_reg $I32))
636+
(res WritableReg (temp_writable_reg $I32))
637+
(_ Unit (emit (MInst.AtomicCas64 rd_lo rd_hi addr
638+
(vr_lo expected) (vr_hi expected)
639+
(vr_lo new) (vr_hi new) res))))
640+
(value_regs (writable_reg_to_reg rd_lo) (writable_reg_to_reg rd_hi))))
641+
642+
(decl atomic_load_i64 (Reg) ValueRegs)
643+
(rule (atomic_load_i64 addr)
644+
(let ((rd_lo WritableReg (temp_writable_reg $I32))
645+
(rd_hi WritableReg (temp_writable_reg $I32))
646+
(_ Unit (emit (MInst.AtomicLoad64 rd_lo rd_hi addr))))
647+
(value_regs (writable_reg_to_reg rd_lo) (writable_reg_to_reg rd_hi))))
648+
649+
(decl atomic_store_i64 (ValueRegs Reg) InstOutput)
650+
(rule (atomic_store_i64 src addr)
651+
(let ((sl WritableReg (temp_writable_reg $I32))
652+
(sh WritableReg (temp_writable_reg $I32))
653+
(res WritableReg (temp_writable_reg $I32)))
654+
(side_effect (SideEffectNoResult.Inst
655+
(MInst.AtomicStore64 addr (vr_lo src) (vr_hi src) sl sh res)))))
580656

581657
;;;; Floating point ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
582658

‎cranelift/codegen/src/isa/arm32/inst/args.rs‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,53 @@ use crate::isa::arm32::inst::*;
44
use crate::machinst::{OperandVisitor, Reg};
55

66
pub use crate::isa::arm32::lower::isle::generated_code::{
7-
ALUOp, AMode, BarrierOp, BfxOp, BitOp, CmpOp, Cond, DspMul3Op, DspMul4Op, DspMulLOp, ExtAddOp,
8-
ExtOp, FpuOp2, FpuOp3, FpuSize, LoadKind, ParAluOp, PkhOp, QAluOp, SatOp, ShiftOp, StoreKind,
7+
ALUOp, AMode, AtomicSize, BarrierOp, BfxOp, BitOp, CmpOp, Cond, DspMul3Op, DspMul4Op,
8+
DspMulLOp, ExtAddOp, ExtOp, FpuOp2, FpuOp3, FpuSize, LoadKind, ParAluOp, PkhOp, QAluOp, SatOp,
9+
ShiftOp, StoreKind,
910
};
1011

12+
impl AtomicSize {
13+
/// The instruction-mnemonic suffix for this access width (`ldrex`,
14+
/// `ldrexb`, `ldrexh`, `ldrexd`).
15+
pub(crate) fn suffix(self) -> &'static str {
16+
match self {
17+
AtomicSize::Byte => "b",
18+
AtomicSize::Half => "h",
19+
AtomicSize::Word => "",
20+
AtomicSize::DWord => "d",
21+
}
22+
}
23+
24+
/// The size field (bits 22:21) of an A32 exclusive/acquire-release
25+
/// instruction encoding.
26+
pub(crate) fn enc_bits(self) -> u32 {
27+
let sz = match self {
28+
AtomicSize::Word => 0b00,
29+
AtomicSize::DWord => 0b01,
30+
AtomicSize::Byte => 0b10,
31+
AtomicSize::Half => 0b11,
32+
};
33+
sz << 21
34+
}
35+
36+
/// True for a sub-word (byte or halfword) access.
37+
pub(crate) fn is_subword(self) -> bool {
38+
matches!(self, AtomicSize::Byte | AtomicSize::Half)
39+
}
40+
41+
/// The sign/zero-extend op that widens a sub-word value of this size to a
42+
/// full 32-bit register.
43+
pub(crate) fn extend_op(self, signed: bool) -> ExtOp {
44+
match (self, signed) {
45+
(AtomicSize::Byte, false) => ExtOp::Uxtb,
46+
(AtomicSize::Byte, true) => ExtOp::Sxtb,
47+
(AtomicSize::Half, false) => ExtOp::Uxth,
48+
(AtomicSize::Half, true) => ExtOp::Sxth,
49+
_ => unreachable!("extend_op is only valid for sub-word sizes"),
50+
}
51+
}
52+
}
53+
1154
/// A memory address resolved to a concrete base register and either an
1255
/// immediate or register offset, ready for encoding.
1356
pub(crate) enum ResolvedAMode {

0 commit comments

Comments
 (0)