|
| 1 | +#include <stddef.h> |
| 2 | + |
| 3 | +#include "mcslock.h" |
| 4 | + |
| 5 | +#define LIKELY(x) __builtin_expect(!!(x), 1) |
| 6 | + |
| 7 | +enum { MCS_PROCEED = 0, MCS_WAIT = 1 }; |
| 8 | + |
| 9 | +#if defined(__i386__) || defined(__x86_64__) |
| 10 | +#define spin_wait() __builtin_ia32_pause() |
| 11 | +#elif defined(__aarch64__) |
| 12 | +#define spin_wait() __asm__ __volatile__("isb\n") |
| 13 | +#else |
| 14 | +#define spin_wait() ((void) 0) |
| 15 | +#endif |
| 16 | + |
| 17 | +static inline void wait_until_equal_u8(uint8_t *loc, uint8_t val, int mm) |
| 18 | +{ |
| 19 | + while (__atomic_load_n(loc, mm) != val) |
| 20 | + spin_wait(); |
| 21 | +} |
| 22 | + |
| 23 | +void mcslock_init(mcslock_t *lock) |
| 24 | +{ |
| 25 | + *lock = NULL; |
| 26 | +} |
| 27 | + |
| 28 | +void mcslock_acquire(mcslock_t *lock, mcsnode_t *node) |
| 29 | +{ |
| 30 | + node->next = NULL; |
| 31 | + /* A0: Read and write lock, synchronized with A0/A1 */ |
| 32 | + mcsnode_t *prev = __atomic_exchange_n(lock, node, __ATOMIC_ACQ_REL); |
| 33 | + if (LIKELY(!prev)) /* Lock uncontended, the lock is acquired */ |
| 34 | + return; |
| 35 | + /* Otherwise, the lock is owned by another thread, waiting for its turn */ |
| 36 | + |
| 37 | + node->wait = MCS_WAIT; |
| 38 | + /* B0: Write next, synchronized with B1/B2 */ |
| 39 | + __atomic_store_n(&prev->next, node, __ATOMIC_RELEASE); |
| 40 | + |
| 41 | + /* Waiting for the previous thread to signal using the assigned node |
| 42 | + * C0: Read wait, synchronized with C1 |
| 43 | + */ |
| 44 | + wait_until_equal_u8(&node->wait, MCS_PROCEED, __ATOMIC_ACQUIRE); |
| 45 | +} |
| 46 | + |
| 47 | +void mcslock_release(mcslock_t *lock, mcsnode_t *node) |
| 48 | +{ |
| 49 | + mcsnode_t *next; |
| 50 | + |
| 51 | + /* Check if any waiting thread exists */ |
| 52 | + /* B1: Read next, synchronized with B0 */ |
| 53 | + if ((next = __atomic_load_n(&node->next, __ATOMIC_ACQUIRE)) == NULL) { |
| 54 | + /* No waiting threads detected, attempt lock release */ |
| 55 | + /* Use temporary variable as it might be overwritten */ |
| 56 | + mcsnode_t *tmp = node; |
| 57 | + |
| 58 | + /* A1: write lock, synchronize with A0 */ |
| 59 | + if (__atomic_compare_exchange_n(lock, &tmp, NULL, 0, __ATOMIC_RELEASE, |
| 60 | + __ATOMIC_RELAXED)) { |
| 61 | + /* No waiting threads yet, lock released successfully */ |
| 62 | + return; |
| 63 | + } |
| 64 | + /* Otherwise, at least one waiting thread exists */ |
| 65 | + |
| 66 | + /* Wait for the first waiting thread to link its node with ours */ |
| 67 | + /* B2: Read next, synchronized with B0 */ |
| 68 | + while ((next = __atomic_load_n(&node->next, __ATOMIC_ACQUIRE)) == NULL) |
| 69 | + spin_wait(); |
| 70 | + } |
| 71 | + |
| 72 | + /* Signal the first waiting thread */ |
| 73 | + /* C1: Write wait, synchronized with C0 */ |
| 74 | + __atomic_store_n(&next->wait, MCS_PROCEED, __ATOMIC_RELEASE); |
| 75 | +} |
0 commit comments