Skip to content

Commit 7823a4a

Browse files
jservjserv
committed
Add lock-free timer
1 parent d8cde00 commit 7823a4a

File tree

7 files changed

+556
-0
lines changed

7 files changed

+556
-0
lines changed

‎README.md

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ purpose of these programs is to be illustrative and educational.
2222
- [ringbuf\_shm](ringbuf-shm/): An optimized lock-free ring buffer with shared memory.
2323
- [mbus](mbus/): A concurrent message bus.
2424
- [hashmap](hashmap/): A concurrent hashmap implementation.
25+
- [lf-timer](lf-timer/): A lock-free timer.
2526
* [Synchronization](https://en.wikipedia.org/wiki/Synchronization_(computer_science))
2627
- [hp\_list](hp_list): A concurrent linked list utilizing Hazard Pointers.
2728
- [rcu\_list](rcu_list/): A concurrent linked list utilizing the simplified RCU algorithm.

‎lf-timer/Makefile

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
CFLAGS = -std=gnu11 -Wall
2+
3+
all:
4+
gcc -o timer lf_timer.c main.c
5+
6+
clean:
7+
rm -f timer

‎lf-timer/common.h

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#pragma once
2+
3+
/* Compiler hints */
4+
#define ALWAYS_INLINE __attribute__((always_inline))
5+
#define INIT_FUNCTION __attribute__((constructor))
6+
#define UNLIKELY(x) __builtin_expect(!!(x), 0)
7+
8+
/* Hardware hints */
9+
#define PREFETCH_FOR_READ(ptr) __builtin_prefetch((ptr), 0, 3)
10+
#define PREFETCH_FOR_WRITE(ptr) __builtin_prefetch((ptr), 1, 3)
11+
12+
#define ALIGNED(x) __attribute__((__aligned__(x)))
13+
14+
#define MIN(a, b) \
15+
({ \
16+
__typeof__(a) tmp_a = (a); \
17+
__typeof__(b) tmp_b = (b); \
18+
tmp_a < tmp_b ? tmp_a : tmp_b; \
19+
})
20+
21+
#if __SIZEOF_POINTER__ == 4
22+
typedef unsigned long long ptrpair_t; /* assume 64 bits */
23+
#else /* __SIZEOF_POINTER__ == 8 */
24+
typedef __int128 ptrpair_t;
25+
#endif

‎lf-timer/lf_timer.c

+344
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,344 @@
1+
#include <inttypes.h>
2+
#include <stdbool.h>
3+
#include <stdint.h>
4+
#include <stdio.h>
5+
#include <stdlib.h>
6+
7+
#include "lf_timer.h"
8+
9+
#define CACHE_LINE 64
10+
#define MAXTIMERS 8192
11+
12+
/* Parameters for smp_fence() */
13+
enum {
14+
LoadLoad = 0x11,
15+
LoadStore = 0x12,
16+
StoreLoad = 0x21,
17+
StoreStore = 0x22,
18+
};
19+
20+
static inline void smp_fence(unsigned int mask)
21+
{
22+
if ((mask & StoreLoad) == StoreLoad) {
23+
__asm__ volatile("mfence" ::: "memory");
24+
} else if (mask != 0) { /* Any fence but StoreLoad */
25+
__asm__ volatile("" ::: "memory");
26+
}
27+
}
28+
29+
#include "common.h"
30+
#include "lockfree.h"
31+
32+
struct timer {
33+
lf_timer_cb cb; /* User-defined callback */
34+
void *arg; /* User-defined argument to callback */
35+
};
36+
37+
struct freelist {
38+
struct timer *head;
39+
uintptr_t count; /* For ABA protection */
40+
};
41+
42+
static struct {
43+
lf_tick_t earliest ALIGNED(CACHE_LINE);
44+
lf_tick_t current;
45+
uint32_t hi_watermark;
46+
47+
/* +4 for sentinels */
48+
lf_tick_t expirations[MAXTIMERS + 4] ALIGNED(CACHE_LINE);
49+
struct timer timers[MAXTIMERS] ALIGNED(CACHE_LINE);
50+
struct freelist freelist;
51+
} g_timer;
52+
53+
INIT_FUNCTION
54+
static void init_timers(void)
55+
{
56+
g_timer.earliest = LF_TIMER_TICK_INVALID;
57+
g_timer.current = 0;
58+
g_timer.hi_watermark = 0;
59+
for (uint32_t i = 0; i < MAXTIMERS; i++) {
60+
/* All timers beyond hi_watermark <= now */
61+
g_timer.expirations[i] = 0;
62+
g_timer.timers[i].cb = NULL;
63+
g_timer.timers[i].arg = &g_timer.timers[i + 1];
64+
}
65+
66+
/* Ensure sentinels trigger expiration compare and loop termination */
67+
g_timer.expirations[MAXTIMERS + 0] = 0;
68+
g_timer.expirations[MAXTIMERS + 1] = 0;
69+
g_timer.expirations[MAXTIMERS + 2] = 0;
70+
g_timer.expirations[MAXTIMERS + 3] = 0;
71+
72+
/* Last timer must end freelist */
73+
g_timer.timers[MAXTIMERS - 1].arg = NULL;
74+
75+
/* Initialize head of freelist */
76+
g_timer.freelist.head = g_timer.timers;
77+
g_timer.freelist.count = 0;
78+
}
79+
80+
/* There might be user-defined data associated with a timer
81+
* (e.g. accessed through the user-defined argument to the callback)
82+
* Set (and reset) a timer has release semantics wrt this data
83+
* Expire a timer thus needs acquire semantics
84+
*/
85+
static void expire_one_timer(lf_tick_t now, lf_tick_t *ptr)
86+
{
87+
lf_tick_t exp;
88+
do {
89+
/* Explicit reloading => smaller code */
90+
exp = __atomic_load_n(ptr, __ATOMIC_RELAXED);
91+
if (!(exp <= now)) { /* exp > now */
92+
/* If timer does not expire anymore it means some thread has
93+
* (re-)set the timer and then also updated g_timer.earliest
94+
*/
95+
return;
96+
}
97+
} while (!__atomic_compare_exchange_n(ptr, &exp, LF_TIMER_TICK_INVALID,
98+
/* weak = */ true, __ATOMIC_ACQUIRE,
99+
__ATOMIC_RELAXED));
100+
uint32_t tim = ptr - &g_timer.expirations[0];
101+
g_timer.timers[tim].cb(tim, exp, g_timer.timers[tim].arg);
102+
}
103+
104+
static lf_tick_t scan_timers(lf_tick_t now, lf_tick_t *cur, lf_tick_t *top)
105+
{
106+
lf_tick_t earliest = LF_TIMER_TICK_INVALID;
107+
lf_tick_t *ptr = cur;
108+
lf_tick_t pair0 = *ptr++;
109+
lf_tick_t pair1 = *ptr++;
110+
111+
/* Optimize: Interleave loads and compares in order to hide load-to-use
112+
* latencies. Sentinel will ensure we eventually terminate the loop
113+
*/
114+
for (;;) {
115+
lf_tick_t w0 = pair0;
116+
lf_tick_t w1 = pair1;
117+
pair0 = *ptr++;
118+
pair1 = *ptr++;
119+
if (UNLIKELY(w0 <= now)) {
120+
lf_tick_t *pw0 = (lf_tick_t *) (ptr - 4);
121+
if (pw0 >= top)
122+
break;
123+
124+
expire_one_timer(now, pw0);
125+
/* If timer did not actually expire, it was reset by some thread
126+
* and g_timer.earliest updated which means we do not have to
127+
* include it in our update of earliest.
128+
*/
129+
} else { /* 'w0' > 'now' */
130+
earliest = MIN(earliest, w0);
131+
}
132+
if (UNLIKELY(w1 <= now)) {
133+
lf_tick_t *pw1 = (lf_tick_t *) (ptr - 4) + 1;
134+
if (pw1 >= top)
135+
break;
136+
expire_one_timer(now, pw1);
137+
} else { /* 'w1' > 'now' */
138+
earliest = MIN(earliest, w1);
139+
}
140+
w0 = pair0;
141+
w1 = pair1;
142+
pair0 = *ptr++;
143+
pair1 = *ptr++;
144+
if (UNLIKELY(w0 <= now)) {
145+
lf_tick_t *pw0 = (lf_tick_t *) (ptr - 4);
146+
if (pw0 >= top)
147+
break;
148+
expire_one_timer(now, pw0);
149+
} else { /* 'w0' > 'now' */
150+
earliest = MIN(earliest, w0);
151+
}
152+
if (UNLIKELY(w1 <= now)) {
153+
lf_tick_t *pw1 = (lf_tick_t *) (ptr - 4) + 1;
154+
if (pw1 >= top)
155+
break;
156+
expire_one_timer(now, pw1);
157+
} else { /* 'w1' > 'now' */
158+
earliest = MIN(earliest, w1);
159+
}
160+
}
161+
return earliest;
162+
}
163+
164+
/* Perform an atomic-min operation on g_timer.earliest */
165+
static inline void update_earliest(lf_tick_t exp)
166+
{
167+
lf_tick_t old;
168+
do {
169+
/* Explicit reloading => smaller code */
170+
old = __atomic_load_n(&g_timer.earliest, __ATOMIC_RELAXED);
171+
if (exp >= old) {
172+
/* Our expiration time is same or later => no update */
173+
return;
174+
}
175+
/* Else our expiration time is earlier than the previous 'earliest' */
176+
} while (UNLIKELY(!__atomic_compare_exchange_n(
177+
&g_timer.earliest, &old, exp,
178+
/* weak = */ true, __ATOMIC_RELEASE, __ATOMIC_RELAXED)));
179+
}
180+
181+
void lf_timer_expire(void)
182+
{
183+
lf_tick_t now = __atomic_load_n(&g_timer.current, __ATOMIC_RELAXED);
184+
lf_tick_t earliest = __atomic_load_n(&g_timer.earliest, __ATOMIC_RELAXED);
185+
if (earliest <= now) {
186+
/* There exists at least one timer that is due for expiration */
187+
PREFETCH_FOR_READ(&g_timer.expirations[0]);
188+
PREFETCH_FOR_READ((char *) &g_timer.expirations[0] + 1 * CACHE_LINE);
189+
PREFETCH_FOR_READ((char *) &g_timer.expirations[0] + 2 * CACHE_LINE);
190+
PREFETCH_FOR_READ((char *) &g_timer.expirations[0] + 3 * CACHE_LINE);
191+
192+
/* Reset 'earliest' */
193+
__atomic_store_n(&g_timer.earliest, LF_TIMER_TICK_INVALID,
194+
__ATOMIC_RELAXED);
195+
196+
/* We need our g_timer.earliest reset to be visible before we start to
197+
* scan the timer array
198+
*/
199+
smp_fence(StoreLoad);
200+
201+
/* Scan expiration ticks looking for expired timers */
202+
earliest = scan_timers(now, &g_timer.expirations[0],
203+
&g_timer.expirations[g_timer.hi_watermark]);
204+
update_earliest(earliest);
205+
}
206+
/* Else: no timers due for expiration */
207+
}
208+
209+
void lf_timer_tick_set(lf_tick_t tck)
210+
{
211+
if (tck == LF_TIMER_TICK_INVALID) {
212+
fprintf(stderr, "invalid tick: %ld\n", tck);
213+
return;
214+
}
215+
lf_tick_t old = __atomic_load_n(&g_timer.current, __ATOMIC_RELAXED);
216+
do {
217+
if (tck <= old) /* Time cannot run backwards */
218+
return;
219+
} while (UNLIKELY(!__atomic_compare_exchange_n(
220+
&g_timer.current, &old, /* Updated on failure */
221+
tck,
222+
/* weak = */ true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)));
223+
}
224+
225+
lf_tick_t lf_timer_tick_get(void)
226+
{
227+
return __atomic_load_n(&g_timer.current, __ATOMIC_RELAXED);
228+
}
229+
230+
lf_timer_t lf_timer_alloc(lf_timer_cb cb, void *arg)
231+
{
232+
union {
233+
struct freelist fl;
234+
ptrpair_t pp;
235+
} old, neu;
236+
237+
do {
238+
old.fl.count =
239+
__atomic_load_n(&g_timer.freelist.count, __ATOMIC_ACQUIRE);
240+
/* count will be read before head, torn read will be detected by CAS */
241+
old.fl.head = __atomic_load_n(&g_timer.freelist.head, __ATOMIC_ACQUIRE);
242+
if (UNLIKELY(old.fl.head == NULL)) {
243+
return LF_TIMER_NULL;
244+
}
245+
neu.fl.head =
246+
old.fl.head->arg; /* Dereferencing old.head => need acquire */
247+
neu.fl.count = old.fl.count + 1;
248+
} while (UNLIKELY(!lockfree_compare_exchange_pp(
249+
(ptrpair_t *) &g_timer.freelist, &old.pp, neu.pp,
250+
/* weak = */ true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)));
251+
252+
uint32_t idx = old.fl.head - g_timer.timers;
253+
g_timer.expirations[idx] = LF_TIMER_TICK_INVALID;
254+
g_timer.timers[idx].cb = cb;
255+
g_timer.timers[idx].arg = arg;
256+
257+
/* Update high watermark of allocated timers */
258+
lockfree_fetch_umax_4(&g_timer.hi_watermark, idx + 1, __ATOMIC_RELEASE);
259+
return idx;
260+
}
261+
262+
void lf_timer_free(lf_timer_t idx)
263+
{
264+
if (UNLIKELY((uint32_t) idx >= g_timer.hi_watermark)) {
265+
fprintf(stderr, "invalid timer: %d\n", idx);
266+
return;
267+
}
268+
269+
if (__atomic_load_n(&g_timer.expirations[idx], __ATOMIC_ACQUIRE) !=
270+
LF_TIMER_TICK_INVALID) {
271+
fprintf(stderr, "cannot free active timer: %d\n", idx);
272+
return;
273+
}
274+
275+
struct timer *tim = &g_timer.timers[idx];
276+
union {
277+
struct freelist fl;
278+
ptrpair_t pp;
279+
} old, neu;
280+
281+
do {
282+
old.fl = g_timer.freelist;
283+
tim->cb = NULL;
284+
tim->arg = old.fl.head;
285+
neu.fl.head = tim;
286+
neu.fl.count = old.fl.count + 1;
287+
} while (UNLIKELY(!lockfree_compare_exchange_pp(
288+
(ptrpair_t *) &g_timer.freelist, &old.pp, neu.pp,
289+
/* weak = */ true, __ATOMIC_RELEASE, __ATOMIC_RELAXED)));
290+
}
291+
292+
static inline bool update_expiration(lf_timer_t idx,
293+
lf_tick_t exp,
294+
bool active,
295+
int mo)
296+
{
297+
if (UNLIKELY((uint32_t) idx >= g_timer.hi_watermark)) {
298+
fprintf(stderr, "invalid timer: %d", idx);
299+
return false;
300+
}
301+
302+
lf_tick_t old;
303+
do {
304+
/* Explicit reloading => smaller code */
305+
old = __atomic_load_n(&g_timer.expirations[idx], __ATOMIC_RELAXED);
306+
if (active ? old == LF_TIMER_TICK_INVALID : // Timer inactive/expired
307+
old != LF_TIMER_TICK_INVALID) { // Timer already active
308+
return false;
309+
}
310+
} while (UNLIKELY(
311+
!__atomic_compare_exchange_n(&g_timer.expirations[idx], &old, exp,
312+
/* weak = */ true, mo, __ATOMIC_RELAXED)));
313+
if (exp != LF_TIMER_TICK_INVALID)
314+
update_earliest(exp);
315+
return true;
316+
}
317+
318+
/* Setting a timer has release order (with regards to user-defined data
319+
* associated with the timer)
320+
*/
321+
bool lf_timer_set(lf_timer_t idx, lf_tick_t exp)
322+
{
323+
if (UNLIKELY(exp == LF_TIMER_TICK_INVALID)) {
324+
fprintf(stderr, "invalid expiration time: %ld\n", exp);
325+
return false;
326+
}
327+
328+
return update_expiration(idx, exp, false, __ATOMIC_RELEASE);
329+
}
330+
331+
bool lf_timer_reset(lf_timer_t idx, lf_tick_t exp)
332+
{
333+
if (UNLIKELY(exp == LF_TIMER_TICK_INVALID)) {
334+
fprintf(stderr, "invalid expiration time: %ld\n", exp);
335+
return false;
336+
}
337+
return update_expiration(idx, exp, true, __ATOMIC_RELEASE);
338+
}
339+
340+
bool lf_timer_cancel(lf_timer_t idx)
341+
{
342+
return update_expiration(idx, LF_TIMER_TICK_INVALID, true,
343+
__ATOMIC_RELAXED);
344+
}

0 commit comments

Comments
 (0)