Skip to content

Commit

Permalink
Avoid AO_stack_t to cross CPU cache line boundary
Browse files Browse the repository at this point in the history
(a cherry-pick of commit ed712f7 from 'release-7_6')

Issue #45 (libatomic_ops).

Enforce proper alignment of AO_stack_t.AO_ptr to avoid the structure
value to cross the CPU cache line boundary.  A workaround for
almost-lock-free push/pop test failures on aarch64, at least.

* src/atomic_ops_stack.h [AO_USE_ALMOST_LOCK_FREE
&& !AO_STACK_ATTR_ALLIGNED] (AO_STACK_ATTR_ALLIGNED): Define.
* src/atomic_ops_stack.h [AO_USE_ALMOST_LOCK_FREE]
(AO_stack_t.AO_ptr): Add AO_STACK_ATTR_ALLIGNED attribute.
  • Loading branch information
ivmai committed Aug 12, 2022
1 parent 410ca73 commit a436af3
Showing 1 changed file with 31 additions and 1 deletion.
32 changes: 31 additions & 1 deletion src/atomic_ops_stack.h
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,36 @@
# error AO_BL_SIZE too big
#endif

#ifndef AO_STACK_ATTR_ALLIGNED
/* Enforce proper alignment of AO_stack_t.AO_ptr to avoid the */
/* structure value to cross the CPU cache line boundary. */
/* A workaround for almost-lock-free push/pop test failures */
/* on aarch64, at least. */
# if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1)
# define AO_STACK_LOG_BL_SZP1 \
(AO_BL_SIZE > 7 ? 4 : AO_BL_SIZE > 3 ? 3 : AO_BL_SIZE > 1 ? 2 : 1)
# define AO_STACK_ATTR_ALLIGNED \
__attribute__((__aligned__(sizeof(AO_t) << AO_STACK_LOG_BL_SZP1)))
# elif defined(_MSC_VER) && _MSC_VER >= 1400 /* Visual Studio 2005+ */
/* MS compiler accepts only a literal number in align, not expression. */
/* AO_STACK_ALLIGN_N is 1 << (AO_N_BITS + AO_STACK_LOG_BL_SZP1). */
# if AO_N_BITS > 2 && AO_BL_SIZE > 7
# define AO_STACK_ALLIGN_N 128
# elif (AO_N_BITS > 2 && AO_BL_SIZE > 3) || AO_BL_SIZE > 7
# define AO_STACK_ALLIGN_N 64
# elif (AO_N_BITS > 2 && AO_BL_SIZE > 1) || AO_BL_SIZE > 3
# define AO_STACK_ALLIGN_N 32
# elif AO_N_BITS > 2 || AO_BL_SIZE > 1
# define AO_STACK_ALLIGN_N 16
# else
# define AO_STACK_ALLIGN_N 8
# endif
# define AO_STACK_ATTR_ALLIGNED __declspec(align(AO_STACK_ALLIGN_N))
# else
# define AO_STACK_ATTR_ALLIGNED /* TODO: alignment is not enforced */
# endif
#endif /* !AO_STACK_ATTR_ALLIGNED */

typedef struct AO__stack_aux {
volatile AO_t AO_stack_bl[AO_BL_SIZE];
} AO_stack_aux;
Expand All @@ -114,7 +144,7 @@ AO_stack_pop_explicit_aux_acquire(volatile AO_t *list, AO_stack_aux *);
/* And now AO_stack_t for the real interface: */

typedef struct AO__stack {
volatile AO_t AO_ptr;
AO_STACK_ATTR_ALLIGNED volatile AO_t AO_ptr;
AO_stack_aux AO_aux;
} AO_stack_t;

Expand Down

0 comments on commit a436af3

Please sign in to comment.