16#include <rte_compat.h>
119#define rte_compiler_barrier() do { \
120 asm volatile ("" : : : "memory"); \
149#ifdef RTE_FORCE_INTRINSICS
153 return __sync_bool_compare_and_swap(dst, exp, src);
172static inline uint16_t
175#ifdef RTE_FORCE_INTRINSICS
176static inline uint16_t
179#if defined(__clang__)
180 return __atomic_exchange_n(dst, val, __ATOMIC_SEQ_CST);
182 return __atomic_exchange_2(dst, val, __ATOMIC_SEQ_CST);
197#define RTE_ATOMIC16_INIT(val) { (val) }
250 __sync_fetch_and_add(&v->
cnt, inc);
264 __sync_fetch_and_sub(&v->
cnt, dec);
276#ifdef RTE_FORCE_INTRINSICS
293#ifdef RTE_FORCE_INTRINSICS
317 return __sync_add_and_fetch(&v->
cnt, inc);
337 return __sync_sub_and_fetch(&v->
cnt, dec);
353#ifdef RTE_FORCE_INTRINSICS
356 return __sync_add_and_fetch(&v->
cnt, 1) == 0;
373#ifdef RTE_FORCE_INTRINSICS
376 return __sync_sub_and_fetch(&v->
cnt, 1) == 0;
393#ifdef RTE_FORCE_INTRINSICS
432#ifdef RTE_FORCE_INTRINSICS
436 return __sync_bool_compare_and_swap(dst, exp, src);
455static inline uint32_t
458#ifdef RTE_FORCE_INTRINSICS
459static inline uint32_t
462#if defined(__clang__)
463 return __atomic_exchange_n(dst, val, __ATOMIC_SEQ_CST);
465 return __atomic_exchange_4(dst, val, __ATOMIC_SEQ_CST);
480#define RTE_ATOMIC32_INIT(val) { (val) }
533 __sync_fetch_and_add(&v->
cnt, inc);
547 __sync_fetch_and_sub(&v->
cnt, dec);
559#ifdef RTE_FORCE_INTRINSICS
576#ifdef RTE_FORCE_INTRINSICS
600 return __sync_add_and_fetch(&v->
cnt, inc);
620 return __sync_sub_and_fetch(&v->
cnt, dec);
636#ifdef RTE_FORCE_INTRINSICS
639 return __sync_add_and_fetch(&v->
cnt, 1) == 0;
656#ifdef RTE_FORCE_INTRINSICS
659 return __sync_sub_and_fetch(&v->
cnt, 1) == 0;
676#ifdef RTE_FORCE_INTRINSICS
714#ifdef RTE_FORCE_INTRINSICS
718 return __sync_bool_compare_and_swap(dst, exp, src);
737static inline uint64_t
740#ifdef RTE_FORCE_INTRINSICS
741static inline uint64_t
744#if defined(__clang__)
745 return __atomic_exchange_n(dst, val, __ATOMIC_SEQ_CST);
747 return __atomic_exchange_8(dst, val, __ATOMIC_SEQ_CST);
762#define RTE_ATOMIC64_INIT(val) { (val) }
773#ifdef RTE_FORCE_INTRINSICS
783 while (success == 0) {
803#ifdef RTE_FORCE_INTRINSICS
813 while (success == 0) {
835#ifdef RTE_FORCE_INTRINSICS
845 while (success == 0) {
865#ifdef RTE_FORCE_INTRINSICS
869 __sync_fetch_and_add(&v->
cnt, inc);
884#ifdef RTE_FORCE_INTRINSICS
888 __sync_fetch_and_sub(&v->
cnt, dec);
901#ifdef RTE_FORCE_INTRINSICS
918#ifdef RTE_FORCE_INTRINSICS
942#ifdef RTE_FORCE_INTRINSICS
946 return __sync_add_and_fetch(&v->
cnt, inc);
966#ifdef RTE_FORCE_INTRINSICS
970 return __sync_sub_and_fetch(&v->
cnt, dec);
987#ifdef RTE_FORCE_INTRINSICS
1007#ifdef RTE_FORCE_INTRINSICS
1027#ifdef RTE_FORCE_INTRINSICS
1042#ifdef RTE_FORCE_INTRINSICS
1060 __extension__ __int128 int128;
1110 const rte_int128_t *src,
static int rte_atomic16_dec_and_test(rte_atomic16_t *v)
static void rte_atomic16_dec(rte_atomic16_t *v)
static int rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
static int rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
static void rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
static int rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
static void rte_atomic_thread_fence(int memorder)
static int rte_atomic64_test_and_set(rte_atomic64_t *v)
static void rte_io_rmb(void)
static void rte_rmb(void)
static void rte_atomic32_clear(rte_atomic32_t *v)
static int64_t rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
static void rte_io_mb(void)
static void rte_io_wmb(void)
static int rte_atomic32_inc_and_test(rte_atomic32_t *v)
static int rte_atomic64_dec_and_test(rte_atomic64_t *v)
static void rte_atomic64_clear(rte_atomic64_t *v)
static void rte_smp_mb(void)
static int16_t rte_atomic16_sub_return(rte_atomic16_t *v, int16_t dec)
static void rte_atomic16_set(rte_atomic16_t *v, int16_t new_value)
static void rte_atomic16_clear(rte_atomic16_t *v)
static void rte_atomic16_add(rte_atomic16_t *v, int16_t inc)
static uint32_t rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val)
static void rte_atomic32_inc(rte_atomic32_t *v)
static void rte_smp_wmb(void)
static void rte_atomic16_sub(rte_atomic16_t *v, int16_t dec)
static int rte_atomic32_test_and_set(rte_atomic32_t *v)
static void rte_atomic32_dec(rte_atomic32_t *v)
static uint16_t rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val)
static uint64_t rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val)
static void rte_atomic64_dec(rte_atomic64_t *v)
static int32_t rte_atomic32_sub_return(rte_atomic32_t *v, int32_t dec)
static void rte_atomic16_init(rte_atomic16_t *v)
static void rte_smp_rmb(void)
static void rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
static __rte_experimental int rte_atomic128_cmp_exchange(rte_int128_t *dst, rte_int128_t *exp, const rte_int128_t *src, unsigned int weak, int success, int failure)
static int rte_atomic16_test_and_set(rte_atomic16_t *v)
static int32_t rte_atomic32_read(const rte_atomic32_t *v)
static void rte_wmb(void)
static void rte_atomic32_add(rte_atomic32_t *v, int32_t inc)
static void rte_atomic32_set(rte_atomic32_t *v, int32_t new_value)
static void rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
static void rte_atomic32_sub(rte_atomic32_t *v, int32_t dec)
static void rte_atomic64_inc(rte_atomic64_t *v)
static int16_t rte_atomic16_read(const rte_atomic16_t *v)
static int64_t rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
static void rte_atomic64_init(rte_atomic64_t *v)
static int16_t rte_atomic16_add_return(rte_atomic16_t *v, int16_t inc)
static void rte_atomic32_init(rte_atomic32_t *v)
static void rte_atomic16_inc(rte_atomic16_t *v)
static int64_t rte_atomic64_read(rte_atomic64_t *v)
static int rte_atomic64_inc_and_test(rte_atomic64_t *v)
static int rte_atomic32_dec_and_test(rte_atomic32_t *v)
static int rte_atomic16_inc_and_test(rte_atomic16_t *v)
static int32_t rte_atomic32_add_return(rte_atomic32_t *v, int32_t inc)