The generic and C11 lock-free stack implementations differ only in memory ordering. The generic version uses a full barrier where its own comments state an acquire fence is sufficient, and seq_cst for all length counter operations.
Only x86 and ThunderX still used the generic version. On x86 the switch removes a locked add per CAS attempt in push and pop; TSO provides the acquire semantics. On ThunderX the pop fence weakens from dmb ish to dmb ishld and the push fence goes away. Measured on a 32-core x86 machine, stack_perf_autotest, cycles per operation, main versus the unified C11 version (n=9 each): Test main unified C11 delta single push/pop 46.62 +-0.30 33.41 +-0.10 -28% empty pop 1.47 +-0.01 0.98 +-0.01 -33% 1 lcore, bulk 8 9.06 +-0.05 8.20 +-0.08 -10% 1 lcore, bulk 32 6.09 +-0.02 6.15 +-0.03 +1% 2 HT, bulk 8 42.05 +-0.31 39.24 +-0.52 -7% 2 HT, bulk 32 11.92 +-0.13 11.89 +-0.10 0 2 cores, bulk 8 78.90 +-0.60 72.96 +-1.11 -7% 2 cores, bulk 32 20.74 +-1.56 7.70 +-0.13 -63% 32 cores, bulk 8 6126 +-72 6121 +-89 0 32 cores, bulk 32 1953.9 +-2.9 1984.6 +-13.3 +1.6% The C11 version is faster because it emits no lock prefixed instructions. Remove the generic version and use the C11 implementation everywhere. Signed-off-by: Stephen Hemminger <[email protected]> Acked-by: Morten Brørup <[email protected]> --- lib/stack/meson.build | 1 - lib/stack/rte_stack_lf.h | 4 - lib/stack/rte_stack_lf_generic.h | 153 ------------------------------- 3 files changed, 158 deletions(-) delete mode 100644 lib/stack/rte_stack_lf_generic.h diff --git a/lib/stack/meson.build b/lib/stack/meson.build index 18177a742f..1fab46208f 100644 --- a/lib/stack/meson.build +++ b/lib/stack/meson.build @@ -7,7 +7,6 @@ headers = files('rte_stack.h') indirect_headers += files( 'rte_stack_std.h', 'rte_stack_lf.h', - 'rte_stack_lf_generic.h', 'rte_stack_lf_c11.h', 'rte_stack_lf_stubs.h', ) diff --git a/lib/stack/rte_stack_lf.h b/lib/stack/rte_stack_lf.h index f2b012cd0e..1bc6ee8f40 100644 --- a/lib/stack/rte_stack_lf.h +++ b/lib/stack/rte_stack_lf.h @@ -8,11 +8,7 @@ #if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)) #include "rte_stack_lf_stubs.h" #else -#ifdef RTE_USE_C11_MEM_MODEL #include "rte_stack_lf_c11.h" -#else -#include "rte_stack_lf_generic.h" -#endif /** * Indicates that RTE_STACK_F_LF is supported. diff --git a/lib/stack/rte_stack_lf_generic.h b/lib/stack/rte_stack_lf_generic.h deleted file mode 100644 index cc69e4d168..0000000000 --- a/lib/stack/rte_stack_lf_generic.h +++ /dev/null @@ -1,153 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2019 Intel Corporation - */ - -#ifndef _RTE_STACK_LF_GENERIC_H_ -#define _RTE_STACK_LF_GENERIC_H_ - -#include <rte_branch_prediction.h> -#include <rte_prefetch.h> - -static __rte_always_inline unsigned int -__rte_stack_lf_count(struct rte_stack *s) -{ - /* stack_lf_push() and stack_lf_pop() do not update the list's contents - * and stack_lf->len atomically, which can cause the list to appear - * shorter than it actually is if this function is called while other - * threads are modifying the list. - * - * However, given the inherently approximate nature of the get_count - * callback -- even if the list and its size were updated atomically, - * the size could change between when get_count executes and when the - * value is returned to the caller -- this is acceptable. - * - * The stack_lf->len updates are placed such that the list may appear to - * have fewer elements than it does, but will never appear to have more - * elements. If the mempool is near-empty to the point that this is a - * concern, the user should consider increasing the mempool size. - */ - /* NOTE: review for potential ordering optimization */ - return rte_atomic_load_explicit(&s->stack_lf.used.len, rte_memory_order_seq_cst); -} - -static __rte_always_inline void -__rte_stack_lf_push_elems(struct rte_stack_lf_list *list, - struct rte_stack_lf_elem *first, - struct rte_stack_lf_elem *last, - unsigned int num) -{ - struct rte_stack_lf_head old_head; - int success; - - old_head = list->head; - - do { - struct rte_stack_lf_head new_head; - - /* An acquire fence (or stronger) is needed for weak memory - * models to establish a synchronized-with relationship between - * the list->head load and store-release operations (as part of - * the rte_atomic128_cmp_exchange()). - */ - rte_smp_mb(); - - /* Swing the top pointer to the first element in the list and - * make the last element point to the old top. - */ - new_head.top = first; - new_head.cnt = old_head.cnt + 1; - - last->next = old_head.top; - - /* old_head is updated on failure */ - success = rte_atomic128_cmp_exchange( - (rte_int128_t *)&list->head, - (rte_int128_t *)&old_head, - (rte_int128_t *)&new_head, - 1, rte_memory_order_release, - rte_memory_order_relaxed); - } while (success == 0); - /* NOTE: review for potential ordering optimization */ - rte_atomic_fetch_add_explicit(&list->len, num, rte_memory_order_seq_cst); -} - -static __rte_always_inline struct rte_stack_lf_elem * -__rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, - unsigned int num, - void **obj_table, - struct rte_stack_lf_elem **last) -{ - struct rte_stack_lf_head old_head; - int success = 0; - - /* Reserve num elements, if available */ - while (1) { - /* NOTE: review for potential ordering optimization */ - uint64_t len = rte_atomic_load_explicit(&list->len, rte_memory_order_seq_cst); - - /* Does the list contain enough elements? */ - if (unlikely(len < num)) - return NULL; - - /* NOTE: review for potential ordering optimization */ - if (rte_atomic_compare_exchange_strong_explicit(&list->len, &len, len - num, - rte_memory_order_seq_cst, rte_memory_order_seq_cst)) - break; - } - - old_head = list->head; - - /* Pop num elements */ - do { - struct rte_stack_lf_head new_head; - struct rte_stack_lf_elem *tmp; - unsigned int i; - - /* An acquire fence (or stronger) is needed for weak memory - * models to ensure the LF LIFO element reads are properly - * ordered with respect to the head pointer read. - */ - rte_smp_mb(); - - rte_prefetch0(old_head.top); - - tmp = old_head.top; - - /* Traverse the list to find the new head. A next pointer will - * either point to another element or NULL; if a thread - * encounters a pointer that has already been popped, the CAS - * will fail. - */ - for (i = 0; i < num && tmp != NULL; i++) { - rte_prefetch0(tmp->next); - if (obj_table) - obj_table[i] = tmp->data; - if (last) - *last = tmp; - tmp = tmp->next; - } - - /* If NULL was encountered, the list was modified while - * traversing it. Retry. - */ - if (i != num) { - old_head = list->head; - continue; - } - - new_head.top = tmp; - new_head.cnt = old_head.cnt + 1; - - /* old_head is updated on failure */ - success = rte_atomic128_cmp_exchange( - (rte_int128_t *)&list->head, - (rte_int128_t *)&old_head, - (rte_int128_t *)&new_head, - 1, rte_memory_order_release, - rte_memory_order_relaxed); - } while (success == 0); - - return old_head.top; -} - -#endif /* _RTE_STACK_LF_GENERIC_H_ */ -- 2.53.0
