Refactor hash key comparison functions to reduce code duplication and improve portability:
- Keep optimized 16-byte and 32-byte comparisons for x86 (SSE) and ARM64 (NEON), but remove the larger size-specific functions (48, 64, 80, 96, 112, 128 bytes) from the architecture headers - Add generic implementation using XOR for platforms without SIMD - Move larger key comparison functions into rte_cuckoo_hash.c where they build upon the 16-byte primitives - Enable optimized key comparisons on all architectures, not just x86 and ARM64 The rte_hash_k32_cmp_eq() function remains exposed because it is used internally by the IP fragmentation library. Signed-off-by: Stephen Hemminger <[email protected]> --- lib/hash/rte_cmp_arm64.h | 62 +++------------------------ lib/hash/rte_cmp_generic.h | 35 ++++++++++++++++ lib/hash/rte_cmp_x86.h | 62 +++------------------------ lib/hash/rte_cuckoo_hash.c | 86 +++++++++++++++++++++++++++++++++----- 4 files changed, 120 insertions(+), 125 deletions(-) create mode 100644 lib/hash/rte_cmp_generic.h diff --git a/lib/hash/rte_cmp_arm64.h b/lib/hash/rte_cmp_arm64.h index a3e85635eb..f209aaf474 100644 --- a/lib/hash/rte_cmp_arm64.h +++ b/lib/hash/rte_cmp_arm64.h @@ -2,7 +2,7 @@ * Copyright(c) 2015 Cavium, Inc */ -/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */ +/* Functions to compare multiple of 16 byte keys */ static inline int rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused) @@ -25,61 +25,9 @@ rte_hash_k16_cmp_eq(const void *key1, const void *key2, } static inline int -rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len) +rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused) { - return rte_hash_k16_cmp_eq(key1, key2, key_len) || - rte_hash_k16_cmp_eq((const char *) key1 + 16, - (const char *) key2 + 16, key_len); -} - -static inline int -rte_hash_k48_cmp_eq(const void *key1, const void *key2, size_t key_len) -{ - return rte_hash_k16_cmp_eq(key1, key2, key_len) || - rte_hash_k16_cmp_eq((const char *) key1 + 16, - (const char *) key2 + 16, key_len) || - rte_hash_k16_cmp_eq((const char *) key1 + 32, - (const char *) key2 + 32, key_len); -} - -static inline int -rte_hash_k64_cmp_eq(const void *key1, const void *key2, size_t key_len) -{ - return rte_hash_k32_cmp_eq(key1, key2, key_len) || - rte_hash_k32_cmp_eq((const char *) key1 + 32, - (const char *) key2 + 32, key_len); -} - -static inline int -rte_hash_k80_cmp_eq(const void *key1, const void *key2, size_t key_len) -{ - return rte_hash_k64_cmp_eq(key1, key2, key_len) || - rte_hash_k16_cmp_eq((const char *) key1 + 64, - (const char *) key2 + 64, key_len); -} - -static inline int -rte_hash_k96_cmp_eq(const void *key1, const void *key2, size_t key_len) -{ - return rte_hash_k64_cmp_eq(key1, key2, key_len) || - rte_hash_k32_cmp_eq((const char *) key1 + 64, - (const char *) key2 + 64, key_len); -} - -static inline int -rte_hash_k112_cmp_eq(const void *key1, const void *key2, size_t key_len) -{ - return rte_hash_k64_cmp_eq(key1, key2, key_len) || - rte_hash_k32_cmp_eq((const char *) key1 + 64, - (const char *) key2 + 64, key_len) || - rte_hash_k16_cmp_eq((const char *) key1 + 96, - (const char *) key2 + 96, key_len); -} - -static inline int -rte_hash_k128_cmp_eq(const void *key1, const void *key2, size_t key_len) -{ - return rte_hash_k64_cmp_eq(key1, key2, key_len) || - rte_hash_k64_cmp_eq((const char *) key1 + 64, - (const char *) key2 + 64, key_len); + return rte_hash_k16_cmp_eq(key1, key2, 16) | + rte_hash_k16_cmp_eq((const uint8_t *)key1 + 16, + (const uint8_t *)key2 + 16, 16); } diff --git a/lib/hash/rte_cmp_generic.h b/lib/hash/rte_cmp_generic.h new file mode 100644 index 0000000000..771180e97a --- /dev/null +++ b/lib/hash/rte_cmp_generic.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2026 Stephen Hemminger + */ + +#ifndef _RTE_CMP_GENERIC_H_ +#define _RTE_CMP_GENERIC_H_ + +/* Function to compare 16 byte keys */ +static inline int +rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused) +{ +#ifdef RTE_ARCH_64 + const unaligned_uint64_t *k1 = key1; + const unaligned_uint64_t *k2 = key2; + + return !!((k1[0] ^ k2[0]) | (k1[1] ^ k2[1])); +#else + const unaligned_uint32_t *k1 = key1; + const unaligned_uint32_t *k2 = key2; + + return !!((k1[0] ^ k2[0]) | (k1[1] ^ k2[1]) | + (k1[2] ^ k2[2]) | (k1[3] ^ k2[3])); +#endif +} + +/* Function to compare 32 byte keys */ +static inline int +rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len) +{ + return rte_hash_k16_cmp_eq(key1, key2, key_len) | + rte_hash_k16_cmp_eq((const uint8_t *) key1 + 16, + (const uint8_t *) key2 + 16, key_len); +} + +#endif /* _RTE_CMP_GENERIC_H_ */ diff --git a/lib/hash/rte_cmp_x86.h b/lib/hash/rte_cmp_x86.h index ddfbef462f..b450150d6d 100644 --- a/lib/hash/rte_cmp_x86.h +++ b/lib/hash/rte_cmp_x86.h @@ -4,7 +4,7 @@ #include <rte_vect.h> -/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */ +/* Function to compare multiple of 16 byte keys */ static inline int rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused) { @@ -16,61 +16,9 @@ rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unu } static inline int -rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len) +rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused) { - return rte_hash_k16_cmp_eq(key1, key2, key_len) || - rte_hash_k16_cmp_eq((const char *) key1 + 16, - (const char *) key2 + 16, key_len); -} - -static inline int -rte_hash_k48_cmp_eq(const void *key1, const void *key2, size_t key_len) -{ - return rte_hash_k16_cmp_eq(key1, key2, key_len) || - rte_hash_k16_cmp_eq((const char *) key1 + 16, - (const char *) key2 + 16, key_len) || - rte_hash_k16_cmp_eq((const char *) key1 + 32, - (const char *) key2 + 32, key_len); -} - -static inline int -rte_hash_k64_cmp_eq(const void *key1, const void *key2, size_t key_len) -{ - return rte_hash_k32_cmp_eq(key1, key2, key_len) || - rte_hash_k32_cmp_eq((const char *) key1 + 32, - (const char *) key2 + 32, key_len); -} - -static inline int -rte_hash_k80_cmp_eq(const void *key1, const void *key2, size_t key_len) -{ - return rte_hash_k64_cmp_eq(key1, key2, key_len) || - rte_hash_k16_cmp_eq((const char *) key1 + 64, - (const char *) key2 + 64, key_len); -} - -static inline int -rte_hash_k96_cmp_eq(const void *key1, const void *key2, size_t key_len) -{ - return rte_hash_k64_cmp_eq(key1, key2, key_len) || - rte_hash_k32_cmp_eq((const char *) key1 + 64, - (const char *) key2 + 64, key_len); -} - -static inline int -rte_hash_k112_cmp_eq(const void *key1, const void *key2, size_t key_len) -{ - return rte_hash_k64_cmp_eq(key1, key2, key_len) || - rte_hash_k32_cmp_eq((const char *) key1 + 64, - (const char *) key2 + 64, key_len) || - rte_hash_k16_cmp_eq((const char *) key1 + 96, - (const char *) key2 + 96, key_len); -} - -static inline int -rte_hash_k128_cmp_eq(const void *key1, const void *key2, size_t key_len) -{ - return rte_hash_k64_cmp_eq(key1, key2, key_len) || - rte_hash_k64_cmp_eq((const char *) key1 + 64, - (const char *) key2 + 64, key_len); + return rte_hash_k16_cmp_eq(key1, key2, 16) | + rte_hash_k16_cmp_eq((const uint8_t *) key1 + 16, + (const uint8_t *) key2 + 16, 16); } diff --git a/lib/hash/rte_cuckoo_hash.c b/lib/hash/rte_cuckoo_hash.c index 619fe0c691..ce66308c5e 100644 --- a/lib/hash/rte_cuckoo_hash.c +++ b/lib/hash/rte_cuckoo_hash.c @@ -42,13 +42,6 @@ RTE_LOG_REGISTER_DEFAULT(hash_logtype, INFO); #define RETURN_IF_TRUE(cond, retval) #endif -#if defined(RTE_ARCH_X86) -#include "rte_cmp_x86.h" -#endif - -#if defined(RTE_ARCH_ARM64) -#include "rte_cmp_arm64.h" -#endif /* * All different options to select a key compare function, @@ -57,7 +50,6 @@ RTE_LOG_REGISTER_DEFAULT(hash_logtype, INFO); */ enum cmp_jump_table_case { KEY_CUSTOM = 0, -#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) KEY_16_BYTES, KEY_32_BYTES, KEY_48_BYTES, @@ -66,11 +58,85 @@ enum cmp_jump_table_case { KEY_96_BYTES, KEY_112_BYTES, KEY_128_BYTES, -#endif KEY_OTHER_BYTES, NUM_KEY_CMP_CASES, }; +/* + * Comparison functions for different key sizes. + * Each function is only called with a specific fixed key size. + * + * Return value is 0 on equality to allow direct use of memcmp. + * Recommend using XOR and | operator to avoid branching + * as long as key is smaller than cache line size. + * + * Key1 always points to key[] in rte_hash_key which is aligned. + * Key2 is parameter to insert which might not be. + * + * Special cases for 16 and 32 bytes to allow for architecture + * specific optimizations. + */ + +#if defined(RTE_ARCH_X86) +#include "rte_cmp_x86.h" +#elif defined(RTE_ARCH_ARM64) +#include "rte_cmp_arm64.h" +#else +#include "rte_cmp_generic.h" +#endif + +static int +rte_hash_k48_cmp_eq(const void *key1, const void *key2, size_t key_len) +{ + return rte_hash_k16_cmp_eq(key1, key2, key_len) | + rte_hash_k16_cmp_eq((const uint8_t *) key1 + 16, + (const uint8_t *) key2 + 16, key_len) | + rte_hash_k16_cmp_eq((const uint8_t *) key1 + 32, + (const uint8_t *) key2 + 32, key_len); +} + +static int +rte_hash_k64_cmp_eq(const void *key1, const void *key2, size_t key_len) +{ + return rte_hash_k32_cmp_eq(key1, key2, key_len) | + rte_hash_k32_cmp_eq((const uint8_t *) key1 + 32, + (const uint8_t *) key2 + 32, key_len); +} + +static int +rte_hash_k80_cmp_eq(const void *key1, const void *key2, size_t key_len) +{ + return rte_hash_k64_cmp_eq(key1, key2, key_len) | + rte_hash_k16_cmp_eq((const uint8_t *) key1 + 64, + (const uint8_t *) key2 + 64, key_len); +} + +static int +rte_hash_k96_cmp_eq(const void *key1, const void *key2, size_t key_len) +{ + return rte_hash_k64_cmp_eq(key1, key2, key_len) | + rte_hash_k32_cmp_eq((const uint8_t *) key1 + 64, + (const uint8_t *) key2 + 64, key_len); +} + +static int +rte_hash_k112_cmp_eq(const void *key1, const void *key2, size_t key_len) +{ + return rte_hash_k64_cmp_eq(key1, key2, key_len) | + rte_hash_k32_cmp_eq((const uint8_t *) key1 + 64, + (const uint8_t *) key2 + 64, key_len) | + rte_hash_k16_cmp_eq((const uint8_t *) key1 + 96, + (const uint8_t *) key2 + 96, key_len); +} + +static int +rte_hash_k128_cmp_eq(const void *key1, const void *key2, size_t key_len) +{ + return rte_hash_k64_cmp_eq(key1, key2, key_len) | + rte_hash_k64_cmp_eq((const uint8_t *) key1 + 64, + (const uint8_t *) key2 + 64, key_len); +} + /* Enum used to select the implementation of the signature comparison function to use * eg: a system supporting SVE might want to use a NEON or scalar implementation. */ @@ -160,7 +226,6 @@ void rte_hash_set_cmp_func(struct rte_hash *h, rte_hash_cmp_eq_t func) */ static const rte_hash_cmp_eq_t cmp_jump_table[NUM_KEY_CMP_CASES] = { [KEY_CUSTOM] = NULL, -#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) [KEY_16_BYTES] = rte_hash_k16_cmp_eq, [KEY_32_BYTES] = rte_hash_k32_cmp_eq, [KEY_48_BYTES] = rte_hash_k48_cmp_eq, @@ -169,7 +234,6 @@ static const rte_hash_cmp_eq_t cmp_jump_table[NUM_KEY_CMP_CASES] = { [KEY_96_BYTES] = rte_hash_k96_cmp_eq, [KEY_112_BYTES] = rte_hash_k112_cmp_eq, [KEY_128_BYTES] = rte_hash_k128_cmp_eq, -#endif [KEY_OTHER_BYTES] = memcmp, }; -- 2.51.0
