The lookup loaded 64-bit words and shifted the entry out of them. That takes LMUL=8 values whatever the next hop size, one register group more than there is, so a group was spilled inside the loop. Gather the entries at the size they are stored with instead.
Signed-off-by: Sun Yuechi <[email protected]> --- lib/fib/dir24_8_rvv.c | 85 ++++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/lib/fib/dir24_8_rvv.c b/lib/fib/dir24_8_rvv.c index 9c14ca0481..9b784bb3c5 100644 --- a/lib/fib/dir24_8_rvv.c +++ b/lib/fib/dir24_8_rvv.c @@ -10,55 +10,64 @@ #include "dir24_8.h" #include "dir24_8_rvv.h" -#define DECLARE_VECTOR_FN(SFX, NH_SZ) \ +/* Byte offset of entry number idx, for each next hop size. */ +#define OFS_1b(idx, vl) ((void)(vl), (idx)) +#define OFS_2b(idx, vl) __riscv_vsll_vx_u32m4(idx, 1, vl) +#define OFS_4b(idx, vl) __riscv_vsll_vx_u32m4(idx, 2, vl) +#define OFS_8b(idx, vl) __riscv_vsll_vx_u32m4(idx, 3, vl) + +/* Entry without its low bit, as 32-bit tbl8 group number. */ +#define GRP_1b(ent, vl) \ + __riscv_vzext_vf4_u32m4(__riscv_vsrl_vx_u8m1(ent, 1, vl), vl) +#define GRP_2b(ent, vl) \ + __riscv_vzext_vf2_u32m4(__riscv_vsrl_vx_u16m2(ent, 1, vl), vl) +#define GRP_4b(ent, vl) __riscv_vsrl_vx_u32m4(ent, 1, vl) +#define GRP_8b(ent, vl) __riscv_vnsrl_wx_u32m4(ent, 1, vl) + +/* Entry without its low bit, as 64-bit next hop. */ +#define NH_1b(ent, vl) \ + __riscv_vzext_vf8_u64m8(__riscv_vsrl_vx_u8m1(ent, 1, vl), vl) +#define NH_2b(ent, vl) \ + __riscv_vzext_vf4_u64m8(__riscv_vsrl_vx_u16m2(ent, 1, vl), vl) +#define NH_4b(ent, vl) \ + __riscv_vzext_vf2_u64m8(__riscv_vsrl_vx_u32m4(ent, 1, vl), vl) +#define NH_8b(ent, vl) __riscv_vsrl_vx_u64m8(ent, 1, vl) + +/* Entries are gathered at their own width, with 32-bit byte offsets. */ +#define DECLARE_VECTOR_FN(SFX, TYPE, BITS, LMUL) \ void \ rte_dir24_8_vec_lookup_bulk_##SFX(void *p, \ const uint32_t *ips, uint64_t *next_hops, unsigned int n) \ { \ - const uint8_t idx_bits = 3 - NH_SZ; \ - const uint32_t idx_mask = (1u << (3 - NH_SZ)) - 1u; \ - const uint64_t e_mask = ~0ULL >> (64 - (8u << NH_SZ)); \ - struct dir24_8_tbl *tbl = (struct dir24_8_tbl *)p; \ - const uint64_t *tbl24 = tbl->tbl24; \ + const struct dir24_8_tbl *tbl = (const struct dir24_8_tbl *)p; \ + const TYPE *tbl24 = (const TYPE *)tbl->tbl24; \ + const TYPE *tbl8 = (const TYPE *)tbl->tbl8; \ size_t vl; \ for (unsigned int i = 0; i < n; i += vl) { \ vl = __riscv_vsetvl_e32m4(n - i); \ vuint32m4_t v_ips = __riscv_vle32_v_u32m4(&ips[i], vl); \ - vuint64m8_t vtbl_word = __riscv_vluxei32_v_u64m8(tbl24, \ - __riscv_vsll_vx_u32m4( \ - __riscv_vsrl_vx_u32m4(v_ips, idx_bits + 8, vl), 3, vl), vl); \ - vuint32m4_t v_tbl_index = __riscv_vsrl_vx_u32m4(v_ips, 8, vl); \ - vuint32m4_t v_entry_idx = __riscv_vand_vx_u32m4(v_tbl_index, idx_mask, vl); \ - vuint32m4_t v_shift = __riscv_vsll_vx_u32m4(v_entry_idx, 3 + NH_SZ, vl); \ - vuint64m8_t vtbl_entry = __riscv_vand_vx_u64m8( \ - __riscv_vsrl_vv_u64m8(vtbl_word, \ - __riscv_vwcvtu_x_x_v_u64m8(v_shift, vl), vl), e_mask, vl); \ - vbool8_t mask = __riscv_vmseq_vx_u64m8_b8( \ - __riscv_vand_vx_u64m8(vtbl_entry, 1, vl), 1, vl); \ - if (__riscv_vcpop_m_b8(mask, vl)) { \ - const uint64_t *tbl8 = tbl->tbl8; \ - v_tbl_index = __riscv_vadd_vv_u32m4_mu(mask, v_tbl_index, \ - __riscv_vsll_vx_u32m4( \ - __riscv_vnsrl_wx_u32m4(vtbl_entry, 1, vl), 8, vl), \ - __riscv_vand_vx_u32m4(v_ips, 0xFF, vl), vl); \ - vtbl_word = __riscv_vluxei32_v_u64m8_mu(mask, vtbl_word, tbl8, \ - __riscv_vsll_vx_u32m4( \ - __riscv_vsrl_vx_u32m4(v_tbl_index, idx_bits, vl), 3, vl), \ - vl); \ - v_entry_idx = __riscv_vand_vx_u32m4(v_tbl_index, idx_mask, vl); \ - v_shift = __riscv_vsll_vx_u32m4(v_entry_idx, 3 + NH_SZ, vl); \ - vtbl_entry = __riscv_vand_vx_u64m8( \ - __riscv_vsrl_vv_u64m8(vtbl_word, \ - __riscv_vwcvtu_x_x_v_u64m8(v_shift, vl), vl), e_mask, vl); \ + vuint##BITS##m##LMUL##_t v_ent = \ + __riscv_vluxei32_v_u##BITS##m##LMUL(tbl24, \ + OFS_##SFX(__riscv_vsrl_vx_u32m4(v_ips, 8, vl), \ + vl), vl); \ + vbool8_t mask = __riscv_vmsne_vx_u##BITS##m##LMUL##_b8( \ + __riscv_vand_vx_u##BITS##m##LMUL(v_ent, \ + DIR24_8_EXT_ENT, vl), 0, vl); \ + if (unlikely(__riscv_vfirst_m_b8(mask, vl) >= 0)) { \ + vuint32m4_t v_idx = __riscv_vadd_vv_u32m4( \ + __riscv_vsll_vx_u32m4(GRP_##SFX(v_ent, vl), \ + 8, vl), \ + __riscv_vand_vx_u32m4(v_ips, 0xFF, vl), vl); \ + v_ent = __riscv_vluxei32_v_u##BITS##m##LMUL##_mu(mask, \ + v_ent, tbl8, OFS_##SFX(v_idx, vl), vl); \ } \ - __riscv_vse64_v_u64m8(&next_hops[i], \ - __riscv_vsrl_vx_u64m8(vtbl_entry, 1, vl), vl); \ + __riscv_vse64_v_u64m8(&next_hops[i], NH_##SFX(v_ent, vl), vl); \ } \ } -DECLARE_VECTOR_FN(1b, 0) -DECLARE_VECTOR_FN(2b, 1) -DECLARE_VECTOR_FN(4b, 2) -DECLARE_VECTOR_FN(8b, 3) +DECLARE_VECTOR_FN(1b, uint8_t, 8, 1) +DECLARE_VECTOR_FN(2b, uint16_t, 16, 2) +DECLARE_VECTOR_FN(4b, uint32_t, 32, 4) +DECLARE_VECTOR_FN(8b, uint64_t, 64, 8) #endif -- 2.56.0
