> Add VRF (Virtual Routing and Forwarding) support to the IPv4
> FIB library, allowing multiple independent routing tables
> within a single FIB instance.
>
> Introduce max_vrfs and vrf_default_nh fields in rte_fib_conf
> to configure the number of VRFs and per-VRF default nexthops.
Thanks Vladimir, allowing multiple VRFs per same LPM table will
definitely be a useful thing to have.
Though, I have the same concern as Maxime:
memory requirements are just overwhelming.
Stupid q - why just not to store a pointer to a vector of next-hops
within the table entry?
And we can provide to the user with ability to specify custom
alloc/free function for these vectors.
That would help to avoid allocating huge chunks of memory at startup.
I understand that it will be one extra memory dereference,
but probably it will be not that critical in terms of performance .
Again for bulk function we might be able to pipeline lookups and
de-references and hide that extra load latency.
> Add four new experimental APIs:
> - rte_fib_vrf_add() and rte_fib_vrf_delete() to manage routes
> per VRF
> - rte_fib_vrf_lookup_bulk() for multi-VRF bulk lookups
> - rte_fib_vrf_get_rib() to retrieve a per-VRF RIB handle
>
> Signed-off-by: Vladimir Medvedkin <[email protected]>
> ---
> lib/fib/dir24_8.c | 241 ++++++++++++++++------
> lib/fib/dir24_8.h | 255 ++++++++++++++++--------
> lib/fib/dir24_8_avx512.c | 420 +++++++++++++++++++++++++++++++--------
> lib/fib/dir24_8_avx512.h | 80 +++++++-
> lib/fib/rte_fib.c | 158 ++++++++++++---
> lib/fib/rte_fib.h | 94 ++++++++-
> 6 files changed, 988 insertions(+), 260 deletions(-)
>
> diff --git a/lib/fib/dir24_8.c b/lib/fib/dir24_8.c
> index 489d2ef427..ad295c5f16 100644
> --- a/lib/fib/dir24_8.c
> +++ b/lib/fib/dir24_8.c
> @@ -32,41 +32,80 @@
> #define ROUNDUP(x, y) RTE_ALIGN_CEIL(x, (1 << (32 - y)))
>
> static inline rte_fib_lookup_fn_t
> -get_scalar_fn(enum rte_fib_dir24_8_nh_sz nh_sz, bool be_addr)
> +get_scalar_fn(const struct dir24_8_tbl *dp, enum rte_fib_dir24_8_nh_sz nh_sz,
> + bool be_addr)
> {
> + bool single_vrf = dp->num_vrfs <= 1;
> +
> switch (nh_sz) {
> case RTE_FIB_DIR24_8_1B:
> - return be_addr ? dir24_8_lookup_bulk_1b_be :
> dir24_8_lookup_bulk_1b;
> + if (single_vrf)
> + return be_addr ? dir24_8_lookup_bulk_1b_be :
> + dir24_8_lookup_bulk_1b;
> + return be_addr ? dir24_8_lookup_bulk_vrf_1b_be :
> + dir24_8_lookup_bulk_vrf_1b;
> case RTE_FIB_DIR24_8_2B:
> - return be_addr ? dir24_8_lookup_bulk_2b_be :
> dir24_8_lookup_bulk_2b;
> + if (single_vrf)
> + return be_addr ? dir24_8_lookup_bulk_2b_be :
> + dir24_8_lookup_bulk_2b;
> + return be_addr ? dir24_8_lookup_bulk_vrf_2b_be :
> + dir24_8_lookup_bulk_vrf_2b;
> case RTE_FIB_DIR24_8_4B:
> - return be_addr ? dir24_8_lookup_bulk_4b_be :
> dir24_8_lookup_bulk_4b;
> + if (single_vrf)
> + return be_addr ? dir24_8_lookup_bulk_4b_be :
> + dir24_8_lookup_bulk_4b;
> + return be_addr ? dir24_8_lookup_bulk_vrf_4b_be :
> + dir24_8_lookup_bulk_vrf_4b;
> case RTE_FIB_DIR24_8_8B:
> - return be_addr ? dir24_8_lookup_bulk_8b_be :
> dir24_8_lookup_bulk_8b;
> + if (single_vrf)
> + return be_addr ? dir24_8_lookup_bulk_8b_be :
> + dir24_8_lookup_bulk_8b;
> + return be_addr ? dir24_8_lookup_bulk_vrf_8b_be :
> + dir24_8_lookup_bulk_vrf_8b;
> default:
> return NULL;
> }
> }
>
> static inline rte_fib_lookup_fn_t
> -get_scalar_fn_inlined(enum rte_fib_dir24_8_nh_sz nh_sz, bool be_addr)
> +get_scalar_fn_inlined(const struct dir24_8_tbl *dp,
> + enum rte_fib_dir24_8_nh_sz nh_sz, bool be_addr)
> {
> + bool single_vrf = dp->num_vrfs <= 1;
> +
> switch (nh_sz) {
> case RTE_FIB_DIR24_8_1B:
> - return be_addr ? dir24_8_lookup_bulk_0_be :
> dir24_8_lookup_bulk_0;
> + if (single_vrf)
> + return be_addr ? dir24_8_lookup_bulk_0_be :
> + dir24_8_lookup_bulk_0;
> + return be_addr ? dir24_8_lookup_bulk_vrf_0_be :
> + dir24_8_lookup_bulk_vrf_0;
> case RTE_FIB_DIR24_8_2B:
> - return be_addr ? dir24_8_lookup_bulk_1_be :
> dir24_8_lookup_bulk_1;
> + if (single_vrf)
> + return be_addr ? dir24_8_lookup_bulk_1_be :
> + dir24_8_lookup_bulk_1;
> + return be_addr ? dir24_8_lookup_bulk_vrf_1_be :
> + dir24_8_lookup_bulk_vrf_1;
> case RTE_FIB_DIR24_8_4B:
> - return be_addr ? dir24_8_lookup_bulk_2_be :
> dir24_8_lookup_bulk_2;
> + if (single_vrf)
> + return be_addr ? dir24_8_lookup_bulk_2_be :
> + dir24_8_lookup_bulk_2;
> + return be_addr ? dir24_8_lookup_bulk_vrf_2_be :
> + dir24_8_lookup_bulk_vrf_2;
> case RTE_FIB_DIR24_8_8B:
> - return be_addr ? dir24_8_lookup_bulk_3_be :
> dir24_8_lookup_bulk_3;
> + if (single_vrf)
> + return be_addr ? dir24_8_lookup_bulk_3_be :
> + dir24_8_lookup_bulk_3;
> + return be_addr ? dir24_8_lookup_bulk_vrf_3_be :
> + dir24_8_lookup_bulk_vrf_3;
> default:
> return NULL;
> }
> }
>
> static inline rte_fib_lookup_fn_t
> -get_vector_fn(enum rte_fib_dir24_8_nh_sz nh_sz, bool be_addr)
> +get_vector_fn(const struct dir24_8_tbl *dp, enum rte_fib_dir24_8_nh_sz nh_sz,
> + bool be_addr)
> {
> #ifdef CC_AVX512_SUPPORT
> if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) <= 0 ||
> @@ -77,24 +116,63 @@ get_vector_fn(enum rte_fib_dir24_8_nh_sz nh_sz, bool
> be_addr)
> if (be_addr && rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) <=
> 0)
> return NULL;
>
> + if (dp->num_vrfs <= 1) {
> + switch (nh_sz) {
> + case RTE_FIB_DIR24_8_1B:
> + return be_addr ? rte_dir24_8_vec_lookup_bulk_1b_be :
> + rte_dir24_8_vec_lookup_bulk_1b;
> + case RTE_FIB_DIR24_8_2B:
> + return be_addr ? rte_dir24_8_vec_lookup_bulk_2b_be :
> + rte_dir24_8_vec_lookup_bulk_2b;
> + case RTE_FIB_DIR24_8_4B:
> + return be_addr ? rte_dir24_8_vec_lookup_bulk_4b_be :
> + rte_dir24_8_vec_lookup_bulk_4b;
> + case RTE_FIB_DIR24_8_8B:
> + return be_addr ? rte_dir24_8_vec_lookup_bulk_8b_be :
> + rte_dir24_8_vec_lookup_bulk_8b;
> + default:
> + return NULL;
> + }
> + }
> +
> + if (dp->num_vrfs >= 256) {
> + switch (nh_sz) {
> + case RTE_FIB_DIR24_8_1B:
> + return be_addr ?
> rte_dir24_8_vec_lookup_bulk_vrf_1b_be_large :
> + rte_dir24_8_vec_lookup_bulk_vrf_1b_large;
> + case RTE_FIB_DIR24_8_2B:
> + return be_addr ?
> rte_dir24_8_vec_lookup_bulk_vrf_2b_be_large :
> + rte_dir24_8_vec_lookup_bulk_vrf_2b_large;
> + case RTE_FIB_DIR24_8_4B:
> + return be_addr ?
> rte_dir24_8_vec_lookup_bulk_vrf_4b_be_large :
> + rte_dir24_8_vec_lookup_bulk_vrf_4b_large;
> + case RTE_FIB_DIR24_8_8B:
> + return be_addr ?
> rte_dir24_8_vec_lookup_bulk_vrf_8b_be_large :
> + rte_dir24_8_vec_lookup_bulk_vrf_8b_large;
> + default:
> + return NULL;
> + }
> + }
> +
> switch (nh_sz) {
> case RTE_FIB_DIR24_8_1B:
> - return be_addr ? rte_dir24_8_vec_lookup_bulk_1b_be :
> - rte_dir24_8_vec_lookup_bulk_1b;
> + return be_addr ? rte_dir24_8_vec_lookup_bulk_vrf_1b_be :
> + rte_dir24_8_vec_lookup_bulk_vrf_1b;
> case RTE_FIB_DIR24_8_2B:
> - return be_addr ? rte_dir24_8_vec_lookup_bulk_2b_be :
> - rte_dir24_8_vec_lookup_bulk_2b;
> + return be_addr ? rte_dir24_8_vec_lookup_bulk_vrf_2b_be :
> + rte_dir24_8_vec_lookup_bulk_vrf_2b;
> case RTE_FIB_DIR24_8_4B:
> - return be_addr ? rte_dir24_8_vec_lookup_bulk_4b_be :
> - rte_dir24_8_vec_lookup_bulk_4b;
> + return be_addr ? rte_dir24_8_vec_lookup_bulk_vrf_4b_be :
> + rte_dir24_8_vec_lookup_bulk_vrf_4b;
> case RTE_FIB_DIR24_8_8B:
> - return be_addr ? rte_dir24_8_vec_lookup_bulk_8b_be :
> - rte_dir24_8_vec_lookup_bulk_8b;
> + return be_addr ? rte_dir24_8_vec_lookup_bulk_vrf_8b_be :
> + rte_dir24_8_vec_lookup_bulk_vrf_8b;
> default:
> return NULL;
> }
> #elif defined(RTE_RISCV_FEATURE_V)
> RTE_SET_USED(be_addr);
> + RTE_SET_USED(dp);
> if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_RISCV_ISA_V) <= 0)
> return NULL;
> switch (nh_sz) {
> @@ -130,16 +208,17 @@ dir24_8_get_lookup_fn(void *p, enum
> rte_fib_lookup_type type, bool be_addr)
>
> switch (type) {
> case RTE_FIB_LOOKUP_DIR24_8_SCALAR_MACRO:
> - return get_scalar_fn(nh_sz, be_addr);
> + return get_scalar_fn(dp, nh_sz, be_addr);
> case RTE_FIB_LOOKUP_DIR24_8_SCALAR_INLINE:
> - return get_scalar_fn_inlined(nh_sz, be_addr);
> + return get_scalar_fn_inlined(dp, nh_sz, be_addr);
> case RTE_FIB_LOOKUP_DIR24_8_SCALAR_UNI:
> - return be_addr ? dir24_8_lookup_bulk_uni_be :
> dir24_8_lookup_bulk_uni;
> + return be_addr ? dir24_8_lookup_bulk_uni_be :
> + dir24_8_lookup_bulk_uni;
> case RTE_FIB_LOOKUP_DIR24_8_VECTOR_AVX512:
> - return get_vector_fn(nh_sz, be_addr);
> + return get_vector_fn(dp, nh_sz, be_addr);
> case RTE_FIB_LOOKUP_DEFAULT:
> - ret_fn = get_vector_fn(nh_sz, be_addr);
> - return ret_fn != NULL ? ret_fn : get_scalar_fn(nh_sz, be_addr);
> + ret_fn = get_vector_fn(dp, nh_sz, be_addr);
> + return ret_fn != NULL ? ret_fn : get_scalar_fn(dp, nh_sz,
> be_addr);
> default:
> return NULL;
> }
> @@ -246,15 +325,18 @@ __rcu_qsbr_free_resource(void *p, void *data,
> unsigned int n __rte_unused)
> }
>
> static void
> -tbl8_recycle(struct dir24_8_tbl *dp, uint32_t ip, uint64_t tbl8_idx)
> +tbl8_recycle(struct dir24_8_tbl *dp, uint16_t vrf_id, uint32_t ip, uint64_t
> tbl8_idx)
> {
> uint32_t i;
> uint64_t nh;
> + uint64_t tbl24_idx;
> uint8_t *ptr8;
> uint16_t *ptr16;
> uint32_t *ptr32;
> uint64_t *ptr64;
>
> + tbl24_idx = get_tbl24_idx(vrf_id, ip);
> +
> switch (dp->nh_sz) {
> case RTE_FIB_DIR24_8_1B:
> ptr8 = &((uint8_t *)dp->tbl8)[tbl8_idx *
> @@ -264,7 +346,7 @@ tbl8_recycle(struct dir24_8_tbl *dp, uint32_t ip, uint64_t
> tbl8_idx)
> if (nh != ptr8[i])
> return;
> }
> - ((uint8_t *)dp->tbl24)[ip >> 8] =
> + ((uint8_t *)dp->tbl24)[tbl24_idx] =
> nh & ~DIR24_8_EXT_ENT;
> break;
> case RTE_FIB_DIR24_8_2B:
> @@ -275,7 +357,7 @@ tbl8_recycle(struct dir24_8_tbl *dp, uint32_t ip, uint64_t
> tbl8_idx)
> if (nh != ptr16[i])
> return;
> }
> - ((uint16_t *)dp->tbl24)[ip >> 8] =
> + ((uint16_t *)dp->tbl24)[tbl24_idx] =
> nh & ~DIR24_8_EXT_ENT;
> break;
> case RTE_FIB_DIR24_8_4B:
> @@ -286,7 +368,7 @@ tbl8_recycle(struct dir24_8_tbl *dp, uint32_t ip, uint64_t
> tbl8_idx)
> if (nh != ptr32[i])
> return;
> }
> - ((uint32_t *)dp->tbl24)[ip >> 8] =
> + ((uint32_t *)dp->tbl24)[tbl24_idx] =
> nh & ~DIR24_8_EXT_ENT;
> break;
> case RTE_FIB_DIR24_8_8B:
> @@ -297,7 +379,7 @@ tbl8_recycle(struct dir24_8_tbl *dp, uint32_t ip, uint64_t
> tbl8_idx)
> if (nh != ptr64[i])
> return;
> }
> - ((uint64_t *)dp->tbl24)[ip >> 8] =
> + ((uint64_t *)dp->tbl24)[tbl24_idx] =
> nh & ~DIR24_8_EXT_ENT;
> break;
> }
> @@ -314,7 +396,7 @@ tbl8_recycle(struct dir24_8_tbl *dp, uint32_t ip, uint64_t
> tbl8_idx)
> }
>
> static int
> -install_to_fib(struct dir24_8_tbl *dp, uint32_t ledge, uint32_t redge,
> +install_to_fib(struct dir24_8_tbl *dp, uint16_t vrf_id, uint32_t ledge,
> uint32_t
> redge,
> uint64_t next_hop)
> {
> uint64_t tbl24_tmp;
> @@ -328,7 +410,7 @@ install_to_fib(struct dir24_8_tbl *dp, uint32_t ledge,
> uint32_t redge,
>
> if (((ledge >> 8) != (redge >> 8)) || (len == 1 << 24)) {
> if ((ROUNDUP(ledge, 24) - ledge) != 0) {
> - tbl24_tmp = get_tbl24(dp, ledge, dp->nh_sz);
> + tbl24_tmp = get_tbl24(dp, vrf_id, ledge, dp->nh_sz);
> if ((tbl24_tmp & DIR24_8_EXT_ENT) !=
> DIR24_8_EXT_ENT) {
> /**
> @@ -346,7 +428,7 @@ install_to_fib(struct dir24_8_tbl *dp, uint32_t ledge,
> uint32_t redge,
> }
> tbl8_free_idx(dp, tmp_tbl8_idx);
> /*update dir24 entry with tbl8 index*/
> - write_to_fib(get_tbl24_p(dp, ledge,
> + write_to_fib(get_tbl24_p(dp, vrf_id, ledge,
> dp->nh_sz), (tbl8_idx << 1)|
> DIR24_8_EXT_ENT,
> dp->nh_sz, 1);
> @@ -360,19 +442,19 @@ install_to_fib(struct dir24_8_tbl *dp, uint32_t ledge,
> uint32_t redge,
> write_to_fib((void *)tbl8_ptr, (next_hop << 1)|
> DIR24_8_EXT_ENT,
> dp->nh_sz, ROUNDUP(ledge, 24) - ledge);
> - tbl8_recycle(dp, ledge, tbl8_idx);
> + tbl8_recycle(dp, vrf_id, ledge, tbl8_idx);
> }
> - write_to_fib(get_tbl24_p(dp, ROUNDUP(ledge, 24), dp->nh_sz),
> + write_to_fib(get_tbl24_p(dp, vrf_id, ROUNDUP(ledge, 24), dp-
> >nh_sz),
> next_hop << 1, dp->nh_sz, len);
> if (redge & ~DIR24_8_TBL24_MASK) {
> - tbl24_tmp = get_tbl24(dp, redge, dp->nh_sz);
> + tbl24_tmp = get_tbl24(dp, vrf_id, redge, dp->nh_sz);
> if ((tbl24_tmp & DIR24_8_EXT_ENT) !=
> DIR24_8_EXT_ENT) {
> tbl8_idx = tbl8_alloc(dp, tbl24_tmp);
> if (tbl8_idx < 0)
> return -ENOSPC;
> /*update dir24 entry with tbl8 index*/
> - write_to_fib(get_tbl24_p(dp, redge,
> + write_to_fib(get_tbl24_p(dp, vrf_id, redge,
> dp->nh_sz), (tbl8_idx << 1)|
> DIR24_8_EXT_ENT,
> dp->nh_sz, 1);
> @@ -385,17 +467,17 @@ install_to_fib(struct dir24_8_tbl *dp, uint32_t ledge,
> uint32_t redge,
> write_to_fib((void *)tbl8_ptr, (next_hop << 1)|
> DIR24_8_EXT_ENT,
> dp->nh_sz, redge & ~DIR24_8_TBL24_MASK);
> - tbl8_recycle(dp, redge, tbl8_idx);
> + tbl8_recycle(dp, vrf_id, redge, tbl8_idx);
> }
> } else if ((redge - ledge) != 0) {
> - tbl24_tmp = get_tbl24(dp, ledge, dp->nh_sz);
> + tbl24_tmp = get_tbl24(dp, vrf_id, ledge, dp->nh_sz);
> if ((tbl24_tmp & DIR24_8_EXT_ENT) !=
> DIR24_8_EXT_ENT) {
> tbl8_idx = tbl8_alloc(dp, tbl24_tmp);
> if (tbl8_idx < 0)
> return -ENOSPC;
> /*update dir24 entry with tbl8 index*/
> - write_to_fib(get_tbl24_p(dp, ledge, dp->nh_sz),
> + write_to_fib(get_tbl24_p(dp, vrf_id, ledge, dp->nh_sz),
> (tbl8_idx << 1)|
> DIR24_8_EXT_ENT,
> dp->nh_sz, 1);
> @@ -409,13 +491,13 @@ install_to_fib(struct dir24_8_tbl *dp, uint32_t ledge,
> uint32_t redge,
> write_to_fib((void *)tbl8_ptr, (next_hop << 1)|
> DIR24_8_EXT_ENT,
> dp->nh_sz, redge - ledge);
> - tbl8_recycle(dp, ledge, tbl8_idx);
> + tbl8_recycle(dp, vrf_id, ledge, tbl8_idx);
> }
> return 0;
> }
>
> static int
> -modify_fib(struct dir24_8_tbl *dp, struct rte_rib *rib, uint32_t ip,
> +modify_fib(struct dir24_8_tbl *dp, struct rte_rib *rib, uint16_t vrf_id,
> uint32_t
> ip,
> uint8_t depth, uint64_t next_hop)
> {
> struct rte_rib_node *tmp = NULL;
> @@ -438,7 +520,7 @@ modify_fib(struct dir24_8_tbl *dp, struct rte_rib *rib,
> uint32_t ip,
> (uint32_t)(1ULL << (32 - tmp_depth));
> continue;
> }
> - ret = install_to_fib(dp, ledge, redge,
> + ret = install_to_fib(dp, vrf_id, ledge, redge,
> next_hop);
> if (ret != 0)
> return ret;
> @@ -454,7 +536,7 @@ modify_fib(struct dir24_8_tbl *dp, struct rte_rib *rib,
> uint32_t ip,
> redge = ip + (uint32_t)(1ULL << (32 - depth));
> if (ledge == redge && ledge != 0)
> break;
> - ret = install_to_fib(dp, ledge, redge,
> + ret = install_to_fib(dp, vrf_id, ledge, redge,
> next_hop);
> if (ret != 0)
> return ret;
> @@ -465,7 +547,7 @@ modify_fib(struct dir24_8_tbl *dp, struct rte_rib *rib,
> uint32_t ip,
> }
>
> int
> -dir24_8_modify(struct rte_fib *fib, uint32_t ip, uint8_t depth,
> +dir24_8_modify(struct rte_fib *fib, uint16_t vrf_id, uint32_t ip, uint8_t
> depth,
> uint64_t next_hop, int op)
> {
> struct dir24_8_tbl *dp;
> @@ -480,8 +562,13 @@ dir24_8_modify(struct rte_fib *fib, uint32_t ip, uint8_t
> depth,
> return -EINVAL;
>
> dp = rte_fib_get_dp(fib);
> - rib = rte_fib_get_rib(fib);
> - RTE_ASSERT((dp != NULL) && (rib != NULL));
> + RTE_ASSERT(dp != NULL);
> +
> + if (vrf_id >= dp->num_vrfs)
> + return -EINVAL;
> +
> + rib = rte_fib_vrf_get_rib(fib, vrf_id);
> + RTE_ASSERT(rib != NULL);
>
> if (next_hop > get_max_nh(dp->nh_sz))
> return -EINVAL;
> @@ -495,7 +582,7 @@ dir24_8_modify(struct rte_fib *fib, uint32_t ip, uint8_t
> depth,
> rte_rib_get_nh(node, &node_nh);
> if (node_nh == next_hop)
> return 0;
> - ret = modify_fib(dp, rib, ip, depth, next_hop);
> + ret = modify_fib(dp, rib, vrf_id, ip, depth, next_hop);
> if (ret == 0)
> rte_rib_set_nh(node, next_hop);
> return 0;
> @@ -518,7 +605,7 @@ dir24_8_modify(struct rte_fib *fib, uint32_t ip, uint8_t
> depth,
> if (par_nh == next_hop)
> goto successfully_added;
> }
> - ret = modify_fib(dp, rib, ip, depth, next_hop);
> + ret = modify_fib(dp, rib, vrf_id, ip, depth, next_hop);
> if (ret != 0) {
> rte_rib_remove(rib, ip, depth);
> return ret;
> @@ -536,9 +623,9 @@ dir24_8_modify(struct rte_fib *fib, uint32_t ip, uint8_t
> depth,
> rte_rib_get_nh(parent, &par_nh);
> rte_rib_get_nh(node, &node_nh);
> if (par_nh != node_nh)
> - ret = modify_fib(dp, rib, ip, depth, par_nh);
> + ret = modify_fib(dp, rib, vrf_id, ip, depth,
> par_nh);
> } else
> - ret = modify_fib(dp, rib, ip, depth, dp->def_nh);
> + ret = modify_fib(dp, rib, vrf_id, ip, depth, dp-
> >def_nh[vrf_id]);
> if (ret == 0) {
> rte_rib_remove(rib, ip, depth);
> if (depth > 24) {
> @@ -562,7 +649,10 @@ dir24_8_create(const char *name, int socket_id, struct
> rte_fib_conf *fib_conf)
> struct dir24_8_tbl *dp;
> uint64_t def_nh;
> uint32_t num_tbl8;
> + uint16_t num_vrfs;
> enum rte_fib_dir24_8_nh_sz nh_sz;
> + uint64_t tbl24_sz;
> + uint16_t vrf;
>
> if ((name == NULL) || (fib_conf == NULL) ||
> (fib_conf->dir24_8.nh_sz < RTE_FIB_DIR24_8_1B) ||
> @@ -580,19 +670,56 @@ dir24_8_create(const char *name, int socket_id, struct
> rte_fib_conf *fib_conf)
> nh_sz = fib_conf->dir24_8.nh_sz;
> num_tbl8 = RTE_ALIGN_CEIL(fib_conf->dir24_8.num_tbl8,
> BITMAP_SLAB_BIT_SIZE);
> + num_vrfs = (fib_conf->max_vrfs == 0) ? 1 : fib_conf->max_vrfs;
> +
> + /* Validate per-VRF default nexthops if provided */
> + if (fib_conf->vrf_default_nh != NULL) {
> + for (vrf = 0; vrf < num_vrfs; vrf++) {
> + if (fib_conf->vrf_default_nh[vrf] > get_max_nh(nh_sz)) {
> + rte_errno = EINVAL;
> + return NULL;
> + }
> + }
> + }
> +
> + tbl24_sz = (uint64_t)num_vrfs * DIR24_8_TBL24_NUM_ENT * (1 <<
> nh_sz);
>
> snprintf(mem_name, sizeof(mem_name), "DP_%s", name);
> dp = rte_zmalloc_socket(name, sizeof(struct dir24_8_tbl) +
> - DIR24_8_TBL24_NUM_ENT * (1 << nh_sz) + sizeof(uint32_t),
> + tbl24_sz + sizeof(uint32_t),
> RTE_CACHE_LINE_SIZE, socket_id);
> if (dp == NULL) {
> rte_errno = ENOMEM;
> return NULL;
> }
>
> - /* Init table with default value */
> - write_to_fib(dp->tbl24, (def_nh << 1), nh_sz, 1 << 24);
> + dp->num_vrfs = num_vrfs;
> + dp->nh_sz = nh_sz;
> + dp->number_tbl8s = num_tbl8;
> +
> + /* Allocate per-VRF default nexthop array */
> + snprintf(mem_name, sizeof(mem_name), "DEFNH_%p", dp);
> + dp->def_nh = rte_zmalloc_socket(mem_name, num_vrfs *
> sizeof(uint64_t),
> + RTE_CACHE_LINE_SIZE, socket_id);
> + if (dp->def_nh == NULL) {
> + rte_errno = ENOMEM;
> + rte_free(dp);
> + return NULL;
> + }
> +
> + /* Initialize all VRFs with default nexthop */
> + for (vrf = 0; vrf < num_vrfs; vrf++) {
> + uint64_t vrf_def_nh = (fib_conf->vrf_default_nh != NULL) ?
> + fib_conf->vrf_default_nh[vrf] : def_nh;
> + dp->def_nh[vrf] = vrf_def_nh;
>
> + /* Init TBL24 for this VRF with default value */
> + uint64_t vrf_offset = (uint64_t)vrf * DIR24_8_TBL24_NUM_ENT;
> + void *vrf_tbl24 = (void *)&((uint8_t *)dp->tbl24)[vrf_offset <<
> nh_sz];
> + write_to_fib(vrf_tbl24, (vrf_def_nh << 1), nh_sz, 1 << 24);
> + }
> +
> + /* Allocate shared TBL8 for all VRFs */
> snprintf(mem_name, sizeof(mem_name), "TBL8_%p", dp);
> uint64_t tbl8_sz = DIR24_8_TBL8_GRP_NUM_ENT * (1ULL << nh_sz) *
> (num_tbl8 + 1);
> @@ -600,12 +727,10 @@ dir24_8_create(const char *name, int socket_id, struct
> rte_fib_conf *fib_conf)
> RTE_CACHE_LINE_SIZE, socket_id);
> if (dp->tbl8 == NULL) {
> rte_errno = ENOMEM;
> + rte_free(dp->def_nh);
> rte_free(dp);
> return NULL;
> }
> - dp->def_nh = def_nh;
> - dp->nh_sz = nh_sz;
> - dp->number_tbl8s = num_tbl8;
>
> snprintf(mem_name, sizeof(mem_name), "TBL8_idxes_%p", dp);
> dp->tbl8_idxes = rte_zmalloc_socket(mem_name,
> @@ -614,6 +739,7 @@ dir24_8_create(const char *name, int socket_id, struct
> rte_fib_conf *fib_conf)
> if (dp->tbl8_idxes == NULL) {
> rte_errno = ENOMEM;
> rte_free(dp->tbl8);
> + rte_free(dp->def_nh);
> rte_free(dp);
> return NULL;
> }
> @@ -629,6 +755,7 @@ dir24_8_free(void *p)
> rte_rcu_qsbr_dq_delete(dp->dq);
> rte_free(dp->tbl8_idxes);
> rte_free(dp->tbl8);
> + rte_free(dp->def_nh);
> rte_free(dp);
> }
>
> diff --git a/lib/fib/dir24_8.h b/lib/fib/dir24_8.h
> index b343b5d686..37a73a3cc2 100644
> --- a/lib/fib/dir24_8.h
> +++ b/lib/fib/dir24_8.h
> @@ -12,6 +12,7 @@
> #include <rte_byteorder.h>
> #include <rte_prefetch.h>
> #include <rte_branch_prediction.h>
> +#include <rte_debug.h>
> #include <rte_rcu_qsbr.h>
>
> /**
> @@ -32,24 +33,19 @@ struct dir24_8_tbl {
> uint32_t number_tbl8s; /**< Total number of tbl8s */
> uint32_t rsvd_tbl8s; /**< Number of reserved tbl8s */
> uint32_t cur_tbl8s; /**< Current number of tbl8s */
> + uint16_t num_vrfs; /**< Number of VRFs */
> enum rte_fib_dir24_8_nh_sz nh_sz; /**< Size of nexthop entry */
> /* RCU config. */
> enum rte_fib_qsbr_mode rcu_mode;/* Blocking, defer queue. */
> struct rte_rcu_qsbr *v; /* RCU QSBR variable. */
> struct rte_rcu_qsbr_dq *dq; /* RCU QSBR defer queue. */
> - uint64_t def_nh; /**< Default next hop */
> + uint64_t *def_nh; /**< Per-VRF default next hop array */
> uint64_t *tbl8; /**< tbl8 table. */
> uint64_t *tbl8_idxes; /**< bitmap containing free tbl8 idxes*/
> /* tbl24 table. */
> alignas(RTE_CACHE_LINE_SIZE) uint64_t tbl24[];
> };
>
> -static inline void *
> -get_tbl24_p(struct dir24_8_tbl *dp, uint32_t ip, uint8_t nh_sz)
> -{
> - return (void *)&((uint8_t *)dp->tbl24)[(ip &
> - DIR24_8_TBL24_MASK) >> (8 - nh_sz)];
> -}
>
> static inline uint8_t
> bits_in_nh(uint8_t nh_sz)
> @@ -63,14 +59,21 @@ get_max_nh(uint8_t nh_sz)
> return ((1ULL << (bits_in_nh(nh_sz) - 1)) - 1);
> }
>
> -static inline uint32_t
> -get_tbl24_idx(uint32_t ip)
> +static inline uint64_t
> +get_tbl24_idx(uint16_t vrf_id, uint32_t ip)
> +{
> + return ((uint64_t)vrf_id << 24) + (ip >> 8);
> +}
> +
> +static inline void *
> +get_tbl24_p(struct dir24_8_tbl *dp, uint16_t vrf_id, uint32_t ip, uint8_t
> nh_sz)
> {
> - return ip >> 8;
> + uint64_t idx = get_tbl24_idx(vrf_id, ip);
> + return (void *)&((uint8_t *)dp->tbl24)[idx << nh_sz];
> }
>
> -static inline uint32_t
> -get_tbl8_idx(uint32_t res, uint32_t ip)
> +static inline uint64_t
> +get_tbl8_idx(uint64_t res, uint32_t ip)
> {
> return (res >> 1) * DIR24_8_TBL8_GRP_NUM_ENT + (uint8_t)ip;
> }
> @@ -87,17 +90,18 @@ get_psd_idx(uint32_t val, uint8_t nh_sz)
> return val & ((1 << (3 - nh_sz)) - 1);
> }
>
> -static inline uint32_t
> -get_tbl_idx(uint32_t val, uint8_t nh_sz)
> +static inline uint64_t
> +get_tbl_idx(uint64_t val, uint8_t nh_sz)
> {
> return val >> (3 - nh_sz);
> }
>
> static inline uint64_t
> -get_tbl24(struct dir24_8_tbl *dp, uint32_t ip, uint8_t nh_sz)
> +get_tbl24(struct dir24_8_tbl *dp, uint16_t vrf_id, uint32_t ip, uint8_t
> nh_sz)
> {
> - return ((dp->tbl24[get_tbl_idx(get_tbl24_idx(ip), nh_sz)] >>
> - (get_psd_idx(get_tbl24_idx(ip), nh_sz) *
> + uint64_t idx = get_tbl24_idx(vrf_id, ip);
> + return ((dp->tbl24[get_tbl_idx(idx, nh_sz)] >>
> + (get_psd_idx(idx, nh_sz) *
> bits_in_nh(nh_sz))) & lookup_msk(nh_sz));
> }
>
> @@ -115,62 +119,92 @@ is_entry_extended(uint64_t ent)
> return (ent & DIR24_8_EXT_ENT) == DIR24_8_EXT_ENT;
> }
>
> -#define LOOKUP_FUNC(suffix, type, bulk_prefetch, nh_sz)
> \
> -static inline void dir24_8_lookup_bulk_##suffix(void *p, const uint32_t
> *ips, \
> - uint64_t *next_hops, const unsigned int n) \
> -{ \
> - struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p; \
> - uint64_t tmp; \
> - uint32_t i; \
> - uint32_t prefetch_offset = \
> - RTE_MIN((unsigned int)bulk_prefetch, n); \
> - \
> - for (i = 0; i < prefetch_offset; i++) \
> - rte_prefetch0(get_tbl24_p(dp, ips[i], nh_sz)); \
> - for (i = 0; i < (n - prefetch_offset); i++) { \
> - rte_prefetch0(get_tbl24_p(dp, \
> - ips[i + prefetch_offset], nh_sz)); \
> - tmp = ((type *)dp->tbl24)[ips[i] >> 8]; \
> - if (unlikely(is_entry_extended(tmp))) \
> - tmp = ((type *)dp->tbl8)[(uint8_t)ips[i] + \
> - ((tmp >> 1) * DIR24_8_TBL8_GRP_NUM_ENT)]; \
> - next_hops[i] = tmp >> 1; \
> - } \
> - for (; i < n; i++) { \
> - tmp = ((type *)dp->tbl24)[ips[i] >> 8]; \
> - if (unlikely(is_entry_extended(tmp))) \
> - tmp = ((type *)dp->tbl8)[(uint8_t)ips[i] + \
> - ((tmp >> 1) * DIR24_8_TBL8_GRP_NUM_ENT)]; \
> - next_hops[i] = tmp >> 1; \
> - } \
> -} \
> -
> -LOOKUP_FUNC(1b, uint8_t, 5, 0)
> -LOOKUP_FUNC(2b, uint16_t, 6, 1)
> -LOOKUP_FUNC(4b, uint32_t, 15, 2)
> -LOOKUP_FUNC(8b, uint64_t, 12, 3)
> +
> +#define LOOKUP_FUNC(suffix, type, bulk_prefetch, nh_sz, is_vrf)
> \
> +static inline void dir24_8_lookup_bulk_##suffix(void *p,
> \
> + const uint16_t *vrf_ids, const uint32_t *ips,
> \
> + uint64_t *next_hops, const unsigned int n)
> \
> +{
> \
> + struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p;
> \
> + uint64_t tmp;
> \
> + uint32_t i;
> \
> + uint32_t prefetch_offset = RTE_MIN((unsigned int)bulk_prefetch, n);
> \
> +
> \
> + if (!is_vrf)
> \
> + RTE_SET_USED(vrf_ids);
> \
> +
> \
> + for (i = 0; i < prefetch_offset; i++) {
> \
> + uint16_t vid = is_vrf ? vrf_ids[i] : 0;
> \
> + RTE_ASSERT(vid < dp->num_vrfs);
> \
> + rte_prefetch0(get_tbl24_p(dp, vid, ips[i], nh_sz));
> \
> + }
> \
> + for (i = 0; i < (n - prefetch_offset); i++) {
> \
> + uint16_t vid = is_vrf ? vrf_ids[i] : 0;
> \
> + uint16_t vid_next = is_vrf ? vrf_ids[i + prefetch_offset] : 0;
> \
> + RTE_ASSERT(vid < dp->num_vrfs);
> \
> + RTE_ASSERT(vid_next < dp->num_vrfs);
> \
> + rte_prefetch0(get_tbl24_p(dp, vid_next,
> \
> + ips[i + prefetch_offset], nh_sz));
> \
> + tmp = ((type *)dp->tbl24)[get_tbl24_idx(vid, ips[i])];
> \
> + if (unlikely(is_entry_extended(tmp)))
> \
> + tmp = ((type *)dp->tbl8)[(uint8_t)ips[i] +
> \
> + ((tmp >> 1) * DIR24_8_TBL8_GRP_NUM_ENT)];
> \
> + next_hops[i] = tmp >> 1;
> \
> + }
> \
> + for (; i < n; i++) {
> \
> + uint16_t vid = is_vrf ? vrf_ids[i] : 0;
> \
> + RTE_ASSERT(vid < dp->num_vrfs); \
> + tmp = ((type *)dp->tbl24)[get_tbl24_idx(vid, ips[i])];
> \
> + if (unlikely(is_entry_extended(tmp)))
> \
> + tmp = ((type *)dp->tbl8)[(uint8_t)ips[i] +
> \
> + ((tmp >> 1) * DIR24_8_TBL8_GRP_NUM_ENT)];
> \
> + next_hops[i] = tmp >> 1;
> \
> + }
> \
> +}
> +
> +LOOKUP_FUNC(1b, uint8_t, 5, 0, false)
> +LOOKUP_FUNC(2b, uint16_t, 6, 1, false)
> +LOOKUP_FUNC(4b, uint32_t, 15, 2, false)
> +LOOKUP_FUNC(8b, uint64_t, 12, 3, false)
> +LOOKUP_FUNC(vrf_1b, uint8_t, 5, 0, true)
> +LOOKUP_FUNC(vrf_2b, uint16_t, 6, 1, true)
> +LOOKUP_FUNC(vrf_4b, uint32_t, 15, 2, true)
> +LOOKUP_FUNC(vrf_8b, uint64_t, 12, 3, true)
>
> static inline void
> -dir24_8_lookup_bulk(struct dir24_8_tbl *dp, const uint32_t *ips,
> - uint64_t *next_hops, const unsigned int n, uint8_t nh_sz)
> +__dir24_8_lookup_bulk(struct dir24_8_tbl *dp, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n,
> + uint8_t nh_sz, bool is_vrf)
> {
> uint64_t tmp;
> uint32_t i;
> uint32_t prefetch_offset = RTE_MIN(15U, n);
>
> - for (i = 0; i < prefetch_offset; i++)
> - rte_prefetch0(get_tbl24_p(dp, ips[i], nh_sz));
> + if (!is_vrf)
> + RTE_SET_USED(vrf_ids);
> +
> + for (i = 0; i < prefetch_offset; i++) {
> + uint16_t vid = is_vrf ? vrf_ids[i] : 0;
> + RTE_ASSERT(vid < dp->num_vrfs);
> + rte_prefetch0(get_tbl24_p(dp, vid, ips[i], nh_sz));
> + }
> for (i = 0; i < (n - prefetch_offset); i++) {
> - rte_prefetch0(get_tbl24_p(dp, ips[i + prefetch_offset],
> - nh_sz));
> - tmp = get_tbl24(dp, ips[i], nh_sz);
> + uint16_t vid = is_vrf ? vrf_ids[i] : 0;
> + uint16_t vid_next = is_vrf ? vrf_ids[i + prefetch_offset] : 0;
> + RTE_ASSERT(vid < dp->num_vrfs);
> + RTE_ASSERT(vid_next < dp->num_vrfs);
> + rte_prefetch0(get_tbl24_p(dp, vid_next,
> + ips[i + prefetch_offset], nh_sz));
> + tmp = get_tbl24(dp, vid, ips[i], nh_sz);
> if (unlikely(is_entry_extended(tmp)))
> tmp = get_tbl8(dp, tmp, ips[i], nh_sz);
>
> next_hops[i] = tmp >> 1;
> }
> for (; i < n; i++) {
> - tmp = get_tbl24(dp, ips[i], nh_sz);
> + uint16_t vid = is_vrf ? vrf_ids[i] : 0;
> + RTE_ASSERT(vid < dp->num_vrfs);
> + tmp = get_tbl24(dp, vid, ips[i], nh_sz);
> if (unlikely(is_entry_extended(tmp)))
> tmp = get_tbl8(dp, tmp, ips[i], nh_sz);
>
> @@ -179,43 +213,79 @@ dir24_8_lookup_bulk(struct dir24_8_tbl *dp, const
> uint32_t *ips,
> }
>
> static inline void
> -dir24_8_lookup_bulk_0(void *p, const uint32_t *ips,
> +dir24_8_lookup_bulk_0(void *p, const uint16_t *vrf_ids, const uint32_t *ips,
> uint64_t *next_hops, const unsigned int n)
> {
> struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p;
>
> - dir24_8_lookup_bulk(dp, ips, next_hops, n, 0);
> + __dir24_8_lookup_bulk(dp, vrf_ids, ips, next_hops, n, 0, false);
> +}
> +
> +static inline void
> +dir24_8_lookup_bulk_vrf_0(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n)
> +{
> + struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p;
> +
> + __dir24_8_lookup_bulk(dp, vrf_ids, ips, next_hops, n, 0, true);
> }
>
> static inline void
> -dir24_8_lookup_bulk_1(void *p, const uint32_t *ips,
> +dir24_8_lookup_bulk_1(void *p, const uint16_t *vrf_ids, const uint32_t *ips,
> uint64_t *next_hops, const unsigned int n)
> {
> struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p;
>
> - dir24_8_lookup_bulk(dp, ips, next_hops, n, 1);
> + __dir24_8_lookup_bulk(dp, vrf_ids, ips, next_hops, n, 1, false);
> }
>
> static inline void
> -dir24_8_lookup_bulk_2(void *p, const uint32_t *ips,
> +dir24_8_lookup_bulk_vrf_1(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n)
> +{
> + struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p;
> +
> + __dir24_8_lookup_bulk(dp, vrf_ids, ips, next_hops, n, 1, true);
> +}
> +
> +static inline void
> +dir24_8_lookup_bulk_2(void *p, const uint16_t *vrf_ids, const uint32_t *ips,
> uint64_t *next_hops, const unsigned int n)
> {
> struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p;
>
> - dir24_8_lookup_bulk(dp, ips, next_hops, n, 2);
> + __dir24_8_lookup_bulk(dp, vrf_ids, ips, next_hops, n, 2, false);
> }
>
> static inline void
> -dir24_8_lookup_bulk_3(void *p, const uint32_t *ips,
> +dir24_8_lookup_bulk_vrf_2(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n)
> +{
> + struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p;
> +
> + __dir24_8_lookup_bulk(dp, vrf_ids, ips, next_hops, n, 2, true);
> +}
> +
> +static inline void
> +dir24_8_lookup_bulk_3(void *p, const uint16_t *vrf_ids, const uint32_t *ips,
> uint64_t *next_hops, const unsigned int n)
> {
> struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p;
>
> - dir24_8_lookup_bulk(dp, ips, next_hops, n, 3);
> + __dir24_8_lookup_bulk(dp, vrf_ids, ips, next_hops, n, 3, false);
> }
>
> static inline void
> -dir24_8_lookup_bulk_uni(void *p, const uint32_t *ips,
> +dir24_8_lookup_bulk_vrf_3(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n)
> +{
> + struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p;
> +
> + __dir24_8_lookup_bulk(dp, vrf_ids, ips, next_hops, n, 3, true);
> +}
> +
> +static inline void
> +dir24_8_lookup_bulk_uni(void *p, const uint16_t *vrf_ids, const uint32_t
> *ips,
> uint64_t *next_hops, const unsigned int n)
> {
> struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p;
> @@ -224,66 +294,83 @@ dir24_8_lookup_bulk_uni(void *p, const uint32_t *ips,
> uint32_t prefetch_offset = RTE_MIN(15U, n);
> uint8_t nh_sz = dp->nh_sz;
>
> - for (i = 0; i < prefetch_offset; i++)
> - rte_prefetch0(get_tbl24_p(dp, ips[i], nh_sz));
> + for (i = 0; i < prefetch_offset; i++) {
> + uint16_t vid = vrf_ids[i];
> + RTE_ASSERT(vid < dp->num_vrfs);
> + rte_prefetch0(get_tbl24_p(dp, vid, ips[i], nh_sz));
> + }
> for (i = 0; i < (n - prefetch_offset); i++) {
> - rte_prefetch0(get_tbl24_p(dp, ips[i + prefetch_offset],
> - nh_sz));
> - tmp = get_tbl24(dp, ips[i], nh_sz);
> + uint16_t vid = vrf_ids[i];
> + uint16_t vid_next = vrf_ids[i + prefetch_offset];
> + RTE_ASSERT(vid < dp->num_vrfs);
> + RTE_ASSERT(vid_next < dp->num_vrfs);
> + rte_prefetch0(get_tbl24_p(dp, vid_next,
> + ips[i + prefetch_offset], nh_sz));
> + tmp = get_tbl24(dp, vid, ips[i], nh_sz);
> if (unlikely(is_entry_extended(tmp)))
> tmp = get_tbl8(dp, tmp, ips[i], nh_sz);
>
> next_hops[i] = tmp >> 1;
> }
> for (; i < n; i++) {
> - tmp = get_tbl24(dp, ips[i], nh_sz);
> + uint16_t vid = vrf_ids[i];
> + RTE_ASSERT(vid < dp->num_vrfs);
> + tmp = get_tbl24(dp, vid, ips[i], nh_sz);
> if (unlikely(is_entry_extended(tmp)))
> tmp = get_tbl8(dp, tmp, ips[i], nh_sz);
>
> next_hops[i] = tmp >> 1;
> }
> }
> -
> #define BSWAP_MAX_LENGTH 64
>
> -typedef void (*dir24_8_lookup_bulk_be_cb)(void *p, const uint32_t *ips,
> uint64_t *next_hops,
> - const unsigned int n);
> +typedef void (*dir24_8_lookup_bulk_be_cb)(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
>
> static inline void
> -dir24_8_lookup_bulk_be(void *p, const uint32_t *ips, uint64_t *next_hops,
> const unsigned int n,
> - dir24_8_lookup_bulk_be_cb cb)
> +dir24_8_lookup_bulk_be(void *p, const uint16_t *vrf_ids, const uint32_t *ips,
> + uint64_t *next_hops, const unsigned int n, dir24_8_lookup_bulk_be_cb
> cb)
> {
> uint32_t le_ips[BSWAP_MAX_LENGTH];
> unsigned int i;
>
> #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
> - cb(p, ips, next_hops, n);
> + cb(p, vrf_ids, ips, next_hops, n);
> #else
> for (i = 0; i < n; i += BSWAP_MAX_LENGTH) {
> int j;
> for (j = 0; j < BSWAP_MAX_LENGTH && i + j < n; j++)
> le_ips[j] = rte_be_to_cpu_32(ips[i + j]);
>
> - cb(p, le_ips, next_hops + i, j);
> + cb(p, vrf_ids + i, le_ips, next_hops + i, j);
> }
> #endif
> }
>
> #define DECLARE_BE_LOOKUP_FN(name) \
> static inline void \
> -name##_be(void *p, const uint32_t *ips, uint64_t *next_hops, const unsigned
> int n) \
> +name##_be(void *p, const uint16_t *vrf_ids, const uint32_t *ips, \
> + uint64_t *next_hops, const unsigned int n) \
> { \
> - dir24_8_lookup_bulk_be(p, ips, next_hops, n, name); \
> + dir24_8_lookup_bulk_be(p, vrf_ids, ips, next_hops, n, name); \
> }
>
> DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_1b)
> DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_2b)
> DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_4b)
> DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_8b)
> +DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_vrf_1b)
> +DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_vrf_2b)
> +DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_vrf_4b)
> +DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_vrf_8b)
> DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_0)
> DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_1)
> DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_2)
> DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_3)
> +DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_vrf_0)
> +DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_vrf_1)
> +DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_vrf_2)
> +DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_vrf_3)
> DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_uni)
>
> void *
> @@ -296,7 +383,7 @@ rte_fib_lookup_fn_t
> dir24_8_get_lookup_fn(void *p, enum rte_fib_lookup_type type, bool be_addr);
>
> int
> -dir24_8_modify(struct rte_fib *fib, uint32_t ip, uint8_t depth,
> +dir24_8_modify(struct rte_fib *fib, uint16_t vrf_id, uint32_t ip, uint8_t
> depth,
> uint64_t next_hop, int op);
>
> int
> diff --git a/lib/fib/dir24_8_avx512.c b/lib/fib/dir24_8_avx512.c
> index 89b43583c7..3e576e410e 100644
> --- a/lib/fib/dir24_8_avx512.c
> +++ b/lib/fib/dir24_8_avx512.c
> @@ -4,75 +4,132 @@
>
> #include <rte_vect.h>
> #include <rte_fib.h>
> +#include <rte_debug.h>
>
> #include "dir24_8.h"
> #include "dir24_8_avx512.h"
>
> +enum vrf_scale {
> + VRF_SCALE_SINGLE = 0,
> + VRF_SCALE_SMALL = 1,
> + VRF_SCALE_LARGE = 2,
> +};
> +
> static __rte_always_inline void
> -dir24_8_vec_lookup_x16(void *p, const uint32_t *ips,
> - uint64_t *next_hops, int size, bool be_addr)
> +dir24_8_vec_lookup_x8_64b_path(struct dir24_8_tbl *dp, __m256i ip_vec_256,
> + __m256i vrf32_256, uint64_t *next_hops, int size)
> {
> - struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p;
> - __mmask16 msk_ext;
> - __mmask16 exp_msk = 0x5555;
> - __m512i ip_vec, idxes, res, bytes;
> - const __m512i zero = _mm512_set1_epi32(0);
> - const __m512i lsb = _mm512_set1_epi32(1);
> - const __m512i lsbyte_msk = _mm512_set1_epi32(0xff);
> - __m512i tmp1, tmp2, res_msk;
> - __m256i tmp256;
> - /* used to mask gather values if size is 1/2 (8/16 bit next hops) */
> + const __m512i zero_64 = _mm512_set1_epi64(0);
> + const __m512i lsb_64 = _mm512_set1_epi64(1);
> + const __m512i lsbyte_msk_64 = _mm512_set1_epi64(0xff);
> + __m512i res_msk_64, vrf64, idxes_64, res, bytes_64;
> + __mmask8 msk_ext_64;
> +
> if (size == sizeof(uint8_t))
> - res_msk = _mm512_set1_epi32(UINT8_MAX);
> + res_msk_64 = _mm512_set1_epi64(UINT8_MAX);
> else if (size == sizeof(uint16_t))
> - res_msk = _mm512_set1_epi32(UINT16_MAX);
> + res_msk_64 = _mm512_set1_epi64(UINT16_MAX);
> + else if (size == sizeof(uint32_t))
> + res_msk_64 = _mm512_set1_epi64(UINT32_MAX);
>
> - ip_vec = _mm512_loadu_si512(ips);
> - if (be_addr) {
> - const __m512i bswap32 = _mm512_set_epi32(
> - 0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203,
> - 0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203,
> - 0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203,
> - 0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203
> - );
> - ip_vec = _mm512_shuffle_epi8(ip_vec, bswap32);
> + vrf64 = _mm512_cvtepu32_epi64(vrf32_256);
> +
> + /* Compute index: (vrf_id << 24) + (ip >> 8) using 64-bit shift */
> + idxes_64 = _mm512_slli_epi64(vrf64, 24);
> + idxes_64 = _mm512_add_epi64(idxes_64, _mm512_cvtepu32_epi64(
> + _mm256_srli_epi32(ip_vec_256, 8)));
> +
> + /* lookup in tbl24 */
> + if (size == sizeof(uint8_t)) {
> + res = _mm512_i64gather_epi64(idxes_64, (const void *)dp-
> >tbl24, 1);
> + res = _mm512_and_epi64(res, res_msk_64);
> + } else if (size == sizeof(uint16_t)) {
> + res = _mm512_i64gather_epi64(idxes_64, (const void *)dp-
> >tbl24, 2);
> + res = _mm512_and_epi64(res, res_msk_64);
> + } else {
> + res = _mm512_i64gather_epi64(idxes_64, (const void *)dp-
> >tbl24, 4);
> + res = _mm512_and_epi64(res, res_msk_64);
> + }
> +
> + /* get extended entries indexes */
> + msk_ext_64 = _mm512_test_epi64_mask(res, lsb_64);
> +
> + if (msk_ext_64 != 0) {
> + bytes_64 = _mm512_cvtepu32_epi64(ip_vec_256);
> + idxes_64 = _mm512_srli_epi64(res, 1);
> + idxes_64 = _mm512_slli_epi64(idxes_64, 8);
> + bytes_64 = _mm512_and_epi64(bytes_64, lsbyte_msk_64);
> + idxes_64 = _mm512_maskz_add_epi64(msk_ext_64, idxes_64,
> bytes_64);
> +
> + if (size == sizeof(uint8_t))
> + idxes_64 = _mm512_mask_i64gather_epi64(zero_64,
> msk_ext_64,
> + idxes_64, (const void *)dp->tbl8, 1);
> + else if (size == sizeof(uint16_t))
> + idxes_64 = _mm512_mask_i64gather_epi64(zero_64,
> msk_ext_64,
> + idxes_64, (const void *)dp->tbl8, 2);
> + else
> + idxes_64 = _mm512_mask_i64gather_epi64(zero_64,
> msk_ext_64,
> + idxes_64, (const void *)dp->tbl8, 4);
> +
> + res = _mm512_mask_blend_epi64(msk_ext_64, res, idxes_64);
> }
>
> - /* mask 24 most significant bits */
> - idxes = _mm512_srli_epi32(ip_vec, 8);
> + res = _mm512_srli_epi64(res, 1);
> + _mm512_storeu_si512(next_hops, res);
> +}
> +
> +static __rte_always_inline void
> +dir24_8_vec_lookup_x16_32b_path(struct dir24_8_tbl *dp, __m512i ip_vec,
> + __m512i idxes, uint64_t *next_hops, int size)
> +{
> + __mmask16 msk_ext;
> + const __mmask16 exp_msk = 0x5555;
> + const __m512i zero_32 = _mm512_set1_epi32(0);
> + const __m512i lsb_32 = _mm512_set1_epi32(1);
> + const __m512i lsbyte_msk_32 = _mm512_set1_epi32(0xff);
> + __m512i res, bytes, tmp1, tmp2;
> + __m256i tmp256;
> + __m512i res_msk_32;
> +
> + if (size == sizeof(uint8_t))
> + res_msk_32 = _mm512_set1_epi32(UINT8_MAX);
> + else if (size == sizeof(uint16_t))
> + res_msk_32 = _mm512_set1_epi32(UINT16_MAX);
>
> - /**
> + /*
> * lookup in tbl24
> * Put it inside branch to make compiler happy with -O0
> */
> if (size == sizeof(uint8_t)) {
> res = _mm512_i32gather_epi32(idxes, (const int *)dp->tbl24, 1);
> - res = _mm512_and_epi32(res, res_msk);
> + res = _mm512_and_epi32(res, res_msk_32);
> } else if (size == sizeof(uint16_t)) {
> res = _mm512_i32gather_epi32(idxes, (const int *)dp->tbl24, 2);
> - res = _mm512_and_epi32(res, res_msk);
> - } else
> + res = _mm512_and_epi32(res, res_msk_32);
> + } else {
> res = _mm512_i32gather_epi32(idxes, (const int *)dp->tbl24, 4);
> + }
>
> /* get extended entries indexes */
> - msk_ext = _mm512_test_epi32_mask(res, lsb);
> + msk_ext = _mm512_test_epi32_mask(res, lsb_32);
>
> if (msk_ext != 0) {
> idxes = _mm512_srli_epi32(res, 1);
> idxes = _mm512_slli_epi32(idxes, 8);
> - bytes = _mm512_and_epi32(ip_vec, lsbyte_msk);
> + bytes = _mm512_and_epi32(ip_vec, lsbyte_msk_32);
> idxes = _mm512_maskz_add_epi32(msk_ext, idxes, bytes);
> if (size == sizeof(uint8_t)) {
> - idxes = _mm512_mask_i32gather_epi32(zero, msk_ext,
> + idxes = _mm512_mask_i32gather_epi32(zero_32,
> msk_ext,
> idxes, (const int *)dp->tbl8, 1);
> - idxes = _mm512_and_epi32(idxes, res_msk);
> + idxes = _mm512_and_epi32(idxes, res_msk_32);
> } else if (size == sizeof(uint16_t)) {
> - idxes = _mm512_mask_i32gather_epi32(zero, msk_ext,
> + idxes = _mm512_mask_i32gather_epi32(zero_32,
> msk_ext,
> idxes, (const int *)dp->tbl8, 2);
> - idxes = _mm512_and_epi32(idxes, res_msk);
> - } else
> - idxes = _mm512_mask_i32gather_epi32(zero, msk_ext,
> + idxes = _mm512_and_epi32(idxes, res_msk_32);
> + } else {
> + idxes = _mm512_mask_i32gather_epi32(zero_32,
> msk_ext,
> idxes, (const int *)dp->tbl8, 4);
> + }
>
> res = _mm512_mask_blend_epi32(msk_ext, res, idxes);
> }
> @@ -86,16 +143,74 @@ dir24_8_vec_lookup_x16(void *p, const uint32_t *ips,
> _mm512_storeu_si512(next_hops + 8, tmp2);
> }
>
> +/* Unified function with vrf_scale parameter similar to be_addr */
> +static __rte_always_inline void
> +dir24_8_vec_lookup_x16(void *p, const uint16_t *vrf_ids, const uint32_t *ips,
> + uint64_t *next_hops, int size, bool be_addr, enum vrf_scale vrf_scale)
> +{
> + struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p;
> + __m512i ip_vec, idxes;
> + __m256i ip_vec_256, vrf32_256;
> +
> + ip_vec = _mm512_loadu_si512(ips);
> + if (be_addr) {
> + const __m512i bswap32 = _mm512_set_epi32(
> + 0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203,
> + 0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203,
> + 0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203,
> + 0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203
> + );
> + ip_vec = _mm512_shuffle_epi8(ip_vec, bswap32);
> + }
> +
> + if (vrf_scale == VRF_SCALE_SINGLE) {
> + /* mask 24 most significant bits */
> + idxes = _mm512_srli_epi32(ip_vec, 8);
> + dir24_8_vec_lookup_x16_32b_path(dp, ip_vec, idxes, next_hops,
> size);
> + } else if (vrf_scale == VRF_SCALE_SMALL) {
> + /* For < 256 VRFs: use 32-bit indices with 32-bit shift */
> + __m512i vrf32;
> + uint32_t i;
> +
> + for (i = 0; i < 16; i++)
> + RTE_ASSERT(vrf_ids[i] < dp->num_vrfs);
> +
> + vrf32 = _mm512_cvtepu16_epi32(_mm256_loadu_si256((const
> void *)vrf_ids));
> +
> + /* mask 24 most significant bits */
> + idxes = _mm512_srli_epi32(ip_vec, 8);
> + idxes = _mm512_add_epi32(idxes, _mm512_slli_epi32(vrf32,
> 24));
> + dir24_8_vec_lookup_x16_32b_path(dp, ip_vec, idxes, next_hops,
> size);
> + } else {
> + /* For >= 256 VRFs: use 64-bit indices to avoid overflow */
> + uint32_t i;
> +
> + for (i = 0; i < 16; i++)
> + RTE_ASSERT(vrf_ids[i] < dp->num_vrfs);
> +
> + /* Extract first 8 IPs and VRF IDs */
> + ip_vec_256 = _mm512_castsi512_si256(ip_vec);
> + vrf32_256 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const
> void *)vrf_ids));
> + dir24_8_vec_lookup_x8_64b_path(dp, ip_vec_256, vrf32_256,
> next_hops, size);
> +
> + /* Process next 8 IPs from the second half of the vector */
> + ip_vec_256 = _mm512_extracti32x8_epi32(ip_vec, 1);
> + vrf32_256 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const
> void *)(vrf_ids + 8)));
> + dir24_8_vec_lookup_x8_64b_path(dp, ip_vec_256, vrf32_256,
> next_hops + 8, size);
> + }
> +}
> +
> +/* Unified function with vrf_scale parameter */
> static __rte_always_inline void
> -dir24_8_vec_lookup_x8_8b(void *p, const uint32_t *ips,
> - uint64_t *next_hops, bool be_addr)
> +dir24_8_vec_lookup_x8_8b(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, bool be_addr, enum vrf_scale
> vrf_scale)
> {
> struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p;
> - const __m512i zero = _mm512_set1_epi32(0);
> - const __m512i lsbyte_msk = _mm512_set1_epi64(0xff);
> - const __m512i lsb = _mm512_set1_epi64(1);
> + const __m512i zero_64 = _mm512_set1_epi64(0);
> + const __m512i lsbyte_msk_64 = _mm512_set1_epi64(0xff);
> + const __m512i lsb_64 = _mm512_set1_epi64(1);
> __m512i res, idxes, bytes;
> - __m256i idxes_256, ip_vec;
> + __m256i ip_vec, vrf32_256;
> __mmask8 msk_ext;
>
> ip_vec = _mm256_loadu_si256((const void *)ips);
> @@ -106,66 +221,207 @@ dir24_8_vec_lookup_x8_8b(void *p, const uint32_t
> *ips,
> );
> ip_vec = _mm256_shuffle_epi8(ip_vec, bswap32);
> }
> - /* mask 24 most significant bits */
> - idxes_256 = _mm256_srli_epi32(ip_vec, 8);
>
> - /* lookup in tbl24 */
> - res = _mm512_i32gather_epi64(idxes_256, (const void *)dp->tbl24, 8);
> + if (vrf_scale == VRF_SCALE_SINGLE) {
> + /* For single VRF: use 32-bit indices without vrf_ids */
> + __m256i idxes_256;
>
> - /* get extended entries indexes */
> - msk_ext = _mm512_test_epi64_mask(res, lsb);
> + /* mask 24 most significant bits */
> + idxes_256 = _mm256_srli_epi32(ip_vec, 8);
>
> - if (msk_ext != 0) {
> - bytes = _mm512_cvtepi32_epi64(ip_vec);
> - idxes = _mm512_srli_epi64(res, 1);
> - idxes = _mm512_slli_epi64(idxes, 8);
> - bytes = _mm512_and_epi64(bytes, lsbyte_msk);
> - idxes = _mm512_maskz_add_epi64(msk_ext, idxes, bytes);
> - idxes = _mm512_mask_i64gather_epi64(zero, msk_ext, idxes,
> - (const void *)dp->tbl8, 8);
> -
> - res = _mm512_mask_blend_epi64(msk_ext, res, idxes);
> - }
> + /* lookup in tbl24 */
> + res = _mm512_i32gather_epi64(idxes_256, (const void *)dp-
> >tbl24, 8);
>
> - res = _mm512_srli_epi64(res, 1);
> - _mm512_storeu_si512(next_hops, res);
> + /* get extended entries indexes */
> + msk_ext = _mm512_test_epi64_mask(res, lsb_64);
> +
> + if (msk_ext != 0) {
> + bytes = _mm512_cvtepu32_epi64(ip_vec);
> + idxes = _mm512_srli_epi64(res, 1);
> + idxes = _mm512_slli_epi64(idxes, 8);
> + bytes = _mm512_and_epi64(bytes, lsbyte_msk_64);
> + idxes = _mm512_maskz_add_epi64(msk_ext, idxes,
> bytes);
> + idxes = _mm512_mask_i64gather_epi64(zero_64,
> msk_ext, idxes,
> + (const void *)dp->tbl8, 8);
> +
> + res = _mm512_mask_blend_epi64(msk_ext, res, idxes);
> + }
> +
> + res = _mm512_srli_epi64(res, 1);
> + _mm512_storeu_si512(next_hops, res);
> + } else if (vrf_scale == VRF_SCALE_SMALL) {
> + /* For < 256 VRFs: use 32-bit indices with 32-bit shift */
> + __m256i idxes_256;
> + uint32_t i;
> +
> + for (i = 0; i < 8; i++)
> + RTE_ASSERT(vrf_ids[i] < dp->num_vrfs);
> +
> + /* mask 24 most significant bits */
> + idxes_256 = _mm256_srli_epi32(ip_vec, 8);
> + vrf32_256 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const
> void *)vrf_ids));
> + idxes_256 = _mm256_add_epi32(idxes_256,
> _mm256_slli_epi32(vrf32_256, 24));
> +
> + /* lookup in tbl24 */
> + res = _mm512_i32gather_epi64(idxes_256, (const void *)dp-
> >tbl24, 8);
> +
> + /* get extended entries indexes */
> + msk_ext = _mm512_test_epi64_mask(res, lsb_64);
> +
> + if (msk_ext != 0) {
> + bytes = _mm512_cvtepu32_epi64(ip_vec);
> + idxes = _mm512_srli_epi64(res, 1);
> + idxes = _mm512_slli_epi64(idxes, 8);
> + bytes = _mm512_and_epi64(bytes, lsbyte_msk_64);
> + idxes = _mm512_maskz_add_epi64(msk_ext, idxes,
> bytes);
> + idxes = _mm512_mask_i64gather_epi64(zero_64,
> msk_ext, idxes,
> + (const void *)dp->tbl8, 8);
> +
> + res = _mm512_mask_blend_epi64(msk_ext, res, idxes);
> + }
> +
> + res = _mm512_srli_epi64(res, 1);
> + _mm512_storeu_si512(next_hops, res);
> + } else {
> + /* For >= 256 VRFs: use 64-bit indices to avoid overflow */
> + uint32_t i;
> +
> + for (i = 0; i < 8; i++)
> + RTE_ASSERT(vrf_ids[i] < dp->num_vrfs);
> +
> + vrf32_256 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const
> void *)vrf_ids));
> + __m512i vrf64 = _mm512_cvtepu32_epi64(vrf32_256);
> +
> + /* Compute index: (vrf_id << 24) + (ip >> 8) using 64-bit
> arithmetic */
> + idxes = _mm512_slli_epi64(vrf64, 24);
> + idxes = _mm512_add_epi64(idxes, _mm512_cvtepu32_epi64(
> + _mm256_srli_epi32(ip_vec, 8)));
> +
> + /* lookup in tbl24 with 64-bit gather */
> + res = _mm512_i64gather_epi64(idxes, (const void *)dp->tbl24, 8);
> +
> + /* get extended entries indexes */
> + msk_ext = _mm512_test_epi64_mask(res, lsb_64);
> +
> + if (msk_ext != 0) {
> + bytes = _mm512_cvtepu32_epi64(ip_vec);
> + idxes = _mm512_srli_epi64(res, 1);
> + idxes = _mm512_slli_epi64(idxes, 8);
> + bytes = _mm512_and_epi64(bytes, lsbyte_msk_64);
> + idxes = _mm512_maskz_add_epi64(msk_ext, idxes,
> bytes);
> + idxes = _mm512_mask_i64gather_epi64(zero_64,
> msk_ext, idxes,
> + (const void *)dp->tbl8, 8);
> +
> + res = _mm512_mask_blend_epi64(msk_ext, res, idxes);
> + }
> +
> + res = _mm512_srli_epi64(res, 1);
> + _mm512_storeu_si512(next_hops, res);
> + }
> }
>
> -#define DECLARE_VECTOR_FN(suffix, nh_type, be_addr) \
> +#define DECLARE_VECTOR_FN(suffix, scalar_suffix, nh_type, be_addr, vrf_scale)
> \
> void \
> -rte_dir24_8_vec_lookup_bulk_##suffix(void *p, const uint32_t *ips, uint64_t
> *next_hops, \
> - const unsigned int n) \
> +rte_dir24_8_vec_lookup_bulk_##suffix(void *p, const uint16_t *vrf_ids, \
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n) \
> { \
> uint32_t i; \
> for (i = 0; i < (n / 16); i++) \
> - dir24_8_vec_lookup_x16(p, ips + i * 16, next_hops + i * 16,
> sizeof(nh_type), \
> - be_addr); \
> - dir24_8_lookup_bulk_##suffix(p, ips + i * 16, next_hops + i * 16, n - i
> *
> 16); \
> + dir24_8_vec_lookup_x16(p, vrf_ids + i * 16, ips + i * 16, \
> + next_hops + i * 16, sizeof(nh_type), be_addr,
> vrf_scale); \
> + dir24_8_lookup_bulk_##scalar_suffix(p, vrf_ids + i * 16, ips + i * 16, \
> + next_hops + i * 16, n - i * 16); \
> +}
> +
> +DECLARE_VECTOR_FN(1b, 1b, uint8_t, false, VRF_SCALE_SINGLE)
> +DECLARE_VECTOR_FN(1b_be, 1b_be, uint8_t, true, VRF_SCALE_SINGLE)
> +DECLARE_VECTOR_FN(2b, 2b, uint16_t, false, VRF_SCALE_SINGLE)
> +DECLARE_VECTOR_FN(2b_be, 2b_be, uint16_t, true, VRF_SCALE_SINGLE)
> +DECLARE_VECTOR_FN(4b, 4b, uint32_t, false, VRF_SCALE_SINGLE)
> +DECLARE_VECTOR_FN(4b_be, 4b_be, uint32_t, true, VRF_SCALE_SINGLE)
> +
> +DECLARE_VECTOR_FN(vrf_1b, vrf_1b, uint8_t, false, VRF_SCALE_SMALL)
> +DECLARE_VECTOR_FN(vrf_1b_be, vrf_1b_be, uint8_t, true, VRF_SCALE_SMALL)
> +DECLARE_VECTOR_FN(vrf_2b, vrf_2b, uint16_t, false, VRF_SCALE_SMALL)
> +DECLARE_VECTOR_FN(vrf_2b_be, vrf_2b_be, uint16_t, true, VRF_SCALE_SMALL)
> +DECLARE_VECTOR_FN(vrf_4b, vrf_4b, uint32_t, false, VRF_SCALE_SMALL)
> +DECLARE_VECTOR_FN(vrf_4b_be, vrf_4b_be, uint32_t, true, VRF_SCALE_SMALL)
> +
> +DECLARE_VECTOR_FN(vrf_1b_large, vrf_1b, uint8_t, false, VRF_SCALE_LARGE)
> +DECLARE_VECTOR_FN(vrf_1b_be_large, vrf_1b_be, uint8_t, true,
> VRF_SCALE_LARGE)
> +DECLARE_VECTOR_FN(vrf_2b_large, vrf_2b, uint16_t, false, VRF_SCALE_LARGE)
> +DECLARE_VECTOR_FN(vrf_2b_be_large, vrf_2b_be, uint16_t, true,
> VRF_SCALE_LARGE)
> +DECLARE_VECTOR_FN(vrf_4b_large, vrf_4b, uint32_t, false, VRF_SCALE_LARGE)
> +DECLARE_VECTOR_FN(vrf_4b_be_large, vrf_4b_be, uint32_t, true,
> VRF_SCALE_LARGE)
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_8b(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n)
> +{
> + uint32_t i;
> + for (i = 0; i < (n / 8); i++)
> + dir24_8_vec_lookup_x8_8b(p, vrf_ids + i * 8, ips + i * 8,
> + next_hops + i * 8, false, VRF_SCALE_SINGLE);
> + dir24_8_lookup_bulk_8b(p, vrf_ids + i * 8, ips + i * 8,
> + next_hops + i * 8, n - i * 8);
> +}
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_8b_be(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n)
> +{
> + uint32_t i;
> + for (i = 0; i < (n / 8); i++)
> + dir24_8_vec_lookup_x8_8b(p, vrf_ids + i * 8, ips + i * 8,
> + next_hops + i * 8, true, VRF_SCALE_SINGLE);
> + dir24_8_lookup_bulk_8b_be(p, vrf_ids + i * 8, ips + i * 8,
> + next_hops + i * 8, n - i * 8);
> +}
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_vrf_8b(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n)
> +{
> + uint32_t i;
> + for (i = 0; i < (n / 8); i++)
> + dir24_8_vec_lookup_x8_8b(p, vrf_ids + i * 8, ips + i * 8,
> + next_hops + i * 8, false, VRF_SCALE_SMALL);
> + dir24_8_lookup_bulk_vrf_8b(p, vrf_ids + i * 8, ips + i * 8,
> + next_hops + i * 8, n - i * 8);
> }
>
> -DECLARE_VECTOR_FN(1b, uint8_t, false)
> -DECLARE_VECTOR_FN(1b_be, uint8_t, true)
> -DECLARE_VECTOR_FN(2b, uint16_t, false)
> -DECLARE_VECTOR_FN(2b_be, uint16_t, true)
> -DECLARE_VECTOR_FN(4b, uint32_t, false)
> -DECLARE_VECTOR_FN(4b_be, uint32_t, true)
> +void
> +rte_dir24_8_vec_lookup_bulk_vrf_8b_be(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n)
> +{
> + uint32_t i;
> + for (i = 0; i < (n / 8); i++)
> + dir24_8_vec_lookup_x8_8b(p, vrf_ids + i * 8, ips + i * 8,
> + next_hops + i * 8, true, VRF_SCALE_SMALL);
> + dir24_8_lookup_bulk_vrf_8b_be(p, vrf_ids + i * 8, ips + i * 8,
> + next_hops + i * 8, n - i * 8);
> +}
>
> void
> -rte_dir24_8_vec_lookup_bulk_8b(void *p, const uint32_t *ips,
> - uint64_t *next_hops, const unsigned int n)
> +rte_dir24_8_vec_lookup_bulk_vrf_8b_large(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n)
> {
> uint32_t i;
> for (i = 0; i < (n / 8); i++)
> - dir24_8_vec_lookup_x8_8b(p, ips + i * 8, next_hops + i * 8,
> false);
> - dir24_8_lookup_bulk_8b(p, ips + i * 8, next_hops + i * 8, n - i * 8);
> + dir24_8_vec_lookup_x8_8b(p, vrf_ids + i * 8, ips + i * 8,
> + next_hops + i * 8, false, VRF_SCALE_LARGE);
> + dir24_8_lookup_bulk_vrf_8b(p, vrf_ids + i * 8, ips + i * 8,
> + next_hops + i * 8, n - i * 8);
> }
>
> void
> -rte_dir24_8_vec_lookup_bulk_8b_be(void *p, const uint32_t *ips,
> - uint64_t *next_hops, const unsigned int n)
> +rte_dir24_8_vec_lookup_bulk_vrf_8b_be_large(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n)
> {
> uint32_t i;
> for (i = 0; i < (n / 8); i++)
> - dir24_8_vec_lookup_x8_8b(p, ips + i * 8, next_hops + i * 8,
> true);
> - dir24_8_lookup_bulk_8b_be(p, ips + i * 8, next_hops + i * 8, n - i * 8);
> + dir24_8_vec_lookup_x8_8b(p, vrf_ids + i * 8, ips + i * 8,
> + next_hops + i * 8, true, VRF_SCALE_LARGE);
> + dir24_8_lookup_bulk_vrf_8b_be(p, vrf_ids + i * 8, ips + i * 8,
> + next_hops + i * 8, n - i * 8);
> }
> diff --git a/lib/fib/dir24_8_avx512.h b/lib/fib/dir24_8_avx512.h
> index 3e2bbc2490..d42ef1d17f 100644
> --- a/lib/fib/dir24_8_avx512.h
> +++ b/lib/fib/dir24_8_avx512.h
> @@ -6,35 +6,99 @@
> #define _DIR248_AVX512_H_
>
> void
> -rte_dir24_8_vec_lookup_bulk_1b(void *p, const uint32_t *ips,
> +rte_dir24_8_vec_lookup_bulk_1b(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_1b_be(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_vrf_1b(void *p, const uint16_t *vrf_ids, const
> uint32_t *ips,
> uint64_t *next_hops, const unsigned int n);
>
> void
> -rte_dir24_8_vec_lookup_bulk_2b(void *p, const uint32_t *ips,
> +rte_dir24_8_vec_lookup_bulk_vrf_1b_be(void *p, const uint16_t *vrf_ids, const
> uint32_t *ips,
> uint64_t *next_hops, const unsigned int n);
>
> void
> -rte_dir24_8_vec_lookup_bulk_4b(void *p, const uint32_t *ips,
> +rte_dir24_8_vec_lookup_bulk_vrf_1b_large(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_vrf_1b_be_large(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_2b(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_2b_be(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_vrf_2b(void *p, const uint16_t *vrf_ids, const
> uint32_t *ips,
> uint64_t *next_hops, const unsigned int n);
>
> void
> -rte_dir24_8_vec_lookup_bulk_8b(void *p, const uint32_t *ips,
> +rte_dir24_8_vec_lookup_bulk_vrf_2b_be(void *p, const uint16_t *vrf_ids, const
> uint32_t *ips,
> uint64_t *next_hops, const unsigned int n);
>
> void
> -rte_dir24_8_vec_lookup_bulk_1b_be(void *p, const uint32_t *ips,
> +rte_dir24_8_vec_lookup_bulk_vrf_2b_large(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_vrf_2b_be_large(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_4b(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_4b_be(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_vrf_4b(void *p, const uint16_t *vrf_ids, const
> uint32_t *ips,
> uint64_t *next_hops, const unsigned int n);
>
> void
> -rte_dir24_8_vec_lookup_bulk_2b_be(void *p, const uint32_t *ips,
> +rte_dir24_8_vec_lookup_bulk_vrf_4b_be(void *p, const uint16_t *vrf_ids, const
> uint32_t *ips,
> uint64_t *next_hops, const unsigned int n);
>
> void
> -rte_dir24_8_vec_lookup_bulk_4b_be(void *p, const uint32_t *ips,
> +rte_dir24_8_vec_lookup_bulk_vrf_4b_large(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_vrf_4b_be_large(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_8b(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_8b_be(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_vrf_8b(void *p, const uint16_t *vrf_ids, const
> uint32_t *ips,
> uint64_t *next_hops, const unsigned int n);
>
> void
> -rte_dir24_8_vec_lookup_bulk_8b_be(void *p, const uint32_t *ips,
> +rte_dir24_8_vec_lookup_bulk_vrf_8b_be(void *p, const uint16_t *vrf_ids, const
> uint32_t *ips,
> uint64_t *next_hops, const unsigned int n);
>
> +void
> +rte_dir24_8_vec_lookup_bulk_vrf_8b_large(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> +void
> +rte_dir24_8_vec_lookup_bulk_vrf_8b_be_large(void *p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
> +
> #endif /* _DIR248_AVX512_H_ */
> diff --git a/lib/fib/rte_fib.c b/lib/fib/rte_fib.c
> index 184210f380..efc0595a7f 100644
> --- a/lib/fib/rte_fib.c
> +++ b/lib/fib/rte_fib.c
> @@ -14,12 +14,15 @@
> #include <rte_string_fns.h>
> #include <rte_tailq.h>
>
> +#include <rte_debug.h>
> #include <rte_rib.h>
> #include <rte_fib.h>
>
> #include "dir24_8.h"
> #include "fib_log.h"
>
> +#define FIB_MAX_LOOKUP_BULK 64U
> +
> RTE_LOG_REGISTER_DEFAULT(fib_logtype, INFO);
>
> TAILQ_HEAD(rte_fib_list, rte_tailq_entry);
> @@ -40,52 +43,61 @@ EAL_REGISTER_TAILQ(rte_fib_tailq)
> struct rte_fib {
> char name[RTE_FIB_NAMESIZE];
> enum rte_fib_type type; /**< Type of FIB struct */
> - unsigned int flags; /**< Flags */
> - struct rte_rib *rib; /**< RIB helper datastructure */
> + uint16_t flags; /**< Flags */
> + uint16_t num_vrfs;/**< Number of VRFs */
> + struct rte_rib **ribs; /**< RIB helper datastructures per VRF
> */
> void *dp; /**< pointer to the dataplane struct*/
> rte_fib_lookup_fn_t lookup; /**< FIB lookup function */
> rte_fib_modify_fn_t modify; /**< modify FIB datastructure */
> - uint64_t def_nh;
> + uint64_t *def_nh;/**< Per-VRF default next hop array */
> };
>
> static void
> -dummy_lookup(void *fib_p, const uint32_t *ips, uint64_t *next_hops,
> - const unsigned int n)
> +dummy_lookup(void *fib_p, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n)
> {
> unsigned int i;
> struct rte_fib *fib = fib_p;
> struct rte_rib_node *node;
> + struct rte_rib *rib;
>
> for (i = 0; i < n; i++) {
> - node = rte_rib_lookup(fib->rib, ips[i]);
> + RTE_ASSERT(vrf_ids[i] < fib->num_vrfs);
> + rib = rte_fib_vrf_get_rib(fib, vrf_ids[i]);
> + node = rte_rib_lookup(rib, ips[i]);
> if (node != NULL)
> rte_rib_get_nh(node, &next_hops[i]);
> else
> - next_hops[i] = fib->def_nh;
> + next_hops[i] = fib->def_nh[vrf_ids[i]];
> }
> }
>
> static int
> -dummy_modify(struct rte_fib *fib, uint32_t ip, uint8_t depth,
> - uint64_t next_hop, int op)
> +dummy_modify(struct rte_fib *fib, uint16_t vrf_id, uint32_t ip,
> + uint8_t depth, uint64_t next_hop, int op)
> {
> struct rte_rib_node *node;
> + struct rte_rib *rib;
> if ((fib == NULL) || (depth > RTE_FIB_MAXDEPTH))
> return -EINVAL;
>
> - node = rte_rib_lookup_exact(fib->rib, ip, depth);
> + rib = rte_fib_vrf_get_rib(fib, vrf_id);
> + if (rib == NULL)
> + return -EINVAL;
> +
> + node = rte_rib_lookup_exact(rib, ip, depth);
>
> switch (op) {
> case RTE_FIB_ADD:
> if (node == NULL)
> - node = rte_rib_insert(fib->rib, ip, depth);
> + node = rte_rib_insert(rib, ip, depth);
> if (node == NULL)
> return -rte_errno;
> return rte_rib_set_nh(node, next_hop);
> case RTE_FIB_DEL:
> if (node == NULL)
> return -ENOENT;
> - rte_rib_remove(fib->rib, ip, depth);
> + rte_rib_remove(rib, ip, depth);
> return 0;
> }
> return -EINVAL;
> @@ -125,7 +137,7 @@ rte_fib_add(struct rte_fib *fib, uint32_t ip, uint8_t
> depth,
> uint64_t next_hop)
> if ((fib == NULL) || (fib->modify == NULL) ||
> (depth > RTE_FIB_MAXDEPTH))
> return -EINVAL;
> - return fib->modify(fib, ip, depth, next_hop, RTE_FIB_ADD);
> + return fib->modify(fib, 0, ip, depth, next_hop, RTE_FIB_ADD);
> }
>
> RTE_EXPORT_SYMBOL(rte_fib_delete)
> @@ -135,7 +147,7 @@ rte_fib_delete(struct rte_fib *fib, uint32_t ip, uint8_t
> depth)
> if ((fib == NULL) || (fib->modify == NULL) ||
> (depth > RTE_FIB_MAXDEPTH))
> return -EINVAL;
> - return fib->modify(fib, ip, depth, 0, RTE_FIB_DEL);
> + return fib->modify(fib, 0, ip, depth, 0, RTE_FIB_DEL);
> }
>
> RTE_EXPORT_SYMBOL(rte_fib_lookup_bulk)
> @@ -143,24 +155,73 @@ int
> rte_fib_lookup_bulk(struct rte_fib *fib, uint32_t *ips,
> uint64_t *next_hops, int n)
> {
> + static const uint16_t zero_vrf_ids[FIB_MAX_LOOKUP_BULK];
> + unsigned int off = 0;
> + unsigned int total = (unsigned int)n;
> +
> FIB_RETURN_IF_TRUE(((fib == NULL) || (ips == NULL) ||
> (next_hops == NULL) || (fib->lookup == NULL)), -EINVAL);
>
> - fib->lookup(fib->dp, ips, next_hops, n);
> + while (off < total) {
> + unsigned int chunk = RTE_MIN(total - off,
> + FIB_MAX_LOOKUP_BULK);
> + fib->lookup(fib->dp, zero_vrf_ids, ips + off,
> + next_hops + off, chunk);
> + off += chunk;
> + }
> +
> + return 0;
> +}
> +
> +RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fib_vrf_lookup_bulk, 26.07)
> +int
> +rte_fib_vrf_lookup_bulk(struct rte_fib *fib, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, int n)
> +{
> + FIB_RETURN_IF_TRUE(((fib == NULL) || (vrf_ids == NULL) ||
> + (ips == NULL) || (next_hops == NULL) ||
> + (fib->lookup == NULL)), -EINVAL);
> +
> + fib->lookup(fib->dp, vrf_ids, ips, next_hops, (unsigned int)n);
> return 0;
> }
>
> +RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fib_vrf_add, 26.07)
> +int
> +rte_fib_vrf_add(struct rte_fib *fib, uint16_t vrf_id, uint32_t ip,
> + uint8_t depth, uint64_t next_hop)
> +{
> + if ((fib == NULL) || (fib->modify == NULL) ||
> + (depth > RTE_FIB_MAXDEPTH))
> + return -EINVAL;
> + return fib->modify(fib, vrf_id, ip, depth, next_hop, RTE_FIB_ADD);
> +}
> +
> +RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fib_vrf_delete, 26.07)
> +int
> +rte_fib_vrf_delete(struct rte_fib *fib, uint16_t vrf_id, uint32_t ip,
> + uint8_t depth)
> +{
> + if ((fib == NULL) || (fib->modify == NULL) ||
> + (depth > RTE_FIB_MAXDEPTH))
> + return -EINVAL;
> + return fib->modify(fib, vrf_id, ip, depth, 0, RTE_FIB_DEL);
> +}
> +
> RTE_EXPORT_SYMBOL(rte_fib_create)
> struct rte_fib *
> rte_fib_create(const char *name, int socket_id, struct rte_fib_conf *conf)
> {
> char mem_name[RTE_FIB_NAMESIZE];
> + char rib_name[RTE_FIB_NAMESIZE];
> int ret;
> struct rte_fib *fib = NULL;
> struct rte_rib *rib = NULL;
> struct rte_tailq_entry *te;
> struct rte_fib_list *fib_list;
> struct rte_rib_conf rib_conf;
> + uint16_t num_vrfs;
> + uint16_t vrf;
>
> /* Check user arguments. */
> if ((name == NULL) || (conf == NULL) || (conf->max_routes < 0) ||
> @@ -170,16 +231,42 @@ rte_fib_create(const char *name, int socket_id, struct
> rte_fib_conf *conf)
> return NULL;
> }
>
> + num_vrfs = (conf->max_vrfs == 0) ? 1 : conf->max_vrfs;
> rib_conf.ext_sz = conf->rib_ext_sz;
> rib_conf.max_nodes = conf->max_routes * 2;
>
> - rib = rte_rib_create(name, socket_id, &rib_conf);
> - if (rib == NULL) {
> - FIB_LOG(ERR,
> - "Can not allocate RIB %s", name);
> + struct rte_rib **ribs = rte_zmalloc_socket("FIB_RIBS",
> + num_vrfs * sizeof(*fib->ribs), RTE_CACHE_LINE_SIZE, socket_id);
> + if (ribs == NULL) {
> + FIB_LOG(ERR, "FIB %s RIB array allocation failed", name);
> + rte_errno = ENOMEM;
> return NULL;
> }
>
> + uint64_t *def_nh = rte_zmalloc_socket("FIB_DEF_NH",
> + num_vrfs * sizeof(*def_nh), RTE_CACHE_LINE_SIZE, socket_id);
> + if (def_nh == NULL) {
> + FIB_LOG(ERR, "FIB %s default nexthop array allocation failed",
> name);
> + rte_errno = ENOMEM;
> + rte_free(ribs);
> + return NULL;
> + }
> +
> + for (vrf = 0; vrf < num_vrfs; vrf++) {
> + if (num_vrfs == 1)
> + snprintf(rib_name, sizeof(rib_name), "%s", name);
> + else
> + snprintf(rib_name, sizeof(rib_name), "%s_vrf%u", name,
> vrf);
> + rib = rte_rib_create(rib_name, socket_id, &rib_conf);
> + if (rib == NULL) {
> + FIB_LOG(ERR, "Can not allocate RIB %s", rib_name);
> + goto free_ribs;
> + }
> + ribs[vrf] = rib;
> + def_nh[vrf] = (conf->vrf_default_nh != NULL) ?
> + conf->vrf_default_nh[vrf] : conf->default_nh;
> + }
> +
> snprintf(mem_name, sizeof(mem_name), "FIB_%s", name);
> fib_list = RTE_TAILQ_CAST(rte_fib_tailq.head, rte_fib_list);
>
> @@ -215,11 +302,13 @@ rte_fib_create(const char *name, int socket_id, struct
> rte_fib_conf *conf)
> goto free_te;
> }
>
> + fib->num_vrfs = num_vrfs;
> + fib->ribs = ribs;
> + fib->def_nh = def_nh;
> +
> rte_strlcpy(fib->name, name, sizeof(fib->name));
> - fib->rib = rib;
> fib->type = conf->type;
> fib->flags = conf->flags;
> - fib->def_nh = conf->default_nh;
> ret = init_dataplane(fib, socket_id, conf);
> if (ret < 0) {
> FIB_LOG(ERR,
> @@ -242,8 +331,12 @@ rte_fib_create(const char *name, int socket_id, struct
> rte_fib_conf *conf)
> rte_free(te);
> exit:
> rte_mcfg_tailq_write_unlock();
> - rte_rib_free(rib);
> +free_ribs:
> + for (vrf = 0; vrf < num_vrfs; vrf++)
> + rte_rib_free(ribs[vrf]);
>
> + rte_free(def_nh);
> + rte_free(ribs);
> return NULL;
> }
>
> @@ -311,7 +404,13 @@ rte_fib_free(struct rte_fib *fib)
> rte_mcfg_tailq_write_unlock();
>
> free_dataplane(fib);
> - rte_rib_free(fib->rib);
> + if (fib->ribs != NULL) {
> + uint16_t vrf;
> + for (vrf = 0; vrf < fib->num_vrfs; vrf++)
> + rte_rib_free(fib->ribs[vrf]);
> + }
> + rte_free(fib->ribs);
> + rte_free(fib->def_nh);
> rte_free(fib);
> rte_free(te);
> }
> @@ -327,7 +426,18 @@ RTE_EXPORT_SYMBOL(rte_fib_get_rib)
> struct rte_rib *
> rte_fib_get_rib(struct rte_fib *fib)
> {
> - return (fib == NULL) ? NULL : fib->rib;
> + return (fib == NULL || fib->ribs == NULL) ? NULL : fib->ribs[0];
> +}
> +
> +RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fib_vrf_get_rib, 26.07)
> +struct rte_rib *
> +rte_fib_vrf_get_rib(struct rte_fib *fib, uint16_t vrf_id)
> +{
> + if (fib == NULL || fib->ribs == NULL)
> + return NULL;
> + if (vrf_id >= fib->num_vrfs)
> + return NULL;
> + return fib->ribs[vrf_id];
> }
>
> RTE_EXPORT_SYMBOL(rte_fib_select_lookup)
> diff --git a/lib/fib/rte_fib.h b/lib/fib/rte_fib.h
> index b16a653535..883195c7d6 100644
> --- a/lib/fib/rte_fib.h
> +++ b/lib/fib/rte_fib.h
> @@ -53,11 +53,11 @@ enum rte_fib_type {
> };
>
> /** Modify FIB function */
> -typedef int (*rte_fib_modify_fn_t)(struct rte_fib *fib, uint32_t ip,
> - uint8_t depth, uint64_t next_hop, int op);
> +typedef int (*rte_fib_modify_fn_t)(struct rte_fib *fib, uint16_t vrf_id,
> + uint32_t ip, uint8_t depth, uint64_t next_hop, int op);
> /** FIB bulk lookup function */
> -typedef void (*rte_fib_lookup_fn_t)(void *fib, const uint32_t *ips,
> - uint64_t *next_hops, const unsigned int n);
> +typedef void (*rte_fib_lookup_fn_t)(void *fib, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, const unsigned int n);
>
> enum rte_fib_op {
> RTE_FIB_ADD,
> @@ -110,6 +110,10 @@ struct rte_fib_conf {
> } dir24_8;
> };
> unsigned int flags; /**< Optional feature flags from RTE_FIB_F_* */
> + /** Number of VRFs to support (0 or 1 = single VRF for backward compat)
> */
> + uint16_t max_vrfs;
> + /** Per-VRF default nexthops (NULL = use default_nh for all) */
> + uint64_t *vrf_default_nh;
> };
>
> /** FIB RCU QSBR configuration structure. */
> @@ -224,6 +228,71 @@ rte_fib_delete(struct rte_fib *fib, uint32_t ip, uint8_t
> depth);
> int
> rte_fib_lookup_bulk(struct rte_fib *fib, uint32_t *ips,
> uint64_t *next_hops, int n);
> +
> +/**
> + * Add a route to the FIB with VRF ID.
> + *
> + * @param fib
> + * FIB object handle
> + * @param vrf_id
> + * VRF ID (0 to max_vrfs-1)
> + * @param ip
> + * IPv4 prefix address to be added to the FIB
> + * @param depth
> + * Prefix length
> + * @param next_hop
> + * Next hop to be added to the FIB
> + * @return
> + * 0 on success, negative value otherwise
> + */
> +__rte_experimental
> +int
> +rte_fib_vrf_add(struct rte_fib *fib, uint16_t vrf_id, uint32_t ip,
> + uint8_t depth, uint64_t next_hop);
> +
> +/**
> + * Delete a rule from the FIB with VRF ID.
> + *
> + * @param fib
> + * FIB object handle
> + * @param vrf_id
> + * VRF ID (0 to max_vrfs-1)
> + * @param ip
> + * IPv4 prefix address to be deleted from the FIB
> + * @param depth
> + * Prefix length
> + * @return
> + * 0 on success, negative value otherwise
> + */
> +__rte_experimental
> +int
> +rte_fib_vrf_delete(struct rte_fib *fib, uint16_t vrf_id, uint32_t ip,
> + uint8_t depth);
> +
> +/**
> + * Lookup multiple IP addresses in the FIB with per-packet VRF IDs.
> + *
> + * @param fib
> + * FIB object handle
> + * @param vrf_ids
> + * Array of VRF IDs
> + * @param ips
> + * Array of IPs to be looked up in the FIB
> + * @param next_hops
> + * Next hop of the most specific rule found for IP in the corresponding
> VRF.
> + * This is an array of eight byte values.
> + * If the lookup for the given IP failed, then corresponding element would
> + * contain default nexthop value configured for that VRF.
> + * @param n
> + * Number of elements in vrf_ids, ips (and next_hops) arrays to lookup.
> + * @return
> + * -EINVAL for incorrect arguments, otherwise 0
> + */
> +__rte_experimental
> +int
> +rte_fib_vrf_lookup_bulk(struct rte_fib *fib, const uint16_t *vrf_ids,
> + const uint32_t *ips, uint64_t *next_hops, int n);
> +
> /**
> * Get pointer to the dataplane specific struct
> *
> @@ -237,7 +306,7 @@ void *
> rte_fib_get_dp(struct rte_fib *fib);
>
> /**
> - * Get pointer to the RIB
> + * Get pointer to the RIB for VRF 0
> *
> * @param fib
> * FIB object handle
> @@ -248,6 +317,21 @@ rte_fib_get_dp(struct rte_fib *fib);
> struct rte_rib *
> rte_fib_get_rib(struct rte_fib *fib);
>
> +/**
> + * Get pointer to the RIB for a specific VRF
> + *
> + * @param fib
> + * FIB object handle
> + * @param vrf_id
> + * VRF ID (0 to max_vrfs-1)
> + * @return
> + * Pointer on the RIB on success
> + * NULL otherwise
> + */
> +__rte_experimental
> +struct rte_rib *
> +rte_fib_vrf_get_rib(struct rte_fib *fib, uint16_t vrf_id);
> +
> /**
> * Set lookup function based on type
> *
> --
> 2.43.0
