From: Matias Elo <[email protected]> DPDK requires that mbuf memory is allocated from huge pages to ensure page locking. Modify zero-copy dpdk pktio to fall back to packet copy if transmitted ODP packet is not allocated from huge page memory.
Signed-off-by: Matias Elo <[email protected]> --- /** Email created from pull request 119 (matiaselo:dev/dpdk_zero_copy) ** https://github.com/Linaro/odp/pull/119 ** Patch: https://github.com/Linaro/odp/pull/119.patch ** Base sha: bb3400df72401d88aa8e79fb0244a2b90f3bd138 ** Merge commit sha: efca05f2f045fc7c6844cef752f200bf9668ceec **/ platform/linux-generic/include/odp_pool_internal.h | 1 + platform/linux-generic/odp_pool.c | 19 ++++++++ platform/linux-generic/pktio/dpdk.c | 53 +++++++++++++--------- 3 files changed, 51 insertions(+), 22 deletions(-) diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index f2d2e2ca..edf75d6e 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -71,6 +71,7 @@ typedef struct pool_t { uint8_t *uarea_base_addr; /* Used by DPDK zero-copy pktio */ + uint8_t mem_from_huge_pages; pool_destroy_cb_fn ext_destroy; void *ext_desc; diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index f2410b6e..ef19ed6f 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -8,6 +8,7 @@ #include <odp/api/shared_memory.h> #include <odp/api/align.h> #include <odp/api/ticketlock.h> +#include <odp/api/system_info.h> #include <odp_pool_internal.h> #include <odp_internal.h> @@ -281,6 +282,22 @@ static void init_buffers(pool_t *pool) } } +static bool shm_is_from_huge_pages(odp_shm_t shm) +{ + odp_shm_info_t info; + uint64_t huge_page_size = odp_sys_huge_page_size(); + + if (huge_page_size == 0) + return 0; + + if (odp_shm_info(shm, &info)) { + ODP_ERR("Failed to fetch shm info\n"); + return 0; + } + + return (info.page_size >= huge_page_size); +} + static odp_pool_t pool_create(const char *name, odp_pool_param_t *params, uint32_t shmflags) { @@ -406,6 +423,8 @@ static odp_pool_t pool_create(const char *name, odp_pool_param_t *params, goto error; } + pool->mem_from_huge_pages = shm_is_from_huge_pages(pool->shm); + pool->base_addr = odp_shm_addr(pool->shm); pool->uarea_shm = ODP_SHM_INVALID; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 9e3e583d..19310651 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -129,14 +129,13 @@ static void mbuf_init(struct rte_mempool *mp, struct rte_mbuf *mbuf, { void *buf_addr = pkt_hdr->buf_hdr.base_data - RTE_PKTMBUF_HEADROOM; - rte_mem_lock_page(buf_addr); - memset(mbuf, 0, sizeof(struct rte_mbuf)); mbuf->priv_size = 0; mbuf->buf_addr = buf_addr; mbuf->buf_physaddr = rte_mem_virt2phy(buf_addr); - if (odp_unlikely(mbuf->buf_physaddr == RTE_BAD_PHYS_ADDR)) + if (odp_unlikely(mbuf->buf_physaddr == RTE_BAD_PHYS_ADDR || + mbuf->buf_physaddr == 0)) ODP_ABORT("Failed to map virt addr to phy"); mbuf->buf_len = (uint16_t)rte_pktmbuf_data_room_size(mp); @@ -163,6 +162,11 @@ static struct rte_mempool *mbuf_pool_create(const char *name, unsigned num; uint16_t data_room_size; + if (!(pool_entry->mem_from_huge_pages)) { + ODP_ERR("DPDK requires memory is allocated from huge pages\n"); + return NULL; + } + num = pool_entry->num; data_room_size = pool_entry->max_seg_len + CONFIG_PACKET_HEADROOM; elt_size = sizeof(struct rte_mbuf) + (unsigned)data_room_size; @@ -485,12 +489,11 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, static inline int pkt_to_mbuf_zero(pktio_entry_t *pktio_entry, struct rte_mbuf *mbuf_table[], const odp_packet_t pkt_table[], uint16_t num, - uint16_t *seg_count) + uint16_t *copy_count) { pkt_dpdk_t *pkt_dpdk = &pktio_entry->s.pkt_dpdk; int i; - - *seg_count = 0; + *copy_count = 0; for (i = 0; i < num; i++) { odp_packet_t pkt = pkt_table[i]; @@ -502,20 +505,26 @@ static inline int pkt_to_mbuf_zero(pktio_entry_t *pktio_entry, if (odp_unlikely(pkt_len > pkt_dpdk->mtu)) goto fail; - if (odp_likely(pkt_hdr->buf_hdr.segcount == 1)) { - if (odp_unlikely(pkt_hdr->extra_type != - PKT_EXTRA_TYPE_DPDK)) - mbuf_init(pkt_dpdk->pkt_pool, mbuf, pkt_hdr); - + if (odp_likely(pkt_hdr->buf_hdr.segcount == 1 && + pkt_hdr->extra_type == PKT_EXTRA_TYPE_DPDK)) { mbuf_update(mbuf, pkt_hdr, pkt_len); } else { - /* Fall back to packet copy */ - if (odp_unlikely(pkt_to_mbuf(pktio_entry, &mbuf, - &pkt, 1) != 1)) - goto fail; - (*seg_count)++; - } + pool_t *pool_entry = pkt_hdr->buf_hdr.pool_ptr; + + if (pkt_hdr->buf_hdr.segcount != 1 || + !pool_entry->mem_from_huge_pages) { + /* Fall back to packet copy */ + if (odp_unlikely(pkt_to_mbuf(pktio_entry, &mbuf, + &pkt, 1) != 1)) + goto fail; + (*copy_count)++; + } else { + mbuf_init(pkt_dpdk->pkt_pool, mbuf, + pkt_hdr); + mbuf_update(mbuf, pkt_hdr, pkt_len); + } + } mbuf_table[i] = mbuf; } return i; @@ -1148,7 +1157,7 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, { struct rte_mbuf *tx_mbufs[num]; pkt_dpdk_t *pkt_dpdk = &pktio_entry->s.pkt_dpdk; - uint16_t seg_count = 0; + uint16_t copy_count = 0; int tx_pkts; int i; int mbufs; @@ -1158,7 +1167,7 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, if (ODP_DPDK_ZERO_COPY) mbufs = pkt_to_mbuf_zero(pktio_entry, tx_mbufs, pkt_table, num, - &seg_count); + ©_count); else mbufs = pkt_to_mbuf(pktio_entry, tx_mbufs, pkt_table, num); @@ -1172,11 +1181,11 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, odp_ticketlock_unlock(&pkt_dpdk->tx_lock[index]); if (ODP_DPDK_ZERO_COPY) { - /* Free copied segmented packets */ - if (odp_unlikely(seg_count)) { + /* Free copied packets */ + if (odp_unlikely(copy_count)) { uint16_t freed = 0; - for (i = 0; i < mbufs && freed != seg_count; i++) { + for (i = 0; i < mbufs && freed != copy_count; i++) { odp_packet_t pkt = pkt_table[i]; odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
