From: Jie Liu <[email protected]>
Add a union of the scalar and vectorized buffer ring pointers to the
Tx queue structure so the vectorized path can use sxe2_tx_buffer_vec
directly. Rename the mbuf fill helper to sxe2_tx_pkts_mbuf_fill_vec,
switch the vectorized Tx burst and mbuf release paths to the
buffer_ring_vec member, and drop the AVX512-specific fill handling and
conditional branching.
Fixes: ac60f302cb ("net/sxe2: add vectorized Rx and Tx")
Cc: [email protected]
Cc: [email protected]
Signed-off-by: Jie Liu <[email protected]>
---
drivers/net/sxe2/sxe2_queue.h | 5 +-
drivers/net/sxe2/sxe2_txrx_vec.c | 63 ++-----
drivers/net/sxe2/sxe2_txrx_vec_avx2.c | 10 +-
drivers/net/sxe2/sxe2_txrx_vec_avx512.c | 133 +-------------
drivers/net/sxe2/sxe2_txrx_vec_common.h | 9 +-
drivers/net/sxe2/sxe2_txrx_vec_neon.c | 219 ++++++++++++++++--------
drivers/net/sxe2/sxe2_txrx_vec_sse.c | 10 +-
7 files changed, 179 insertions(+), 270 deletions(-)
diff --git a/drivers/net/sxe2/sxe2_queue.h b/drivers/net/sxe2/sxe2_queue.h
index 10bdaf5b8d..e53a1ce852 100644
--- a/drivers/net/sxe2/sxe2_queue.h
+++ b/drivers/net/sxe2/sxe2_queue.h
@@ -62,7 +62,10 @@ struct sxe2_txq_ops {
};
struct sxe2_tx_queue {
volatile union sxe2_tx_data_desc *desc_ring;
- struct sxe2_tx_buffer *buffer_ring;
+ union {
+ struct sxe2_tx_buffer *buffer_ring;
+ struct sxe2_tx_buffer_vec *buffer_ring_vec;
+ };
volatile uint32_t *tdt_reg_addr;
uint64_t offloads;
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.c b/drivers/net/sxe2/sxe2_txrx_vec.c
index 9cfa565548..05940c8cb5 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec.c
@@ -170,67 +170,28 @@ int32_t __rte_cold sxe2_tx_vec_support_check(struct
rte_eth_dev *dev, uint32_t *
static void sxe2_tx_queue_mbufs_release_vec(struct sxe2_tx_queue *txq)
{
- struct sxe2_tx_buffer *buffer;
+ struct sxe2_tx_buffer_vec *buffer_vec;
uint16_t i;
- if (unlikely(txq == NULL || txq->buffer_ring == NULL)) {
+ if (unlikely(txq == NULL || txq->buffer_ring_vec == NULL)) {
PMD_LOG_ERR(TX, "Tx release mbufs vec, invalid params.");
return;
}
- i = txq->next_dd - (txq->rs_thresh - 1);
-#ifdef CC_AVX512_SUPPORT
- struct rte_eth_dev *dev;
- struct sxe2_tx_buffer_vec *buffer_vec;
-
- dev = &rte_eth_devices[txq->port_id];
- if (dev->tx_pkt_burst == sxe2_tx_pkts_vec_avx512 ||
- dev->tx_pkt_burst == sxe2_tx_pkts_vec_avx512_simple) {
- buffer_vec = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ i = txq->next_dd - (txq->rs_thresh - 1);
+ buffer_vec = txq->buffer_ring_vec;
- if (txq->next_use < i) {
- for ( ; i < txq->ring_depth; ++i) {
- if (buffer_vec[i].mbuf != NULL) {
-
rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
- buffer_vec[i].mbuf = NULL;
- }
- }
- i = 0;
- }
- for ( ; i < txq->next_use; ++i) {
- if (buffer_vec[i].mbuf != NULL) {
- rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
- buffer_vec[i].mbuf = NULL;
- }
- }
- } else {
-#endif
- buffer = txq->buffer_ring;
- buffer = txq->buffer_ring;
- if (txq->next_use < i) {
- for ( ; i < txq->ring_depth; ++i) {
- if (buffer[i].mbuf != NULL) {
- rte_pktmbuf_free_seg(buffer[i].mbuf);
- buffer[i].mbuf = NULL;
- }
- }
- i = 0;
+ if (txq->next_use < i) {
+ for ( ; i < txq->ring_depth; ++i) {
+ rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
+ buffer_vec[i].mbuf = NULL;
}
- for (; i < txq->next_use; ++i) {
- if (buffer[i].mbuf != NULL) {
- rte_pktmbuf_free_seg(buffer[i].mbuf);
- buffer[i].mbuf = NULL;
- }
- }
-#ifdef CC_AVX512_SUPPORT
+ i = 0;
}
-#endif
- for (; i < txq->next_use; ++i) {
- if (buffer[i].mbuf != NULL) {
- rte_pktmbuf_free_seg(buffer[i].mbuf);
- buffer[i].mbuf = NULL;
- }
+ for ( ; i < txq->next_use; ++i) {
+ rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
+ buffer_vec[i].mbuf = NULL;
}
}
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_avx2.c
b/drivers/net/sxe2/sxe2_txrx_vec_avx2.c
index 0618e6d988..da96ca3064 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec_avx2.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec_avx2.c
@@ -115,7 +115,7 @@ sxe2_tx_pkts_vec_avx2_batch(struct sxe2_tx_queue *txq,
struct rte_mbuf **tx_pkts
uint16_t nb_pkts, bool with_offloads)
{
volatile union sxe2_tx_data_desc *desc;
- struct sxe2_tx_buffer *buffer;
+ struct sxe2_tx_buffer_vec *buffer;
uint16_t next_use;
uint16_t res_num;
uint16_t tx_num;
@@ -134,14 +134,14 @@ sxe2_tx_pkts_vec_avx2_batch(struct sxe2_tx_queue *txq,
struct rte_mbuf **tx_pkts
next_use = txq->next_use;
desc = &txq->desc_ring[next_use];
- buffer = &txq->buffer_ring[next_use];
+ buffer = &txq->buffer_ring_vec[next_use];
txq->desc_free_num -= nb_pkts;
res_num = txq->ring_depth - txq->next_use;
if (tx_num >= res_num) {
- sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, res_num);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, res_num);
sxe2_tx_desc_fill_avx2(desc, tx_pkts, res_num,
SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
@@ -157,10 +157,10 @@ sxe2_tx_pkts_vec_avx2_batch(struct sxe2_tx_queue *txq,
struct rte_mbuf **tx_pkts
next_use = 0;
txq->next_rs = txq->rs_thresh - 1;
desc = &txq->desc_ring[next_use];
- buffer = &txq->buffer_ring[next_use];
+ buffer = &txq->buffer_ring_vec[next_use];
}
- sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, tx_num);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, tx_num);
sxe2_tx_desc_fill_avx2(desc, tx_pkts, tx_num,
SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
index a830c7a33b..6c8415ee5a 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
@@ -1,8 +1,6 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
*/
-
-#ifndef SXE2_TEST
#include <rte_vect.h>
#include "sxe2_ethdev.h"
@@ -12,114 +10,6 @@
#include "sxe2_txrx_vec_common.h"
#include "sxe2_vsi.h"
-static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct
sxe2_tx_queue *txq)
-{
- struct sxe2_tx_buffer_vec *buffer;
- struct rte_mbuf *mbuf;
- struct rte_mbuf *mbuf_free_arr[SXE2_TX_FREE_BUFFER_SIZE_MAX_VEC];
- struct rte_mempool *mp;
- struct rte_mempool_cache *cache;
- void **cache_objs;
- uint32_t copied;
- uint32_t i;
- int32_t ret;
- uint16_t rs_thresh;
- uint16_t free_num;
-
- if (rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_DESC_DONE) !=
- (txq->desc_ring[txq->next_dd].wb.dd &
- rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_MASK))) {
- ret = 0;
- goto l_end;
- }
-
- rs_thresh = txq->rs_thresh;
-
- buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
- buffer += txq->next_dd - (rs_thresh - 1);
-
- if ((txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) &&
- (rs_thresh & 31) == 0) {
- mp = buffer[0].mbuf->pool;
- cache = rte_mempool_default_cache(mp, rte_lcore_id());
-
- if (cache == NULL || cache->len)
- goto normal;
-
- if (rs_thresh > RTE_MEMPOOL_CACHE_MAX_SIZE) {
- (void)rte_mempool_ops_enqueue_bulk(mp, (void *)buffer,
rs_thresh);
- goto done;
- }
- cache_objs = &cache->objs[cache->len];
-
- copied = 0;
- while (copied < rs_thresh) {
- const __m512i objs0 =
_mm512_loadu_si512(&buffer[copied]);
- const __m512i objs1 = _mm512_loadu_si512(&buffer[copied
+ 8]);
- const __m512i objs2 = _mm512_loadu_si512(&buffer[copied
+ 16]);
- const __m512i objs3 = _mm512_loadu_si512(&buffer[copied
+ 24]);
-
- _mm512_storeu_si512(&cache_objs[copied], objs0);
- _mm512_storeu_si512(&cache_objs[copied + 8], objs1);
- _mm512_storeu_si512(&cache_objs[copied + 16], objs2);
- _mm512_storeu_si512(&cache_objs[copied + 24], objs3);
- copied += 32;
- }
- cache->len += rs_thresh;
-
- if (cache->len >= cache->flushthresh) {
- (void)rte_mempool_ops_enqueue_bulk(mp,
- &cache->objs[cache->size], cache->len -
cache->size);
- cache->len = cache->size;
- }
- goto done;
- }
-
-normal:
- mbuf = rte_pktmbuf_prefree_seg(buffer[0].mbuf);
-
- if (likely(mbuf)) {
- mbuf_free_arr[0] = mbuf;
- free_num = 1;
-
- for (i = 1; i < rs_thresh; ++i) {
- mbuf = rte_pktmbuf_prefree_seg(buffer[i].mbuf);
-
- if (likely(mbuf)) {
- if (likely(mbuf->pool ==
mbuf_free_arr[0]->pool)) {
- mbuf_free_arr[free_num] = mbuf;
- free_num++;
- } else {
-
rte_mempool_put_bulk(mbuf_free_arr[0]->pool,
- (void *)mbuf_free_arr,
free_num);
-
- mbuf_free_arr[0] = mbuf;
- free_num = 1;
- }
- }
- }
-
- rte_mempool_put_bulk(mbuf_free_arr[0]->pool,
- (void *)mbuf_free_arr,
free_num);
- } else {
- for (i = 1; i < rs_thresh; ++i) {
- mbuf = rte_pktmbuf_prefree_seg(buffer[i].mbuf);
- if (mbuf != NULL)
- rte_mempool_put(mbuf->pool, mbuf);
- }
- }
-
-done:
- txq->desc_free_num += txq->rs_thresh;
- txq->next_dd += txq->rs_thresh;
- if (txq->next_dd >= txq->ring_depth)
- txq->next_dd = txq->rs_thresh - 1;
- ret = rs_thresh;
-
-l_end:
- return ret;
-}
-
static __rte_always_inline void
sxe2_tx_desc_fill_one_avx512(volatile union sxe2_tx_data_desc *desc, struct
rte_mbuf *pkt,
uint64_t desc_cmd, bool with_offloads)
@@ -207,16 +97,6 @@ void sxe2_tx_desc_fill_avx512(volatile union
sxe2_tx_data_desc *desc, struct rte
}
}
-static __rte_always_inline void
-sxe2_tx_pkts_mbuf_fill_avx512(struct sxe2_tx_buffer_vec *buffer,
- struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
-{
- uint16_t i;
-
- for (i = 0; i < nb_pkts; ++i)
- buffer[i].mbuf = tx_pkts[i];
-}
-
static __rte_always_inline uint16_t
sxe2_tx_pkts_vec_avx512_batch(struct sxe2_tx_queue *txq, struct rte_mbuf
**tx_pkts,
uint16_t nb_pkts, bool with_offloads)
@@ -228,7 +108,7 @@ sxe2_tx_pkts_vec_avx512_batch(struct sxe2_tx_queue *txq,
struct rte_mbuf **tx_pk
uint16_t tx_num;
if (txq->desc_free_num < txq->free_thresh)
- (void)sxe2_tx_bufs_free_vec_avx512(txq);
+ (void)sxe2_tx_bufs_free_vec(txq);
nb_pkts = RTE_MIN(txq->desc_free_num, nb_pkts);
if (unlikely(nb_pkts == 0)) {
@@ -241,15 +121,14 @@ sxe2_tx_pkts_vec_avx512_batch(struct sxe2_tx_queue *txq,
struct rte_mbuf **tx_pk
next_use = txq->next_use;
desc = &txq->desc_ring[next_use];
- buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
- buffer += next_use;
+ buffer = &txq->buffer_ring_vec[next_use];
txq->desc_free_num -= nb_pkts;
res_num = txq->ring_depth - txq->next_use;
if (tx_num >= res_num) {
- sxe2_tx_pkts_mbuf_fill_avx512(buffer, tx_pkts, res_num);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, res_num);
sxe2_tx_desc_fill_avx512(desc, tx_pkts, res_num,
SXE2_TX_DATA_DESC_CMD_EOP,
with_offloads);
@@ -265,10 +144,10 @@ sxe2_tx_pkts_vec_avx512_batch(struct sxe2_tx_queue *txq,
struct rte_mbuf **tx_pk
next_use = 0;
txq->next_rs = txq->rs_thresh - 1;
desc = txq->desc_ring;
- buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ buffer = &txq->buffer_ring_vec[next_use];
}
- sxe2_tx_pkts_mbuf_fill_avx512(buffer, tx_pkts, tx_num);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, tx_num);
sxe2_tx_desc_fill_avx512(desc, tx_pkts, tx_num,
SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
@@ -863,5 +742,3 @@ uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void
*rx_queue,
return sxe2_rx_pkts_scattered_common_vec_avx512(rx_queue,
rx_pkts, nb_pkts, true);
}
-
-#endif
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_common.h
b/drivers/net/sxe2/sxe2_txrx_vec_common.h
index 9ac99cf0fa..ede4c236b1 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec_common.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec_common.h
@@ -25,10 +25,11 @@
#define SXE2_TX_FREE_BUFFER_SIZE_MAX_VEC 64
static __rte_always_inline void
-sxe2_tx_pkts_mbuf_fill(struct sxe2_tx_buffer *buffer,
- struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+sxe2_tx_pkts_mbuf_fill_vec(struct sxe2_tx_buffer_vec *buffer,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
uint16_t i;
+
for (i = 0; i < nb_pkts; ++i)
buffer[i].mbuf = tx_pkts[i];
}
@@ -36,7 +37,7 @@ sxe2_tx_pkts_mbuf_fill(struct sxe2_tx_buffer *buffer,
static __rte_always_inline int32_t
sxe2_tx_bufs_free_vec(struct sxe2_tx_queue *txq)
{
- struct sxe2_tx_buffer *buffer;
+ struct sxe2_tx_buffer_vec *buffer;
struct rte_mbuf *mbuf;
struct rte_mbuf *mbuf_free_arr[SXE2_TX_FREE_BUFFER_SIZE_MAX_VEC];
int32_t ret;
@@ -50,7 +51,7 @@ sxe2_tx_bufs_free_vec(struct sxe2_tx_queue *txq)
goto l_end;
}
rs_thresh = txq->rs_thresh;
- buffer = &txq->buffer_ring[txq->next_dd - (rs_thresh - 1)];
+ buffer = &txq->buffer_ring_vec[txq->next_dd - (rs_thresh - 1)];
mbuf = rte_pktmbuf_prefree_seg(buffer[0].mbuf);
if (likely(mbuf)) {
mbuf_free_arr[0] = mbuf;
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_neon.c
b/drivers/net/sxe2/sxe2_txrx_vec_neon.c
index 4e5cb87cd5..0b139b8269 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec_neon.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec_neon.c
@@ -34,12 +34,58 @@ sxe2_tx_desc_fill_one_neon(volatile union sxe2_tx_data_desc
*desc,
vst1q_u64(RTE_CAST_PTR(uint64_t *, desc), data_desc);
}
+static __rte_always_inline void
+sxe2_tx_desc_fill_4_neon_simple(volatile union sxe2_tx_data_desc *desc,
+ struct rte_mbuf **pkts)
+{
+ uint64x2_t d0, d1, d2, d3;
+ uint64x2x4_t v;
+ const uint64_t cmd_base = ((uint64_t)SXE2_TX_DESC_DTYPE_DATA) |
+ ((uint64_t)SXE2_TX_DATA_DESC_CMD_EOP) <<
+ SXE2_TX_DATA_DESC_CMD_SHIFT;
+
+ d0 = (uint64x2_t){
+ rte_pktmbuf_iova(pkts[0]),
+ cmd_base |
+ ((uint64_t)pkts[0]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT
|
+ ((uint64_t)SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[0]->l2_len))
+ << SXE2_TX_DATA_DESC_OFFSET_SHIFT
+ };
+ d1 = (uint64x2_t){
+ rte_pktmbuf_iova(pkts[1]),
+ cmd_base |
+ ((uint64_t)pkts[1]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT
|
+ ((uint64_t)SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[1]->l2_len))
+ << SXE2_TX_DATA_DESC_OFFSET_SHIFT
+ };
+ d2 = (uint64x2_t){
+ rte_pktmbuf_iova(pkts[2]),
+ cmd_base |
+ ((uint64_t)pkts[2]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT
|
+ ((uint64_t)SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[2]->l2_len))
+ << SXE2_TX_DATA_DESC_OFFSET_SHIFT
+ };
+ d3 = (uint64x2_t){
+ rte_pktmbuf_iova(pkts[3]),
+ cmd_base |
+ ((uint64_t)pkts[3]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT
|
+ ((uint64_t)SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[3]->l2_len))
+ << SXE2_TX_DATA_DESC_OFFSET_SHIFT
+ };
+
+ v.val[0] = d0;
+ v.val[1] = d1;
+ v.val[2] = d2;
+ v.val[3] = d3;
+ vst1q_u64_x4(RTE_CAST_PTR(uint64_t *, desc), v);
+}
+
static __rte_always_inline uint16_t
sxe2_tx_pkts_vec_neon_batch(struct sxe2_tx_queue *txq, struct rte_mbuf
**tx_pkts,
uint16_t nb_pkts, bool with_offloads)
{
volatile union sxe2_tx_data_desc *desc;
- struct sxe2_tx_buffer *buffer;
+ struct sxe2_tx_buffer_vec *buffer;
uint16_t next_use;
uint16_t res_num;
uint16_t tx_num;
@@ -59,18 +105,26 @@ sxe2_tx_pkts_vec_neon_batch(struct sxe2_tx_queue *txq,
struct rte_mbuf **tx_pkts
next_use = txq->next_use;
desc = &txq->desc_ring[next_use];
- buffer = &txq->buffer_ring[next_use];
+ buffer = &txq->buffer_ring_vec[next_use];
txq->desc_free_num -= nb_pkts;
res_num = txq->ring_depth - txq->next_use;
if (tx_num >= res_num) {
- sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, res_num);
-
- for (i = 0; i < res_num - 1; ++i, ++tx_pkts, ++desc) {
- sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
- SXE2_TX_DATA_DESC_CMD_EOP,
with_offloads);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, res_num);
+ if (with_offloads) {
+ for (i = 0; i < res_num - 1; ++i, ++tx_pkts, ++desc) {
+ sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
+ SXE2_TX_DATA_DESC_CMD_EOP,
with_offloads);
+ }
+ } else {
+ for (i = 0; i + 3 < res_num - 1; i += 4, tx_pkts += 4,
desc += 4)
+ sxe2_tx_desc_fill_4_neon_simple(desc, tx_pkts);
+ for (; i < res_num - 1; ++i, ++tx_pkts, ++desc) {
+ sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
+ SXE2_TX_DATA_DESC_CMD_EOP,
false);
+ }
}
sxe2_tx_desc_fill_one_neon(desc, *tx_pkts++,
@@ -82,14 +136,23 @@ sxe2_tx_pkts_vec_neon_batch(struct sxe2_tx_queue *txq,
struct rte_mbuf **tx_pkts
next_use = 0;
txq->next_rs = txq->rs_thresh - 1;
desc = &txq->desc_ring[next_use];
- buffer = &txq->buffer_ring[next_use];
+ buffer = &txq->buffer_ring_vec[next_use];
}
- sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, tx_num);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, tx_num);
- for (i = 0; i < tx_num; ++i, ++tx_pkts, ++desc) {
- sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
- SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ if (with_offloads) {
+ for (i = 0; i < tx_num; ++i, ++tx_pkts, ++desc) {
+ sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
+ SXE2_TX_DATA_DESC_CMD_EOP, true);
+ }
+ } else {
+ for (i = 0; i + 3 < tx_num; i += 4, tx_pkts += 4, desc += 4)
+ sxe2_tx_desc_fill_4_neon_simple(desc, tx_pkts);
+ for (; i < tx_num; ++i, ++tx_pkts, ++desc) {
+ sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
+ SXE2_TX_DATA_DESC_CMD_EOP, false);
+ }
}
next_use += tx_num;
@@ -150,22 +213,24 @@ uint16_t sxe2_tx_pkts_vec_neon(void *tx_queue,
}
static __rte_always_inline void
-sxe2_rx_desc_ptype_fill_neon(uint16x8_t staterr, struct rte_mbuf
**__rte_restrict rx_pkts)
+sxe2_rx_desc_ptype_fill_neon(uint32x4_t desc_lo,
+ struct rte_mbuf
**__rte_restrict rx_pkts,
+ const uint32_t
*__rte_restrict ptype_tbl)
{
- uint16x8_t ptype_mask = {
- 0, 0x3FFULL,
- 0, 0x3FFULL,
- 0, 0x3FFULL,
- 0, 0x3FFULL,
+ const uint32x4_t ptype_mask = {
+ SXE2_RX_DESC_PTYPE_MASK_NO_SHIFT << 16,
+ SXE2_RX_DESC_PTYPE_MASK_NO_SHIFT << 16,
+ SXE2_RX_DESC_PTYPE_MASK_NO_SHIFT << 16,
+ SXE2_RX_DESC_PTYPE_MASK_NO_SHIFT << 16,
};
uint16x8_t ptype_all;
- ptype_all = vandq_u16(staterr, ptype_mask);
+ ptype_all = vreinterpretq_u16_u32(vandq_u32(desc_lo, ptype_mask));
- rx_pkts[3]->packet_type = sxe2_ptype_tbl[vgetq_lane_u16(ptype_all, 3)];
- rx_pkts[2]->packet_type = sxe2_ptype_tbl[vgetq_lane_u16(ptype_all, 7)];
- rx_pkts[1]->packet_type = sxe2_ptype_tbl[vgetq_lane_u16(ptype_all, 1)];
- rx_pkts[0]->packet_type = sxe2_ptype_tbl[vgetq_lane_u16(ptype_all, 5)];
+ rx_pkts[0]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 1)];
+ rx_pkts[1]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 3)];
+ rx_pkts[2]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 5)];
+ rx_pkts[3]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 7)];
}
static __rte_always_inline uint32x4_t
@@ -208,9 +273,10 @@ sxe2_rx_desc_fnav_flags_neon(uint64x2_t descs_arr[4])
static __rte_always_inline void
sxe2_rx_desc_offloads_para_fill_neon(struct sxe2_rx_queue *rxq,
volatile union sxe2_rx_desc *desc,
- uint64x2_t descs[4], struct rte_mbuf **rx_pkts)
+ uint64x2_t descs[4], uint32x4_t desc_lo, uint32x4_t
desc_hi,
+ struct rte_mbuf **rx_pkts)
{
- uint32x4_t desc_lo, desc_hi, flags, tmp_flags;
+ uint32x4_t flags, tmp_flags;
const uint64x2_t mbuf_init = {rxq->mbuf_init_value, 0};
uint64x2_t rearm0, rearm1, rearm2, rearm3;
@@ -267,23 +333,6 @@ sxe2_rx_desc_offloads_para_fill_neon(struct sxe2_rx_queue
*rxq,
0, 0, 0, 0, 0, 0, 0, 0
};
- {
- uint32x4_t d0 = vreinterpretq_u32_u64(descs[0]);
- uint32x4_t d1 = vreinterpretq_u32_u64(descs[1]);
- uint32x4_t d2 = vreinterpretq_u32_u64(descs[2]);
- uint32x4_t d3 = vreinterpretq_u32_u64(descs[3]);
- uint64x2_t f64, t64;
-
- flags = vzip2q_u32(d1, d0);
- tmp_flags = vzip2q_u32(d3, d2);
- f64 = vreinterpretq_u64_u32(flags);
- t64 = vreinterpretq_u64_u32(tmp_flags);
- desc_lo = vreinterpretq_u32_u64(vcombine_u64(vget_low_u64(f64),
-
vget_low_u64(t64)));
- desc_hi = vreinterpretq_u32_u64(vcombine_u64(vget_high_u64(f64),
-
vget_high_u64(t64)));
- }
-
desc_lo = vandq_u32(desc_lo, desc_msk);
desc_hi = vandq_u32(desc_hi, rss_msk);
@@ -442,25 +491,39 @@ sxe2_rx_pkts_common_vec_neon(struct sxe2_rx_queue *rxq,
struct rte_mbuf **rx_pkt
uint64x2_t descs[SXE2_RX_NUM_PER_LOOP_NEON];
uint8x16_t pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
uint64x2_t mbp1, mbp2;
+ uint32x4_t desc_lo, desc_hi;
uint16x8_t staterr;
uint16x8_t tmp;
uint16_t bit_num;
descs[3] = vld1q_u64(RTE_CAST_PTR(uint64_t *, desc + 3));
- rte_atomic_thread_fence(rte_memory_order_acquire);
descs[2] = vld1q_u64(RTE_CAST_PTR(uint64_t *, desc + 2));
- rte_atomic_thread_fence(rte_memory_order_acquire);
descs[1] = vld1q_u64(RTE_CAST_PTR(uint64_t *, desc + 1));
- rte_atomic_thread_fence(rte_memory_order_acquire);
descs[0] = vld1q_u64(RTE_CAST_PTR(uint64_t *, desc));
rte_atomic_thread_fence(rte_memory_order_acquire);
-
descs[3] = vld1q_lane_u64(RTE_CAST_PTR(uint64_t *, desc + 3),
descs[3], 0);
descs[2] = vld1q_lane_u64(RTE_CAST_PTR(uint64_t *, desc + 2),
descs[2], 0);
descs[1] = vld1q_lane_u64(RTE_CAST_PTR(uint64_t *, desc + 1),
descs[1], 0);
descs[0] = vld1q_lane_u64(RTE_CAST_PTR(uint64_t *, desc),
descs[0], 0);
+ {
+ uint32x4_t d0 = vreinterpretq_u32_u64(descs[0]);
+ uint32x4_t d1 = vreinterpretq_u32_u64(descs[1]);
+ uint32x4_t d2 = vreinterpretq_u32_u64(descs[2]);
+ uint32x4_t d3 = vreinterpretq_u32_u64(descs[3]);
+
+ uint32x4_t q1_01 = vzip2q_u32(d0, d1);
+ uint32x4_t q1_23 = vzip2q_u32(d2, d3);
+ uint64x2_t q1_01_64 = vreinterpretq_u64_u32(q1_01);
+ uint64x2_t q1_23_64 = vreinterpretq_u64_u32(q1_23);
+
+ desc_lo =
vreinterpretq_u32_u64(vcombine_u64(vget_low_u64(q1_01_64),
+
vget_low_u64(q1_23_64)));
+ desc_hi =
vreinterpretq_u32_u64(vcombine_u64(vget_high_u64(q1_01_64),
+
vget_high_u64(q1_23_64)));
+ }
+
mbp1 = vld1q_u64((uint64_t *)&buffer[i]);
mbp2 = vld1q_u64((uint64_t *)&buffer[i + 2]);
@@ -480,7 +543,8 @@ sxe2_rx_pkts_common_vec_neon(struct sxe2_rx_queue *rxq,
struct rte_mbuf **rx_pkt
pkt_mb1 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[0]),
rvp_shuf_mask);
if (do_offload) {
- sxe2_rx_desc_offloads_para_fill_neon(rxq, desc, descs,
&rx_pkts[i]);
+ sxe2_rx_desc_offloads_para_fill_neon(rxq, desc, descs,
desc_lo,
+ desc_hi,
&rx_pkts[i]);
} else {
const uint64x2_t mbuf_init = {
rxq->mbuf_init_value,
@@ -515,55 +579,48 @@ sxe2_rx_pkts_common_vec_neon(struct sxe2_rx_queue *rxq,
struct rte_mbuf **rx_pkt
rte_prefetch_non_temporal(desc +
SXE2_RX_NUM_PER_LOOP_NEON);
{
- uint32x4_t d0 = vreinterpretq_u32_u64(descs[0]);
- uint32x4_t d1 = vreinterpretq_u32_u64(descs[1]);
- uint32x4_t d2 = vreinterpretq_u32_u64(descs[2]);
- uint32x4_t d3 = vreinterpretq_u32_u64(descs[3]);
- uint32x4_t sterr_tmp1 = vzip2q_u32(d1, d0);
- uint32x4_t sterr_tmp2 = vzip2q_u32(d3, d2);
- uint32x4_t sterr_u32 = vzip1q_u32(sterr_tmp1,
sterr_tmp2);
-
- staterr = vreinterpretq_u16_u32(sterr_u32);
+ uint16x8_t sterr_tmp1 =
vzip2q_u16(vreinterpretq_u16_u64(descs[0]),
+
vreinterpretq_u16_u64(descs[2]));
+ uint16x8_t sterr_tmp2 =
vzip2q_u16(vreinterpretq_u16_u64(descs[1]),
+
vreinterpretq_u16_u64(descs[3]));
+ staterr = vzip1q_u16(sterr_tmp1, sterr_tmp2);
}
- sxe2_rx_desc_ptype_fill_neon(staterr, &rx_pkts[i]);
+ sxe2_rx_desc_ptype_fill_neon(desc_lo, &rx_pkts[i],
sxe2_ptype_tbl);
if (umbcast_flags != NULL) {
- uint32x4_t umbcast_mask = {
- SXE2_RX_DESC_STATUS_UMBCAST_MASK,
SXE2_RX_DESC_STATUS_UMBCAST_MASK,
- SXE2_RX_DESC_STATUS_UMBCAST_MASK,
SXE2_RX_DESC_STATUS_UMBCAST_MASK,
- };
-
+ const uint32x4_t umbcast_mask =
+ vdupq_n_u32(SXE2_RX_DESC_STATUS_UMBCAST_MASK);
uint8x16_t umbcast_shuf_mask = {
- 0x0B, 0x03, 0x0F, 0x07,
+ 3, 7, 11, 15,
0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF,
};
uint8x16_t umbcast_bits =
-
vreinterpretq_u8_u32(vandq_u32(vreinterpretq_u32_u16(staterr),
- umbcast_mask));
+ vreinterpretq_u8_u32(vandq_u32(desc_lo,
umbcast_mask));
umbcast_bits = vqtbl1q_u8(umbcast_bits,
umbcast_shuf_mask);
- vst1q_lane_u32((uint32_t *)umbcast_flags,
- vreinterpretq_u32_u8(umbcast_bits), 0);
+ *(uint32_t *)umbcast_flags =
+
vgetq_lane_u32(vreinterpretq_u32_u8(umbcast_bits), 0);
umbcast_flags += SXE2_RX_NUM_PER_LOOP_NEON;
}
if (split_rxe_flags) {
uint8x16_t eop_shuf_mask = {
- 0x08, 0x00, 0x0C, 0x04,
+ 0, 2, 4, 6,
0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF};
uint8x16_t eop_bits;
uint32x4_t rxe_mask = {
- 0x2080, 0x2080, 0x2080, 0x2080
+ 0x20802080, 0x20802080, 0x20802080, 0x20802080
};
uint32x4_t rxe_bits;
uint32x4_t eop_mask;
- eop_mask = vshlq_n_u32(vdupq_n_u32(1),
SXE2_RX_DESC_STATUS_EOP_SHIFT);
+ eop_mask = vdupq_n_u32((1U <<
SXE2_RX_DESC_STATUS_EOP_SHIFT) |
+ (1U << (SXE2_RX_DESC_STATUS_EOP_SHIFT +
16)));
eop_bits =
vandq_u8(vmvnq_u8(vreinterpretq_u8_u16(staterr)),
vreinterpretq_u8_u32(eop_mask));
@@ -587,12 +644,22 @@ sxe2_rx_pkts_common_vec_neon(struct sxe2_rx_queue *rxq,
struct rte_mbuf **rx_pkt
}
{
- uint32x4_t dd_mask = vdupq_n_u32(1);
- uint32x4_t sterr_dd =
vandq_u32(vreinterpretq_u32_u16(staterr), dd_mask);
- uint16x4_t packed_lo = vmovn_u32(sterr_dd);
- uint64_t dd64 =
vget_lane_u64(vreinterpret_u64_u16(packed_lo), 0);
-
- bit_num = (uint16_t)rte_popcount64(dd64);
+ const uint16x8_t dd_check = {
+ 0x0001, 0x0001, 0x0001, 0x0001,
+ 0, 0, 0, 0
+ };
+ uint16x8_t sterr_dd;
+ uint64_t stat;
+ sterr_dd = vandq_u16(staterr, dd_check);
+ sterr_dd = vshlq_n_u16(sterr_dd, 15);
+ sterr_dd =
+
vreinterpretq_u16_s16(vshrq_n_s16(vreinterpretq_s16_u16(sterr_dd),
+ 15));
+ stat = ~vgetq_lane_u64(vreinterpretq_u64_u16(sterr_dd),
0);
+ if (likely(stat == 0))
+ bit_num = SXE2_RX_NUM_PER_LOOP_NEON;
+ else
+ bit_num = (uint16_t)(rte_ctz64(stat) / 16);
}
done_num += bit_num;
if (likely(bit_num != SXE2_RX_NUM_PER_LOOP_NEON))
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_sse.c
b/drivers/net/sxe2/sxe2_txrx_vec_sse.c
index c3e8a2983b..181bb40041 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec_sse.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec_sse.c
@@ -40,7 +40,7 @@ sxe2_tx_pkts_vec_sse_batch(struct sxe2_tx_queue *txq,
uint16_t nb_pkts, bool with_offloads)
{
volatile union sxe2_tx_data_desc *desc;
- struct sxe2_tx_buffer *buffer;
+ struct sxe2_tx_buffer_vec *buffer;
uint16_t next_use;
uint16_t res_num;
uint16_t tx_num;
@@ -57,11 +57,11 @@ sxe2_tx_pkts_vec_sse_batch(struct sxe2_tx_queue *txq,
tx_num = nb_pkts;
next_use = txq->next_use;
desc = &txq->desc_ring[next_use];
- buffer = &txq->buffer_ring[next_use];
+ buffer = &txq->buffer_ring_vec[next_use];
txq->desc_free_num -= nb_pkts;
res_num = txq->ring_depth - txq->next_use;
if (tx_num >= res_num) {
- sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, res_num);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, res_num);
for (i = 0; i < res_num - 1; ++i, ++tx_pkts, ++desc) {
sxe2_tx_desc_fill_one_sse(desc, *tx_pkts,
SXE2_TX_DATA_DESC_CMD_EOP,
@@ -74,9 +74,9 @@ sxe2_tx_pkts_vec_sse_batch(struct sxe2_tx_queue *txq,
next_use = 0;
txq->next_rs = txq->rs_thresh - 1;
desc = &txq->desc_ring[next_use];
- buffer = &txq->buffer_ring[next_use];
+ buffer = &txq->buffer_ring_vec[next_use];
}
- sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, tx_num);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, tx_num);
for (i = 0; i < tx_num; ++i, ++tx_pkts, ++desc) {
sxe2_tx_desc_fill_one_sse(desc, *tx_pkts,
SXE2_TX_DATA_DESC_CMD_EOP,
--
2.52.0
