Implement hardware TCP Segmentation Offload (TSO) in Microsoft Azure
MANA PMD:
- Define GDMA_WR_OOB_IN_SGL and GDMA_WR_PAD_BY_SGE0 in mana.h.
- Advertise RTE_ETH_TX_OFFLOAD_TCP_TSO in MANA_DEV_TX_OFFLOAD_SUPPORT.
- Handle GDMA_WR_OOB_IN_SGL and GDMA_WR_PAD_BY_SGE0 in
write_dma_client_oob() in gdma.c to configure client_oob_in_sgl and
last_v_bytes.
- In mana_tx_burst() in tx.c:
* Parse RTE_MBUF_F_TX_TCP_SEG and validate total SGEs against
priv->max_send_sge and MAX_SGL_ENTRIES_FOR_TRANSMIT.
* Zero out IPv4 total_length and hdr_checksum (or IPv6 payload_len)
and compute pseudo-header checksum in the TCP header.
* Enable tx_compute_TCP_checksum and set tx_transport_header_offset.
* Format SGL so SGE0 contains packet headers (L2 + L3 + L4) and
subsequent SGEs carry segment payloads, allocating MRs through
mana_alloc_pmd_mr().
* Set GDMA_WR_OOB_IN_SGL | GDMA_WR_PAD_BY_SGE0 work request flags and
assign client_data_unit to m_pkt->tso_segsz.
- Update mana.ini to advertise TSO feature support.
Signed-off-by: Tathagat Priyadarshi <[email protected]>
---
doc/guides/nics/features/mana.ini | 1 +
drivers/net/mana/gdma.c | 5 +
drivers/net/mana/mana.h | 6 +-
drivers/net/mana/tx.c | 177 +++++++++++++++++++++++++-----
4 files changed, 162 insertions(+), 27 deletions(-)
diff --git a/doc/guides/nics/features/mana.ini
b/doc/guides/nics/features/mana.ini
index db555ffe27..edf9469486 100644
--- a/doc/guides/nics/features/mana.ini
+++ b/doc/guides/nics/features/mana.ini
@@ -11,6 +11,7 @@ Rx interrupt = Y
RSS hash = Y
L3 checksum offload = Y
L4 checksum offload = Y
+TSO = Y
Basic stats = Y
MTU update = Y
Multiprocess aware = Y
diff --git a/drivers/net/mana/gdma.c b/drivers/net/mana/gdma.c
index a92aef59d2..65896b6a69 100644
--- a/drivers/net/mana/gdma.c
+++ b/drivers/net/mana/gdma.c
@@ -38,6 +38,11 @@ write_dma_client_oob(uint8_t *work_queue_buffer_pointer,
header->inline_client_oob_size_in_dwords =
client_oob_size / sizeof(uint32_t);
header->client_data_unit = work_request->client_data_unit;
+ if (work_request->flags & GDMA_WR_OOB_IN_SGL) {
+ header->client_oob_in_sgl = 1;
+ if (work_request->flags & GDMA_WR_PAD_BY_SGE0)
+ header->last_v_bytes = work_request->sgl[0].size;
+ }
DP_LOG(DEBUG, "queue buf %p sgl %u oob_h %u du %u oob_buf %p oob_b %u",
work_queue_buffer_pointer, header->num_sgl_entries,
diff --git a/drivers/net/mana/mana.h b/drivers/net/mana/mana.h
index 79879b7626..17aa4a09ac 100644
--- a/drivers/net/mana/mana.h
+++ b/drivers/net/mana/mana.h
@@ -28,12 +28,16 @@ struct mana_shared_data {
RTE_ETH_RX_OFFLOAD_CHECKSUM | \
RTE_ETH_RX_OFFLOAD_RSS_HASH)
+#define GDMA_WR_OOB_IN_SGL (1 << 0)
+#define GDMA_WR_PAD_BY_SGE0 (1 << 1)
+
#define MANA_DEV_TX_OFFLOAD_SUPPORT ( \
RTE_ETH_TX_OFFLOAD_VLAN_INSERT | \
RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \
RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | \
RTE_ETH_TX_OFFLOAD_TCP_CKSUM | \
- RTE_ETH_TX_OFFLOAD_UDP_CKSUM)
+ RTE_ETH_TX_OFFLOAD_UDP_CKSUM | \
+ RTE_ETH_TX_OFFLOAD_TCP_TSO)
#define INDIRECTION_TABLE_NUM_ELEMENTS 64
#define TOEPLITZ_HASH_KEY_SIZE_IN_BYTES 40
diff --git a/drivers/net/mana/tx.c b/drivers/net/mana/tx.c
index d35030cbe5..9401928bad 100644
--- a/drivers/net/mana/tx.c
+++ b/drivers/net/mana/tx.c
@@ -273,14 +273,31 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
struct transmit_oob_v2 tx_oob;
struct one_sgl sgl;
uint16_t seg_idx;
+ uint16_t num_sge;
+ uint16_t expected_sge;
+ bool is_tso = false;
+ uint32_t gso_hs = 0;
if (txq->desc_ring_len >= txq->num_desc)
break;
- /* Drop the packet if it exceeds max segments */
- if (m_pkt->nb_segs > priv->max_send_sge) {
+ if (m_pkt->ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
+ gso_hs = m_pkt->l2_len + m_pkt->l3_len + m_pkt->l4_len;
+ if (unlikely(m_pkt->data_len < gso_hs)) {
+ DP_LOG(ERR, "TSO header cannot be fragmented");
+ break;
+ }
+ if (likely(gso_hs != 0 && m_pkt->pkt_len > gso_hs &&
+ m_pkt->tso_segsz != 0))
+ is_tso = true;
+ }
+
+ expected_sge = m_pkt->nb_segs +
+ (is_tso && m_pkt->data_len > gso_hs ? 1 : 0);
+ if (expected_sge > priv->max_send_sge ||
+ expected_sge > MAX_SGL_ENTRIES_FOR_TRANSMIT) {
DP_LOG(ERR, "send packet segments %d exceeding max",
- m_pkt->nb_segs);
+ expected_sge);
break;
}
@@ -305,7 +322,43 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
tx_oob.short_oob.tx_compute_IP_header_checksum =
m_pkt->ol_flags & RTE_MBUF_F_TX_IP_CKSUM ? 1 : 0;
- if ((m_pkt->ol_flags & RTE_MBUF_F_TX_L4_MASK) ==
+ if (is_tso) {
+ struct rte_tcp_hdr *tcp_hdr =
+ rte_pktmbuf_mtod_offset(m_pkt,
+ struct rte_tcp_hdr *,
+ m_pkt->l2_len + m_pkt->l3_len);
+
+ if (m_pkt->ol_flags & RTE_MBUF_F_TX_IPV4) {
+ struct rte_ipv4_hdr *ip_hdr =
+ rte_pktmbuf_mtod_offset(m_pkt,
+ struct rte_ipv4_hdr *,
+ m_pkt->l2_len);
+
+ ip_hdr->total_length = 0;
+ ip_hdr->hdr_checksum = 0;
+ tcp_hdr->cksum = rte_ipv4_phdr_cksum(ip_hdr,
+ m_pkt->ol_flags);
+ } else if (m_pkt->ol_flags & RTE_MBUF_F_TX_IPV6) {
+ struct rte_ipv6_hdr *ip_hdr =
+ rte_pktmbuf_mtod_offset(m_pkt,
+ struct rte_ipv6_hdr *,
+ m_pkt->l2_len);
+
+ ip_hdr->payload_len = 0;
+ tcp_hdr->cksum = rte_ipv6_phdr_cksum(ip_hdr,
+ m_pkt->ol_flags);
+ } else {
+ DP_LOG(ERR, "Invalid IP version for TSO");
+ break;
+ }
+
+ tx_oob.short_oob.tx_compute_IP_header_checksum =
+ (m_pkt->ol_flags & RTE_MBUF_F_TX_IPV4) ? 1 : 0;
+ tx_oob.short_oob.tx_compute_TCP_checksum = 1;
+ tx_oob.short_oob.tx_compute_UDP_checksum = 0;
+ tx_oob.short_oob.tx_transport_header_offset =
+ m_pkt->l2_len + m_pkt->l3_len;
+ } else if ((m_pkt->ol_flags & RTE_MBUF_F_TX_L4_MASK) ==
RTE_MBUF_F_TX_TCP_CKSUM) {
struct rte_tcp_hdr *tcp_hdr;
@@ -405,34 +458,100 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
m_pkt->pkt_len);
/* Create SGL for packet data buffers */
- for (seg_idx = 0; seg_idx < m_pkt->nb_segs; seg_idx++) {
+ if (is_tso) {
struct mana_mr_cache *mr =
- mana_alloc_pmd_mr(&txq->mr_btree, priv, m_seg);
+ mana_alloc_pmd_mr(&txq->mr_btree, priv, m_pkt);
- if (!mr) {
- DP_LOG(ERR, "failed to get MR, pkt_idx %u",
+ if (unlikely(!mr)) {
+ DP_LOG(ERR, "failed to get MR for TSO hdr,
pkt_idx %u",
pkt_idx);
break;
}
- sgl.gdma_sgl[seg_idx].address =
- rte_cpu_to_le_64(rte_pktmbuf_mtod(m_seg,
- uint64_t));
- sgl.gdma_sgl[seg_idx].size = m_seg->data_len;
- sgl.gdma_sgl[seg_idx].memory_key = mr->lkey;
+ /* SGE0: packet header only (L2 + L3 + L4) */
+ sgl.gdma_sgl[0].address =
+ rte_cpu_to_le_64(rte_pktmbuf_mtod(m_pkt,
uint64_t));
+ sgl.gdma_sgl[0].size = gso_hs;
+ sgl.gdma_sgl[0].memory_key = mr->lkey;
+ num_sge = 1;
+
+ /* If first mbuf has payload data beyond the header */
+ if (m_pkt->data_len > gso_hs) {
+ sgl.gdma_sgl[1].address =
+ rte_cpu_to_le_64(rte_pktmbuf_mtod(m_pkt,
+
uint64_t) +
+ gso_hs);
+ sgl.gdma_sgl[1].size = m_pkt->data_len - gso_hs;
+ sgl.gdma_sgl[1].memory_key = mr->lkey;
+ num_sge = 2;
+ }
- DP_LOG(DEBUG,
- "seg idx %u addr 0x%" PRIx64 " size %x key %x",
- seg_idx, sgl.gdma_sgl[seg_idx].address,
- sgl.gdma_sgl[seg_idx].size,
- sgl.gdma_sgl[seg_idx].memory_key);
+ /* Subsequent mbuf segments */
+ m_seg = m_pkt->next;
+ while (m_seg) {
+ if (unlikely(num_sge >=
+ MAX_SGL_ENTRIES_FOR_TRANSMIT)) {
+ DP_LOG(ERR,
+ "TSO SGL entries exceed max %d",
+ MAX_SGL_ENTRIES_FOR_TRANSMIT);
+ break;
+ }
+ mr = mana_alloc_pmd_mr(&txq->mr_btree, priv,
+ m_seg);
+ if (unlikely(!mr)) {
+ DP_LOG(ERR,
+ "failed to get MR for TSO seg,
pkt_idx %u",
+ pkt_idx);
+ break;
+ }
+ sgl.gdma_sgl[num_sge].address =
+ rte_cpu_to_le_64(rte_pktmbuf_mtod(m_seg,
+
uint64_t));
+ sgl.gdma_sgl[num_sge].size = m_seg->data_len;
+ sgl.gdma_sgl[num_sge].memory_key = mr->lkey;
+ num_sge++;
+ m_seg = m_seg->next;
+ }
- m_seg = m_seg->next;
- }
+ if (unlikely(m_seg != NULL || num_sge < 2)) {
+ DP_LOG(ERR, "failed to build TSO SGL, num_sge
%u",
+ num_sge);
+ break;
+ }
+ } else {
+ for (seg_idx = 0; seg_idx < m_pkt->nb_segs; seg_idx++) {
+ struct mana_mr_cache *mr =
+ mana_alloc_pmd_mr(&txq->mr_btree, priv,
+ m_seg);
+
+ if (!mr) {
+ DP_LOG(ERR,
+ "failed to get MR, pkt_idx %u",
+ pkt_idx);
+ break;
+ }
+
+ sgl.gdma_sgl[seg_idx].address =
+ rte_cpu_to_le_64(rte_pktmbuf_mtod(m_seg,
+
uint64_t));
+ sgl.gdma_sgl[seg_idx].size = m_seg->data_len;
+ sgl.gdma_sgl[seg_idx].memory_key = mr->lkey;
+
+ DP_LOG(DEBUG,
+ "seg idx %u addr 0x%" PRIx64 " size %x
key %x",
+ seg_idx, sgl.gdma_sgl[seg_idx].address,
+ sgl.gdma_sgl[seg_idx].size,
+ sgl.gdma_sgl[seg_idx].memory_key);
+
+ m_seg = m_seg->next;
+ }
- /* Skip this packet if we can't populate all segments */
- if (seg_idx != m_pkt->nb_segs)
- break;
+ /* Skip this packet if we can't populate all segments */
+ if (seg_idx != m_pkt->nb_segs)
+ break;
+
+ num_sge = m_pkt->nb_segs;
+ }
/* If we can at least queue post two WQEs and there are at
* least two packets to send, use TX CQE suppression for the
@@ -450,7 +569,7 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
work_req.gdma_header.struct_size = sizeof(work_req);
work_req.sgl = sgl.gdma_sgl;
- work_req.num_sgl_elements = m_pkt->nb_segs;
+ work_req.num_sgl_elements = num_sge;
if (tx_oob.short_oob.packet_format == SHORT_PACKET_FORMAT)
work_req.inline_oob_size_in_bytes =
sizeof(struct transmit_short_oob_v2);
@@ -458,8 +577,14 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
work_req.inline_oob_size_in_bytes =
sizeof(struct transmit_oob_v2);
work_req.inline_oob_data = &tx_oob;
- work_req.flags = 0;
- work_req.client_data_unit = NOT_USING_CLIENT_DATA_UNIT;
+ if (is_tso) {
+ work_req.flags = GDMA_WR_OOB_IN_SGL |
+ GDMA_WR_PAD_BY_SGE0;
+ work_req.client_data_unit = m_pkt->tso_segsz;
+ } else {
+ work_req.flags = 0;
+ work_req.client_data_unit = NOT_USING_CLIENT_DATA_UNIT;
+ }
ret = gdma_post_work_request(&txq->gdma_sq, &work_req,
&wqe_size_in_bu);
--
2.43.0
