Move all preparation, packet copy and dump and transmission code in the common file. One consequence is that all debug logs are now under netdev_dpdk_common vlog.
Signed-off-by: David Marchand <[email protected]> --- lib/netdev-dpdk-common.c | 228 +++++++++++++++++++ lib/netdev-dpdk-common.h | 271 +++++++++++++++++++++++ lib/netdev-dpdk.c | 460 --------------------------------------- 3 files changed, 499 insertions(+), 460 deletions(-) diff --git a/lib/netdev-dpdk-common.c b/lib/netdev-dpdk-common.c index ee7826e150..224c41d132 100644 --- a/lib/netdev-dpdk-common.c +++ b/lib/netdev-dpdk-common.c @@ -16,6 +16,9 @@ #include <config.h> +#include <errno.h> +#include <stdarg.h> + #include <rte_malloc.h> #include <rte_mbuf.h> #include <rte_mempool.h> @@ -30,9 +33,12 @@ #include "ovs-thread.h" #include "smap.h" #include "sset.h" +#include "unaligned.h" +#include "userspace-tso.h" #include "util.h" VLOG_DEFINE_THIS_MODULE(netdev_dpdk_common); +static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20); static bool per_port_memory = false; /* Status of per port memory support */ @@ -544,6 +550,49 @@ static const struct dpdk_qos_ops *const qos_confs[] = { NULL }; +void +netdev_dpdk_common_vlog_rl(enum vlog_level level, const char *format, ...) +{ + va_list args; + + if (vlog_should_drop(&this_module, level, &rl)) { + return; + } + + va_start(args, format); + vlog_valist(&this_module, level, format, args); + va_end(args); +} + +void +netdev_dpdk_mbuf_dump(const char *prefix, const char *message, + const struct rte_mbuf *mbuf) +{ + static struct vlog_rate_limit dump_rl = VLOG_RATE_LIMIT_INIT(5, 5); + char *response = NULL; + FILE *stream; + size_t size; + + if (VLOG_DROP_DBG(&dump_rl)) { + return; + } + + stream = open_memstream(&response, &size); + if (!stream) { + VLOG_ERR("Unable to open memstream for mbuf dump: %s.", + ovs_strerror(errno)); + return; + } + + rte_pktmbuf_dump(stream, mbuf, rte_pktmbuf_pkt_len(mbuf)); + + fclose(stream); + + VLOG_DBG(prefix ? "%s: %s:\n%s" : "%s%s:\n%s", + prefix ? prefix : "", message, response); + free(response); +} + static bool srtcm_policer_pkt_handle(struct rte_meter_srtcm *meter, struct rte_meter_srtcm_profile *profile, @@ -601,6 +650,185 @@ dpdk_qos_ingress_policer_run(struct dpdk_qos_ingress_policer *policer, return cnt; } +uint32_t +netdev_dpdk_extbuf_size(uint32_t data_len) +{ + uint32_t buf_len = data_len; + + buf_len += sizeof(struct rte_mbuf_ext_shared_info) + sizeof(uintptr_t); + buf_len = RTE_ALIGN_CEIL(buf_len, sizeof(uintptr_t)); + + return buf_len; +} + +void * +netdev_dpdk_extbuf_allocate(uint32_t buf_len) +{ + return rte_malloc(NULL, buf_len, RTE_CACHE_LINE_SIZE); +} + +static void +netdev_dpdk_extbuf_free(void *addr OVS_UNUSED, void *opaque) +{ + rte_free(opaque); +} + +void +netdev_dpdk_extbuf_replace(struct dp_packet *b, void *buf, uint32_t data_len) +{ + struct rte_mbuf *pkt = (struct rte_mbuf *) b; + struct rte_mbuf_ext_shared_info *shinfo; + uint16_t buf_len = data_len; + + shinfo = rte_pktmbuf_ext_shinfo_init_helper(buf, &buf_len, + netdev_dpdk_extbuf_free, + buf); + ovs_assert(shinfo != NULL); + + if (RTE_MBUF_HAS_EXTBUF(pkt)) { + rte_pktmbuf_detach_extbuf(pkt); + } + rte_pktmbuf_attach_extbuf(pkt, buf, rte_malloc_virt2iova(buf), buf_len, + shinfo); + /* OVS only supports mono segment. + * Packet size did not change, restore the current segment length. */ + pkt->data_len = pkt->pkt_len; +} + +static struct rte_mbuf * +dpdk_pktmbuf_attach_extbuf(struct rte_mbuf *pkt, uint32_t data_len) +{ + uint32_t total_len = RTE_PKTMBUF_HEADROOM + data_len; + struct rte_mbuf_ext_shared_info *shinfo = NULL; + uint16_t buf_len; + void *buf; + + total_len = netdev_dpdk_extbuf_size(total_len); + if (OVS_UNLIKELY(total_len > UINT16_MAX)) { + VLOG_ERR("Can't copy packet: too big %u", total_len); + return NULL; + } + + buf_len = total_len; + buf = netdev_dpdk_extbuf_allocate(buf_len); + if (OVS_UNLIKELY(buf == NULL)) { + VLOG_ERR("Failed to allocate memory using rte_malloc: %u", buf_len); + return NULL; + } + + /* Initialize shinfo. */ + shinfo = rte_pktmbuf_ext_shinfo_init_helper(buf, &buf_len, + netdev_dpdk_extbuf_free, + buf); + if (OVS_UNLIKELY(shinfo == NULL)) { + netdev_dpdk_extbuf_free(NULL, buf); + VLOG_ERR("Failed to initialize shared info for mbuf while " + "attempting to attach an external buffer."); + return NULL; + } + + rte_pktmbuf_attach_extbuf(pkt, buf, rte_malloc_virt2iova(buf), buf_len, + shinfo); + rte_pktmbuf_reset_headroom(pkt); + + return pkt; +} + +struct rte_mbuf * +dpdk_pktmbuf_alloc(struct rte_mempool *mp, uint32_t data_len) +{ + struct rte_mbuf *pkt = rte_pktmbuf_alloc(mp); + + if (OVS_UNLIKELY(!pkt)) { + return NULL; + } + + if (rte_pktmbuf_tailroom(pkt) >= data_len) { + return pkt; + } + + if (dpdk_pktmbuf_attach_extbuf(pkt, data_len)) { + return pkt; + } + + rte_pktmbuf_free(pkt); + + return NULL; +} + +struct dp_packet * +dpdk_copy_dp_packet_to_mbuf(struct rte_mempool *mp, struct dp_packet *pkt_orig) +{ + struct rte_mbuf *mbuf_dest; + struct dp_packet *pkt_dest; + uint32_t pkt_len; + + pkt_len = dp_packet_size(pkt_orig); + mbuf_dest = dpdk_pktmbuf_alloc(mp, pkt_len); + if (OVS_UNLIKELY(mbuf_dest == NULL)) { + return NULL; + } + + pkt_dest = CONTAINER_OF(mbuf_dest, struct dp_packet, mbuf); + memcpy(dp_packet_data(pkt_dest), dp_packet_data(pkt_orig), pkt_len); + dp_packet_set_size(pkt_dest, pkt_len); + + mbuf_dest->tx_offload = pkt_orig->mbuf.tx_offload; + mbuf_dest->packet_type = pkt_orig->mbuf.packet_type; + mbuf_dest->ol_flags |= (pkt_orig->mbuf.ol_flags & + ~(RTE_MBUF_F_EXTERNAL | RTE_MBUF_F_INDIRECT)); + mbuf_dest->tso_segsz = pkt_orig->mbuf.tso_segsz; + + memcpy(&pkt_dest->l2_pad_size, &pkt_orig->l2_pad_size, + sizeof(struct dp_packet) - offsetof(struct dp_packet, l2_pad_size)); + + if (dp_packet_l3(pkt_dest)) { + if (dp_packet_eth(pkt_dest)) { + mbuf_dest->l2_len = (char *) dp_packet_l3(pkt_dest) + - (char *) dp_packet_eth(pkt_dest); + } else { + mbuf_dest->l2_len = 0; + } + if (dp_packet_l4(pkt_dest)) { + mbuf_dest->l3_len = (char *) dp_packet_l4(pkt_dest) + - (char *) dp_packet_l3(pkt_dest); + } else { + mbuf_dest->l3_len = 0; + } + } + + return pkt_dest; +} + +/* Replace packets in a 'batch' with their corresponding copies using + * DPDK memory. + * + * Returns the number of good packets in the batch. */ +size_t +dpdk_copy_batch_to_mbuf(struct netdev_dpdk_common *common, + struct dp_packet_batch *batch) +{ + size_t i, size = dp_packet_batch_size(batch); + struct dp_packet *packet; + + DP_PACKET_BATCH_REFILL_FOR_EACH (i, size, packet, batch) { + if (OVS_UNLIKELY(packet->source == DPBUF_DPDK)) { + dp_packet_batch_add(batch, packet); + } else { + struct dp_packet *pktcopy; + + pktcopy = dpdk_copy_dp_packet_to_mbuf(common->dpdk_mp->mp, packet); + if (pktcopy) { + dp_packet_batch_add(batch, pktcopy); + } + + dp_packet_delete(packet); + } + } + + return dp_packet_batch_size(batch); +} + struct dpdk_qos_ingress_policer * dpdk_qos_ingress_policer_construct(uint32_t rate, uint32_t burst) { diff --git a/lib/netdev-dpdk-common.h b/lib/netdev-dpdk-common.h index 1c63a5bc78..8e8d025a2e 100644 --- a/lib/netdev-dpdk-common.h +++ b/lib/netdev-dpdk-common.h @@ -27,8 +27,10 @@ #include <rte_ethdev.h> #include <rte_mempool.h> +#include "dp-packet.h" #include "netdev-provider.h" #include "openvswitch/compiler.h" +#include "openvswitch/vlog.h" #include "ovs-thread.h" #include "packets.h" @@ -327,4 +329,273 @@ int netdev_dpdk_common_queue_dump_next(const struct netdev_dpdk_common *common, int netdev_dpdk_common_queue_dump_done(const struct netdev *netdev, void *state_); +void netdev_dpdk_mbuf_dump(const char *prefix, const char *message, + const struct rte_mbuf *mbuf); + +void netdev_dpdk_common_vlog_rl(enum vlog_level, const char *format, ...) + OVS_PRINTF_FORMAT(2, 3); + +uint32_t netdev_dpdk_extbuf_size(uint32_t data_len); +void *netdev_dpdk_extbuf_allocate(uint32_t buf_len); +void netdev_dpdk_extbuf_replace(struct dp_packet *b, void *buf, + uint32_t data_len); + +struct rte_mbuf *dpdk_pktmbuf_alloc(struct rte_mempool *mp, uint32_t data_len); +struct dp_packet *dpdk_copy_dp_packet_to_mbuf(struct rte_mempool *mp, + struct dp_packet *pkt_orig); +size_t dpdk_copy_batch_to_mbuf(struct netdev_dpdk_common *common, + struct dp_packet_batch *batch); + +/* Prepare the packet for HWOL. + * Return True if the packet is OK to continue. */ +static inline bool +netdev_dpdk_prep_hwol_packet(struct netdev_dpdk_common *common, + struct rte_mbuf *mbuf) +{ + struct dp_packet *pkt = CONTAINER_OF(mbuf, struct dp_packet, mbuf); + uint64_t unexpected = mbuf->ol_flags & RTE_MBUF_F_TX_OFFLOAD_MASK; + struct netdev *netdev = &common->up; + const struct ip_header *ip; + bool is_sctp; + bool l3_csum; + bool l4_csum; + bool is_tcp; + bool is_udp; + void *l2; + void *l3; + void *l4; + + if (OVS_UNLIKELY(unexpected)) { + netdev_dpdk_common_vlog_rl(VLL_WARN, + "%s: Unexpected Tx offload flags: %#"PRIx64, + netdev_get_name(netdev), unexpected); + netdev_dpdk_mbuf_dump(netdev_get_name(netdev), + "Packet with unexpected ol_flags", mbuf); + return false; + } + + if (!dp_packet_ip_checksum_partial(pkt) + && !dp_packet_inner_ip_checksum_partial(pkt) + && !dp_packet_l4_checksum_partial(pkt) + && !dp_packet_inner_l4_checksum_partial(pkt) + && !mbuf->tso_segsz) { + + return true; + } + + if (dp_packet_tunnel(pkt)) { + mbuf->outer_l2_len = (char *) dp_packet_l3(pkt) - + (char *) dp_packet_eth(pkt); + mbuf->outer_l3_len = (char *) dp_packet_l4(pkt) - + (char *) dp_packet_l3(pkt); + + if (dp_packet_tunnel_geneve(pkt)) { + mbuf->ol_flags |= RTE_MBUF_F_TX_TUNNEL_GENEVE; + } else if (dp_packet_tunnel_vxlan(pkt)) { + mbuf->ol_flags |= RTE_MBUF_F_TX_TUNNEL_VXLAN; + } else { + ovs_assert(dp_packet_tunnel_gre(pkt)); + mbuf->ol_flags |= RTE_MBUF_F_TX_TUNNEL_GRE; + } + + if (dp_packet_ip_checksum_partial(pkt)) { + mbuf->ol_flags |= RTE_MBUF_F_TX_OUTER_IP_CKSUM; + } + + if (dp_packet_l4_checksum_partial(pkt)) { + ovs_assert(dp_packet_l4_proto_udp(pkt)); + mbuf->ol_flags |= RTE_MBUF_F_TX_OUTER_UDP_CKSUM; + } + + ip = dp_packet_l3(pkt); + mbuf->ol_flags |= IP_VER(ip->ip_ihl_ver) == 4 + ? RTE_MBUF_F_TX_OUTER_IPV4 + : RTE_MBUF_F_TX_OUTER_IPV6; + + /* Inner L2 length must account for the tunnel header length. */ + l2 = dp_packet_l4(pkt); + l3 = dp_packet_inner_l3(pkt); + l3_csum = dp_packet_inner_ip_checksum_partial(pkt); + l4 = dp_packet_inner_l4(pkt); + l4_csum = dp_packet_inner_l4_checksum_partial(pkt); + is_tcp = dp_packet_inner_l4_proto_tcp(pkt); + is_udp = dp_packet_inner_l4_proto_udp(pkt); + is_sctp = dp_packet_inner_l4_proto_sctp(pkt); + } else { + mbuf->outer_l2_len = 0; + mbuf->outer_l3_len = 0; + + l2 = dp_packet_eth(pkt); + l3 = dp_packet_l3(pkt); + l3_csum = dp_packet_ip_checksum_partial(pkt); + l4 = dp_packet_l4(pkt); + l4_csum = dp_packet_l4_checksum_partial(pkt); + is_tcp = dp_packet_l4_proto_tcp(pkt); + is_udp = dp_packet_l4_proto_udp(pkt); + is_sctp = dp_packet_l4_proto_sctp(pkt); + } + + ovs_assert(l4); + + ip = l3; + mbuf->ol_flags |= IP_VER(ip->ip_ihl_ver) == 4 + ? RTE_MBUF_F_TX_IPV4 : RTE_MBUF_F_TX_IPV6; + + if (l3_csum) { + mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM; + } + + if (l4_csum) { + if (is_tcp) { + mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM; + } else if (is_udp) { + mbuf->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM; + } else { + ovs_assert(is_sctp); + mbuf->ol_flags |= RTE_MBUF_F_TX_SCTP_CKSUM; + } + } + + mbuf->l2_len = (char *) l3 - (char *) l2; + mbuf->l3_len = (char *) l4 - (char *) l3; + + if (mbuf->tso_segsz) { + struct tcp_header *th = l4; + int hdr_len; + + mbuf->l4_len = TCP_OFFSET(th->tcp_ctl) * 4; + + hdr_len = mbuf->l2_len + mbuf->l3_len + mbuf->l4_len; + if (dp_packet_tunnel(pkt)) { + hdr_len += mbuf->outer_l2_len + mbuf->outer_l3_len; + } + + if (OVS_UNLIKELY((hdr_len + mbuf->tso_segsz) + > common->max_packet_len)) { + netdev_dpdk_common_vlog_rl(VLL_WARN, + "%s: Oversized TSO packet. hdr: %"PRIu32 + ", gso: %"PRIu32", max len: %"PRIu32"", + netdev->name, hdr_len, mbuf->tso_segsz, + common->max_packet_len); + return false; + } + mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_SEG; + + /* DPDK API mandates IPv4 checksum when requesting TSO. */ + if (IP_VER(ip->ip_ihl_ver) == 4) { + mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM; + } + } + + return true; +} + +/* Prepare a batch for HWOL. + * Return the number of good packets in the batch. */ +static inline int +netdev_dpdk_prep_hwol_batch(struct netdev_dpdk_common *common, + struct rte_mbuf **pkts, int pkt_cnt) +{ + int i = 0; + int cnt = 0; + struct rte_mbuf *pkt; + + /* Prepare and filter bad HWOL packets. */ + for (i = 0; i < pkt_cnt; i++) { + pkt = pkts[i]; + if (!netdev_dpdk_prep_hwol_packet(common, pkt)) { + rte_pktmbuf_free(pkt); + continue; + } + + if (OVS_UNLIKELY(i != cnt)) { + pkts[cnt] = pkt; + } + cnt++; + } + + return cnt; +} + +static inline int +netdev_dpdk_filter_packet_len(struct netdev_dpdk_common *common, + struct rte_mbuf **pkts, int pkt_cnt) +{ + int i = 0; + int cnt = 0; + struct rte_mbuf *pkt; + + /* Filter oversized packets. The TSO packets are filtered out + * during the offloading preparation for performance reasons. */ + for (i = 0; i < pkt_cnt; i++) { + pkt = pkts[i]; + if (OVS_UNLIKELY((pkt->pkt_len > common->max_packet_len) + && !pkt->tso_segsz)) { + netdev_dpdk_common_vlog_rl(VLL_WARN, + "%s: Too big size %" PRIu32 + " max_packet_len %d", + common->up.name, pkt->pkt_len, + common->max_packet_len); + rte_pktmbuf_free(pkt); + continue; + } + + if (OVS_UNLIKELY(i != cnt)) { + pkts[cnt] = pkt; + } + cnt++; + } + + return cnt; +} + +static inline size_t +netdev_dpdk_common_send(struct netdev_dpdk_common *common, + struct dp_packet_batch *batch, + struct netdev_dpdk_sw_stats *stats) +{ + struct rte_mbuf **pkts = (struct rte_mbuf **) batch->packets; + size_t cnt, pkt_cnt = dp_packet_batch_size(batch); + struct dpdk_qos_conf *qos_conf; + struct dp_packet *packet; + bool need_copy = false; + + memset(stats, 0, sizeof *stats); + + DP_PACKET_BATCH_FOR_EACH (i, packet, batch) { + if (packet->source != DPBUF_DPDK) { + need_copy = true; + break; + } + } + + /* Copy dp-packets to mbufs. */ + if (OVS_UNLIKELY(need_copy)) { + cnt = dpdk_copy_batch_to_mbuf(common, batch); + stats->tx_failure_drops += pkt_cnt - cnt; + pkt_cnt = cnt; + } + + /* Drop oversized packets. */ + cnt = netdev_dpdk_filter_packet_len(common, pkts, pkt_cnt); + stats->tx_mtu_exceeded_drops += pkt_cnt - cnt; + pkt_cnt = cnt; + + if (common->up.ol_flags) { + /* Prepare each mbuf for hardware offloading. */ + cnt = netdev_dpdk_prep_hwol_batch(common, pkts, pkt_cnt); + stats->tx_invalid_hwol_drops += pkt_cnt - cnt; + pkt_cnt = cnt; + } + + /* Apply Quality of Service policy. */ + qos_conf = ovsrcu_get(struct dpdk_qos_conf *, &common->qos_conf); + if (qos_conf) { + cnt = dpdk_qos_run(qos_conf, pkts, pkt_cnt, true); + stats->tx_qos_drops += pkt_cnt - cnt; + } + + return cnt; +} + #endif /* NETDEV_DPDK_COMMON_H */ diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c index 05ac3c3a8c..3f46f164e5 100644 --- a/lib/netdev-dpdk.c +++ b/lib/netdev-dpdk.c @@ -362,9 +362,6 @@ static void netdev_dpdk_clear_xstats(struct netdev_dpdk *dev); int netdev_dpdk_get_vid(const struct netdev_dpdk *dev); -static void netdev_dpdk_mbuf_dump(const char *prefix, const char *message, - const struct rte_mbuf *); - static bool is_eth_class(const struct netdev_class *class) { @@ -2043,204 +2040,6 @@ netdev_dpdk_batch_init_packet_fields(struct dp_packet_batch *batch) } } -/* Prepare the packet for HWOL. - * Return True if the packet is OK to continue. */ -static bool -netdev_dpdk_prep_hwol_packet(struct netdev_dpdk_common *common, - struct rte_mbuf *mbuf) -{ - struct dp_packet *pkt = CONTAINER_OF(mbuf, struct dp_packet, mbuf); - uint64_t unexpected = mbuf->ol_flags & RTE_MBUF_F_TX_OFFLOAD_MASK; - struct netdev *netdev = &common->up; - const struct ip_header *ip; - bool is_sctp; - bool l3_csum; - bool l4_csum; - bool is_tcp; - bool is_udp; - void *l2; - void *l3; - void *l4; - - if (OVS_UNLIKELY(unexpected)) { - VLOG_WARN_RL(&rl, "%s: Unexpected Tx offload flags: %#"PRIx64, - netdev_get_name(netdev), unexpected); - netdev_dpdk_mbuf_dump(netdev_get_name(netdev), - "Packet with unexpected ol_flags", mbuf); - return false; - } - - if (!dp_packet_ip_checksum_partial(pkt) - && !dp_packet_inner_ip_checksum_partial(pkt) - && !dp_packet_l4_checksum_partial(pkt) - && !dp_packet_inner_l4_checksum_partial(pkt) - && !mbuf->tso_segsz) { - - return true; - } - - if (dp_packet_tunnel(pkt)) { - mbuf->outer_l2_len = (char *) dp_packet_l3(pkt) - - (char *) dp_packet_eth(pkt); - mbuf->outer_l3_len = (char *) dp_packet_l4(pkt) - - (char *) dp_packet_l3(pkt); - - if (dp_packet_tunnel_geneve(pkt)) { - mbuf->ol_flags |= RTE_MBUF_F_TX_TUNNEL_GENEVE; - } else if (dp_packet_tunnel_vxlan(pkt)) { - mbuf->ol_flags |= RTE_MBUF_F_TX_TUNNEL_VXLAN; - } else { - ovs_assert(dp_packet_tunnel_gre(pkt)); - mbuf->ol_flags |= RTE_MBUF_F_TX_TUNNEL_GRE; - } - - if (dp_packet_ip_checksum_partial(pkt)) { - mbuf->ol_flags |= RTE_MBUF_F_TX_OUTER_IP_CKSUM; - } - - if (dp_packet_l4_checksum_partial(pkt)) { - ovs_assert(dp_packet_l4_proto_udp(pkt)); - mbuf->ol_flags |= RTE_MBUF_F_TX_OUTER_UDP_CKSUM; - } - - ip = dp_packet_l3(pkt); - mbuf->ol_flags |= IP_VER(ip->ip_ihl_ver) == 4 - ? RTE_MBUF_F_TX_OUTER_IPV4 - : RTE_MBUF_F_TX_OUTER_IPV6; - - /* Inner L2 length must account for the tunnel header length. */ - l2 = dp_packet_l4(pkt); - l3 = dp_packet_inner_l3(pkt); - l3_csum = dp_packet_inner_ip_checksum_partial(pkt); - l4 = dp_packet_inner_l4(pkt); - l4_csum = dp_packet_inner_l4_checksum_partial(pkt); - is_tcp = dp_packet_inner_l4_proto_tcp(pkt); - is_udp = dp_packet_inner_l4_proto_udp(pkt); - is_sctp = dp_packet_inner_l4_proto_sctp(pkt); - } else { - mbuf->outer_l2_len = 0; - mbuf->outer_l3_len = 0; - - l2 = dp_packet_eth(pkt); - l3 = dp_packet_l3(pkt); - l3_csum = dp_packet_ip_checksum_partial(pkt); - l4 = dp_packet_l4(pkt); - l4_csum = dp_packet_l4_checksum_partial(pkt); - is_tcp = dp_packet_l4_proto_tcp(pkt); - is_udp = dp_packet_l4_proto_udp(pkt); - is_sctp = dp_packet_l4_proto_sctp(pkt); - } - - ovs_assert(l4); - - ip = l3; - mbuf->ol_flags |= IP_VER(ip->ip_ihl_ver) == 4 - ? RTE_MBUF_F_TX_IPV4 : RTE_MBUF_F_TX_IPV6; - - if (l3_csum) { - mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM; - } - - if (l4_csum) { - if (is_tcp) { - mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM; - } else if (is_udp) { - mbuf->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM; - } else { - ovs_assert(is_sctp); - mbuf->ol_flags |= RTE_MBUF_F_TX_SCTP_CKSUM; - } - } - - mbuf->l2_len = (char *) l3 - (char *) l2; - mbuf->l3_len = (char *) l4 - (char *) l3; - - if (mbuf->tso_segsz) { - struct tcp_header *th = l4; - int hdr_len; - - mbuf->l4_len = TCP_OFFSET(th->tcp_ctl) * 4; - - hdr_len = mbuf->l2_len + mbuf->l3_len + mbuf->l4_len; - if (dp_packet_tunnel(pkt)) { - hdr_len += mbuf->outer_l2_len + mbuf->outer_l3_len; - } - - if (OVS_UNLIKELY((hdr_len + mbuf->tso_segsz) - > common->max_packet_len)) { - VLOG_WARN_RL(&rl, "%s: Oversized TSO packet. hdr: %"PRIu32", " - "gso: %"PRIu32", max len: %"PRIu32"", - netdev->name, hdr_len, mbuf->tso_segsz, - common->max_packet_len); - return false; - } - mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_SEG; - - /* DPDK API mandates IPv4 checksum when requesting TSO. */ - if (IP_VER(ip->ip_ihl_ver) == 4) { - mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM; - } - } - - return true; -} - -/* Prepare a batch for HWOL. - * Return the number of good packets in the batch. */ -static int -netdev_dpdk_prep_hwol_batch(struct netdev_dpdk_common *common, - struct rte_mbuf **pkts, int pkt_cnt) -{ - int i = 0; - int cnt = 0; - struct rte_mbuf *pkt; - - /* Prepare and filter bad HWOL packets. */ - for (i = 0; i < pkt_cnt; i++) { - pkt = pkts[i]; - if (!netdev_dpdk_prep_hwol_packet(common, pkt)) { - rte_pktmbuf_free(pkt); - continue; - } - - if (OVS_UNLIKELY(i != cnt)) { - pkts[cnt] = pkt; - } - cnt++; - } - - return cnt; -} - -static void -netdev_dpdk_mbuf_dump(const char *prefix, const char *message, - const struct rte_mbuf *mbuf) -{ - static struct vlog_rate_limit dump_rl = VLOG_RATE_LIMIT_INIT(5, 5); - char *response = NULL; - FILE *stream; - size_t size; - - if (VLOG_DROP_DBG(&dump_rl)) { - return; - } - - stream = open_memstream(&response, &size); - if (!stream) { - VLOG_ERR("Unable to open memstream for mbuf dump: %s.", - ovs_strerror(errno)); - return; - } - - rte_pktmbuf_dump(stream, mbuf, rte_pktmbuf_pkt_len(mbuf)); - - fclose(stream); - - VLOG_DBG(prefix ? "%s: %s:\n%s" : "%s%s:\n%s", - prefix ? prefix : "", message, response); - free(response); -} - /* Tries to transmit 'pkts' to txq 'qid' of device 'dev'. Takes ownership of * 'pkts', even in case of failure. * @@ -2409,265 +2208,6 @@ netdev_dpdk_eth_rxq_recv(struct netdev_rxq *rxq, struct dp_packet_batch *batch, return 0; } -static int -netdev_dpdk_filter_packet_len(struct netdev_dpdk_common *common, - struct rte_mbuf **pkts, int pkt_cnt) -{ - int i = 0; - int cnt = 0; - struct rte_mbuf *pkt; - - /* Filter oversized packets. The TSO packets are filtered out - * during the offloading preparation for performance reasons. */ - for (i = 0; i < pkt_cnt; i++) { - pkt = pkts[i]; - if (OVS_UNLIKELY((pkt->pkt_len > common->max_packet_len) - && !pkt->tso_segsz)) { - VLOG_WARN_RL(&rl, "%s: Too big size %" PRIu32 " " - "max_packet_len %d", common->up.name, pkt->pkt_len, - common->max_packet_len); - rte_pktmbuf_free(pkt); - continue; - } - - if (OVS_UNLIKELY(i != cnt)) { - pkts[cnt] = pkt; - } - cnt++; - } - - return cnt; -} - -uint32_t -netdev_dpdk_extbuf_size(uint32_t data_len) -{ - uint32_t buf_len = data_len; - - buf_len += sizeof(struct rte_mbuf_ext_shared_info) + sizeof(uintptr_t); - buf_len = RTE_ALIGN_CEIL(buf_len, sizeof(uintptr_t)); - - return buf_len; -} - -void * -netdev_dpdk_extbuf_allocate(uint32_t buf_len) -{ - return rte_malloc(NULL, buf_len, RTE_CACHE_LINE_SIZE); -} - -static void -netdev_dpdk_extbuf_free(void *addr OVS_UNUSED, void *opaque) -{ - rte_free(opaque); -} - -void -netdev_dpdk_extbuf_replace(struct dp_packet *b, void *buf, uint32_t data_len) -{ - struct rte_mbuf *pkt = (struct rte_mbuf *) b; - struct rte_mbuf_ext_shared_info *shinfo; - uint16_t buf_len = data_len; - - shinfo = rte_pktmbuf_ext_shinfo_init_helper(buf, &buf_len, - netdev_dpdk_extbuf_free, - buf); - ovs_assert(shinfo != NULL); - - if (RTE_MBUF_HAS_EXTBUF(pkt)) { - rte_pktmbuf_detach_extbuf(pkt); - } - rte_pktmbuf_attach_extbuf(pkt, buf, rte_malloc_virt2iova(buf), buf_len, - shinfo); - /* OVS only supports mono segment. - * Packet size did not change, restore the current segment length. */ - pkt->data_len = pkt->pkt_len; -} - -static struct rte_mbuf * -dpdk_pktmbuf_attach_extbuf(struct rte_mbuf *pkt, uint32_t data_len) -{ - uint32_t total_len = RTE_PKTMBUF_HEADROOM + data_len; - struct rte_mbuf_ext_shared_info *shinfo = NULL; - uint16_t buf_len; - void *buf; - - total_len = netdev_dpdk_extbuf_size(total_len); - if (OVS_UNLIKELY(total_len > UINT16_MAX)) { - VLOG_ERR("Can't copy packet: too big %u", total_len); - return NULL; - } - - buf_len = total_len; - buf = netdev_dpdk_extbuf_allocate(buf_len); - if (OVS_UNLIKELY(buf == NULL)) { - VLOG_ERR("Failed to allocate memory using rte_malloc: %u", buf_len); - return NULL; - } - - /* Initialize shinfo. */ - shinfo = rte_pktmbuf_ext_shinfo_init_helper(buf, &buf_len, - netdev_dpdk_extbuf_free, - buf); - if (OVS_UNLIKELY(shinfo == NULL)) { - netdev_dpdk_extbuf_free(NULL, buf); - VLOG_ERR("Failed to initialize shared info for mbuf while " - "attempting to attach an external buffer."); - return NULL; - } - - rte_pktmbuf_attach_extbuf(pkt, buf, rte_malloc_virt2iova(buf), buf_len, - shinfo); - rte_pktmbuf_reset_headroom(pkt); - - return pkt; -} - -static struct rte_mbuf * -dpdk_pktmbuf_alloc(struct rte_mempool *mp, uint32_t data_len) -{ - struct rte_mbuf *pkt = rte_pktmbuf_alloc(mp); - - if (OVS_UNLIKELY(!pkt)) { - return NULL; - } - - if (rte_pktmbuf_tailroom(pkt) >= data_len) { - return pkt; - } - - if (dpdk_pktmbuf_attach_extbuf(pkt, data_len)) { - return pkt; - } - - rte_pktmbuf_free(pkt); - - return NULL; -} - -static struct dp_packet * -dpdk_copy_dp_packet_to_mbuf(struct rte_mempool *mp, struct dp_packet *pkt_orig) -{ - struct rte_mbuf *mbuf_dest; - struct dp_packet *pkt_dest; - uint32_t pkt_len; - - pkt_len = dp_packet_size(pkt_orig); - mbuf_dest = dpdk_pktmbuf_alloc(mp, pkt_len); - if (OVS_UNLIKELY(mbuf_dest == NULL)) { - return NULL; - } - - pkt_dest = CONTAINER_OF(mbuf_dest, struct dp_packet, mbuf); - memcpy(dp_packet_data(pkt_dest), dp_packet_data(pkt_orig), pkt_len); - dp_packet_set_size(pkt_dest, pkt_len); - - mbuf_dest->tx_offload = pkt_orig->mbuf.tx_offload; - mbuf_dest->packet_type = pkt_orig->mbuf.packet_type; - mbuf_dest->ol_flags |= (pkt_orig->mbuf.ol_flags & - ~(RTE_MBUF_F_EXTERNAL | RTE_MBUF_F_INDIRECT)); - mbuf_dest->tso_segsz = pkt_orig->mbuf.tso_segsz; - - memcpy(&pkt_dest->l2_pad_size, &pkt_orig->l2_pad_size, - sizeof(struct dp_packet) - offsetof(struct dp_packet, l2_pad_size)); - - if (dp_packet_l3(pkt_dest)) { - if (dp_packet_eth(pkt_dest)) { - mbuf_dest->l2_len = (char *) dp_packet_l3(pkt_dest) - - (char *) dp_packet_eth(pkt_dest); - } else { - mbuf_dest->l2_len = 0; - } - if (dp_packet_l4(pkt_dest)) { - mbuf_dest->l3_len = (char *) dp_packet_l4(pkt_dest) - - (char *) dp_packet_l3(pkt_dest); - } else { - mbuf_dest->l3_len = 0; - } - } - - return pkt_dest; -} - -/* Replace packets in a 'batch' with their corresponding copies using - * DPDK memory. - * - * Returns the number of good packets in the batch. */ -static size_t -dpdk_copy_batch_to_mbuf(struct netdev *netdev, struct dp_packet_batch *batch) -{ - struct netdev_dpdk_common *common = netdev_dpdk_common_cast(netdev); - size_t i, size = dp_packet_batch_size(batch); - struct dp_packet *packet; - - DP_PACKET_BATCH_REFILL_FOR_EACH (i, size, packet, batch) { - if (OVS_UNLIKELY(packet->source == DPBUF_DPDK)) { - dp_packet_batch_add(batch, packet); - } else { - struct dp_packet *pktcopy; - - pktcopy = dpdk_copy_dp_packet_to_mbuf(common->dpdk_mp->mp, packet); - if (pktcopy) { - dp_packet_batch_add(batch, pktcopy); - } - - dp_packet_delete(packet); - } - } - - return dp_packet_batch_size(batch); -} - -static size_t -netdev_dpdk_common_send(struct netdev_dpdk_common *common, - struct dp_packet_batch *batch, - struct netdev_dpdk_sw_stats *stats) -{ - struct rte_mbuf **pkts = (struct rte_mbuf **) batch->packets; - size_t cnt, pkt_cnt = dp_packet_batch_size(batch); - struct netdev *netdev = &common->up; - struct dpdk_qos_conf *qos_conf; - struct dp_packet *packet; - bool need_copy = false; - - memset(stats, 0, sizeof *stats); - - DP_PACKET_BATCH_FOR_EACH (i, packet, batch) { - if (packet->source != DPBUF_DPDK) { - need_copy = true; - break; - } - } - - /* Copy dp-packets to mbufs. */ - if (OVS_UNLIKELY(need_copy)) { - cnt = dpdk_copy_batch_to_mbuf(netdev, batch); - stats->tx_failure_drops += pkt_cnt - cnt; - pkt_cnt = cnt; - } - - /* Drop oversized packets. */ - cnt = netdev_dpdk_filter_packet_len(common, pkts, pkt_cnt); - stats->tx_mtu_exceeded_drops += pkt_cnt - cnt; - pkt_cnt = cnt; - - if (netdev->ol_flags) { - /* Prepare each mbuf for hardware offloading. */ - cnt = netdev_dpdk_prep_hwol_batch(common, pkts, pkt_cnt); - stats->tx_invalid_hwol_drops += pkt_cnt - cnt; - pkt_cnt = cnt; - } - - /* Apply Quality of Service policy. */ - qos_conf = ovsrcu_get(struct dpdk_qos_conf *, &common->qos_conf); - if (qos_conf) { - cnt = dpdk_qos_run(qos_conf, pkts, pkt_cnt, true); - stats->tx_qos_drops += pkt_cnt - cnt; - } - - return cnt; -} - static int netdev_dpdk_vhost_send(struct netdev *netdev, int qid, struct dp_packet_batch *batch, -- 2.54.0 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
