Re: [ovs-dev] [PATCH v2 2/3] netdev-dpdk: Consider packets marked for TSO.
Hi Ian,
Agreed with all that's said below and will work on the changes, just a
small comment in-line.
On 11/01/2019 12:26, Ian Stokes wrote:
> On 1/10/2019 4:58 PM, Tiago Lam wrote:
>> Previously, TSO was being explicity disabled on vhost interfaces,
>> meaning the guests wouldn't have TSO support negotiated in. With TSO
>> negotiated and enabled, packets are now marked for TSO, through the
>> PKT_TX_TCP_SEG flag.
>>
>> In order to deal with this type of packets, a new function,
>> netdev_dpdk_prep_tso_packet(), has been introduced, with the main
>> purpose of setting correctly the l2, l3 and l4 length members of the
>> mbuf struct, and the appropriate ol_flags. This function supports TSO
>> both in IPv4 and IPv6.
>>
>> netdev_dpdk_prep_tso_packet() is then only called when packets are
>> marked with the PKT_TX_TCP_SEG flag, meaning they have been marked for
>> TSO, and when the packet will be traversing the NIC.
>>
>> Additionally, if a packet is marked for TSO but the egress netdev
>> doesn't support it, the packet is dropped.
>>
>
> Hi Tiago,
>
> a high level first pass, I still need to test this in more detail so not
> a full review yet but some minor issues to be addressed below.
>
>> Co-authored-by: Mark Kavanagh
>>
>> Signed-off-by: Mark Kavanagh
>> Signed-off-by: Tiago Lam
>> ---
>> lib/dp-packet.h| 14 +++
>> lib/netdev-bsd.c | 11 -
>> lib/netdev-dpdk.c | 121
>> ++---
>> lib/netdev-dummy.c | 11 -
>> lib/netdev-linux.c | 15 +++
>> 5 files changed, 146 insertions(+), 26 deletions(-)
>>
>> diff --git a/lib/dp-packet.h b/lib/dp-packet.h
>> index 970aaf2..c384416 100644
>> --- a/lib/dp-packet.h
>> +++ b/lib/dp-packet.h
>> @@ -104,6 +104,8 @@ static inline void dp_packet_set_size(struct dp_packet
>> *, uint32_t);
>> static inline uint16_t dp_packet_get_allocated(const struct dp_packet *);
>> static inline void dp_packet_set_allocated(struct dp_packet *, uint16_t);
>>
>> +static inline bool dp_packet_is_tso(struct dp_packet *b);
>> +
>> void *dp_packet_resize_l2(struct dp_packet *, int increment);
>> void *dp_packet_resize_l2_5(struct dp_packet *, int increment);
>> static inline void *dp_packet_eth(const struct dp_packet *);
>> @@ -761,6 +763,12 @@ dp_packet_set_allocated(struct dp_packet *b, uint16_t s)
>> b->mbuf.buf_len = s;
>> }
>>
>> +static inline bool
>> +dp_packet_is_tso(struct dp_packet *b)
>> +{
>> +return (b->mbuf.ol_flags & PKT_TX_TCP_SEG) ? true : false;
>> +}
>> +
>> static inline void
>> dp_packet_copy_mbuf_flags(struct dp_packet *dst, const struct dp_packet
>> *src)
>> {
>> @@ -972,6 +980,12 @@ dp_packet_get_allocated(const struct dp_packet *b)
>> return b->allocated_;
>> }
>>
>> +static inline bool
>> +dp_packet_is_tso(struct dp_packet *b)
>> +{
> This will cause a unused parameter warning. Need to pass OVS_UNUSED
> above also.
>
>> +return false;
>> +}
>> +
>> static inline void
>> dp_packet_set_allocated(struct dp_packet *b, uint16_t s)
>> {
>> diff --git a/lib/netdev-bsd.c b/lib/netdev-bsd.c
>> index 278c8a9..5e8c5cc 100644
>> --- a/lib/netdev-bsd.c
>> +++ b/lib/netdev-bsd.c
>> @@ -700,13 +700,22 @@ netdev_bsd_send(struct netdev *netdev_, int qid
>> OVS_UNUSED,
>> }
>>
>> DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
>> +size_t size = dp_packet_size(packet);
>> +
>> +/* TSO not supported in BSD netdev */
>> +if (dp_packet_is_tso(packet)) {
>> +VLOG_WARN_RL(, "%s: No TSO enabled on port, TSO packet
>> dropped "
>> + "%" PRIu32 " ", name, size);
>> +
>> +continue;
>> +}
>> +
>> /* We need the whole data to send the packet on the device */
>> if (!dp_packet_is_linear(packet)) {
>> dp_packet_linearize(packet);
>> }
>>
>> const void *data = dp_packet_data(packet);
>> -size_t size = dp_packet_size(packet);
>>
>> while (!error) {
>> ssize_t retval;
>> diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
>> index 77d04fc..ad7223a 100644
>> --- a/lib/netdev-dpdk.c
>> +++ b/lib/netdev-dpdk.c
>> @@ -1375,14 +1375,16 @@ netdev_dpdk_vhost_construct(struct netdev *netdev)
>> goto out;
>> }
>>
>> -err = rte_vhost_driver_disable_features(dev->vhost_id,
>> -1ULL << VIRTIO_NET_F_HOST_TSO4
>> -| 1ULL << VIRTIO_NET_F_HOST_TSO6
>> -| 1ULL << VIRTIO_NET_F_CSUM);
>> -if (err) {
>> -VLOG_ERR("rte_vhost_driver_disable_features failed for vhost user "
>> - "port: %s\n", name);
>> -goto out;
>> +if (!dpdk_multi_segment_mbufs) {
>> +err = rte_vhost_driver_disable_features(dev->vhost_id,
>> +1ULL << VIRTIO_NET_F_HOST_TSO4
>> +| 1ULL <<
Re: [ovs-dev] [PATCH v2 2/3] netdev-dpdk: Consider packets marked for TSO.
On 1/11/2019 12:26 PM, Ian Stokes wrote:
On 1/10/2019 4:58 PM, Tiago Lam wrote:
Previously, TSO was being explicity disabled on vhost interfaces,
meaning the guests wouldn't have TSO support negotiated in. With TSO
negotiated and enabled, packets are now marked for TSO, through the
PKT_TX_TCP_SEG flag.
In order to deal with this type of packets, a new function,
netdev_dpdk_prep_tso_packet(), has been introduced, with the main
purpose of setting correctly the l2, l3 and l4 length members of the
mbuf struct, and the appropriate ol_flags. This function supports TSO
both in IPv4 and IPv6.
netdev_dpdk_prep_tso_packet() is then only called when packets are
marked with the PKT_TX_TCP_SEG flag, meaning they have been marked for
TSO, and when the packet will be traversing the NIC.
Additionally, if a packet is marked for TSO but the egress netdev
doesn't support it, the packet is dropped.
Hi Tiago,
a high level first pass, I still need to test this in more detail so not
a full review yet but some minor issues to be addressed below.
One more addition to below.
[snip]
diff --git a/lib/netdev-linux.c b/lib/netdev-linux.c
index fa79b2a..1476096 100644
--- a/lib/netdev-linux.c
+++ b/lib/netdev-linux.c
@@ -1379,6 +1379,13 @@ netdev_linux_sock_batch_send(int sock, int
ifindex,
struct dp_packet *packet;
DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
+ /* TSO not supported in Linux netdev */
+ if (dp_packet_is_tso(packet)) {
+ VLOG_WARN_RL(, "%d: No TSO enabled on port, TSO packet
dropped "
+ "%ld", sock, size);
Will cause compilation warnings:
format ‘%ld’ expects argument of type ‘long int’, but argument 6 has
type ‘size_t’
Ian
___
dev mailing list
d...@openvswitch.org
https://mail.openvswitch.org/mailman/listinfo/ovs-dev
Re: [ovs-dev] [PATCH v2 2/3] netdev-dpdk: Consider packets marked for TSO.
On 1/10/2019 4:58 PM, Tiago Lam wrote:
Previously, TSO was being explicity disabled on vhost interfaces,
meaning the guests wouldn't have TSO support negotiated in. With TSO
negotiated and enabled, packets are now marked for TSO, through the
PKT_TX_TCP_SEG flag.
In order to deal with this type of packets, a new function,
netdev_dpdk_prep_tso_packet(), has been introduced, with the main
purpose of setting correctly the l2, l3 and l4 length members of the
mbuf struct, and the appropriate ol_flags. This function supports TSO
both in IPv4 and IPv6.
netdev_dpdk_prep_tso_packet() is then only called when packets are
marked with the PKT_TX_TCP_SEG flag, meaning they have been marked for
TSO, and when the packet will be traversing the NIC.
Additionally, if a packet is marked for TSO but the egress netdev
doesn't support it, the packet is dropped.
Hi Tiago,
a high level first pass, I still need to test this in more detail so not
a full review yet but some minor issues to be addressed below.
Co-authored-by: Mark Kavanagh
Signed-off-by: Mark Kavanagh
Signed-off-by: Tiago Lam
---
lib/dp-packet.h| 14 +++
lib/netdev-bsd.c | 11 -
lib/netdev-dpdk.c | 121 ++---
lib/netdev-dummy.c | 11 -
lib/netdev-linux.c | 15 +++
5 files changed, 146 insertions(+), 26 deletions(-)
diff --git a/lib/dp-packet.h b/lib/dp-packet.h
index 970aaf2..c384416 100644
--- a/lib/dp-packet.h
+++ b/lib/dp-packet.h
@@ -104,6 +104,8 @@ static inline void dp_packet_set_size(struct dp_packet *,
uint32_t);
static inline uint16_t dp_packet_get_allocated(const struct dp_packet *);
static inline void dp_packet_set_allocated(struct dp_packet *, uint16_t);
+static inline bool dp_packet_is_tso(struct dp_packet *b);
+
void *dp_packet_resize_l2(struct dp_packet *, int increment);
void *dp_packet_resize_l2_5(struct dp_packet *, int increment);
static inline void *dp_packet_eth(const struct dp_packet *);
@@ -761,6 +763,12 @@ dp_packet_set_allocated(struct dp_packet *b, uint16_t s)
b->mbuf.buf_len = s;
}
+static inline bool
+dp_packet_is_tso(struct dp_packet *b)
+{
+return (b->mbuf.ol_flags & PKT_TX_TCP_SEG) ? true : false;
+}
+
static inline void
dp_packet_copy_mbuf_flags(struct dp_packet *dst, const struct dp_packet *src)
{
@@ -972,6 +980,12 @@ dp_packet_get_allocated(const struct dp_packet *b)
return b->allocated_;
}
+static inline bool
+dp_packet_is_tso(struct dp_packet *b)
+{
This will cause a unused parameter warning. Need to pass OVS_UNUSED
above also.
+return false;
+}
+
static inline void
dp_packet_set_allocated(struct dp_packet *b, uint16_t s)
{
diff --git a/lib/netdev-bsd.c b/lib/netdev-bsd.c
index 278c8a9..5e8c5cc 100644
--- a/lib/netdev-bsd.c
+++ b/lib/netdev-bsd.c
@@ -700,13 +700,22 @@ netdev_bsd_send(struct netdev *netdev_, int qid
OVS_UNUSED,
}
DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
+size_t size = dp_packet_size(packet);
+
+/* TSO not supported in BSD netdev */
+if (dp_packet_is_tso(packet)) {
+VLOG_WARN_RL(, "%s: No TSO enabled on port, TSO packet dropped "
+ "%" PRIu32 " ", name, size);
+
+continue;
+}
+
/* We need the whole data to send the packet on the device */
if (!dp_packet_is_linear(packet)) {
dp_packet_linearize(packet);
}
const void *data = dp_packet_data(packet);
-size_t size = dp_packet_size(packet);
while (!error) {
ssize_t retval;
diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
index 77d04fc..ad7223a 100644
--- a/lib/netdev-dpdk.c
+++ b/lib/netdev-dpdk.c
@@ -1375,14 +1375,16 @@ netdev_dpdk_vhost_construct(struct netdev *netdev)
goto out;
}
-err = rte_vhost_driver_disable_features(dev->vhost_id,
-1ULL << VIRTIO_NET_F_HOST_TSO4
-| 1ULL << VIRTIO_NET_F_HOST_TSO6
-| 1ULL << VIRTIO_NET_F_CSUM);
-if (err) {
-VLOG_ERR("rte_vhost_driver_disable_features failed for vhost user "
- "port: %s\n", name);
-goto out;
+if (!dpdk_multi_segment_mbufs) {
+err = rte_vhost_driver_disable_features(dev->vhost_id,
+1ULL << VIRTIO_NET_F_HOST_TSO4
+| 1ULL << VIRTIO_NET_F_HOST_TSO6
+| 1ULL << VIRTIO_NET_F_CSUM);
+if (err) {
+VLOG_ERR("rte_vhost_driver_disable_features failed for vhost user "
+ "client port: %s\n", dev->up.name);
+goto out;
+}
}
err = rte_vhost_driver_start(dev->vhost_id);
@@ -2027,6 +2029,44 @@ netdev_dpdk_rxq_dealloc(struct netdev_rxq *rxq)
rte_free(rx);
}
+/* Should only be called if PKT_TX_TCP_SEG is set in ol_flags.
+ *
[ovs-dev] [PATCH v2 2/3] netdev-dpdk: Consider packets marked for TSO.
Previously, TSO was being explicity disabled on vhost interfaces,
meaning the guests wouldn't have TSO support negotiated in. With TSO
negotiated and enabled, packets are now marked for TSO, through the
PKT_TX_TCP_SEG flag.
In order to deal with this type of packets, a new function,
netdev_dpdk_prep_tso_packet(), has been introduced, with the main
purpose of setting correctly the l2, l3 and l4 length members of the
mbuf struct, and the appropriate ol_flags. This function supports TSO
both in IPv4 and IPv6.
netdev_dpdk_prep_tso_packet() is then only called when packets are
marked with the PKT_TX_TCP_SEG flag, meaning they have been marked for
TSO, and when the packet will be traversing the NIC.
Additionally, if a packet is marked for TSO but the egress netdev
doesn't support it, the packet is dropped.
Co-authored-by: Mark Kavanagh
Signed-off-by: Mark Kavanagh
Signed-off-by: Tiago Lam
---
lib/dp-packet.h| 14 +++
lib/netdev-bsd.c | 11 -
lib/netdev-dpdk.c | 121 ++---
lib/netdev-dummy.c | 11 -
lib/netdev-linux.c | 15 +++
5 files changed, 146 insertions(+), 26 deletions(-)
diff --git a/lib/dp-packet.h b/lib/dp-packet.h
index 970aaf2..c384416 100644
--- a/lib/dp-packet.h
+++ b/lib/dp-packet.h
@@ -104,6 +104,8 @@ static inline void dp_packet_set_size(struct dp_packet *,
uint32_t);
static inline uint16_t dp_packet_get_allocated(const struct dp_packet *);
static inline void dp_packet_set_allocated(struct dp_packet *, uint16_t);
+static inline bool dp_packet_is_tso(struct dp_packet *b);
+
void *dp_packet_resize_l2(struct dp_packet *, int increment);
void *dp_packet_resize_l2_5(struct dp_packet *, int increment);
static inline void *dp_packet_eth(const struct dp_packet *);
@@ -761,6 +763,12 @@ dp_packet_set_allocated(struct dp_packet *b, uint16_t s)
b->mbuf.buf_len = s;
}
+static inline bool
+dp_packet_is_tso(struct dp_packet *b)
+{
+return (b->mbuf.ol_flags & PKT_TX_TCP_SEG) ? true : false;
+}
+
static inline void
dp_packet_copy_mbuf_flags(struct dp_packet *dst, const struct dp_packet *src)
{
@@ -972,6 +980,12 @@ dp_packet_get_allocated(const struct dp_packet *b)
return b->allocated_;
}
+static inline bool
+dp_packet_is_tso(struct dp_packet *b)
+{
+return false;
+}
+
static inline void
dp_packet_set_allocated(struct dp_packet *b, uint16_t s)
{
diff --git a/lib/netdev-bsd.c b/lib/netdev-bsd.c
index 278c8a9..5e8c5cc 100644
--- a/lib/netdev-bsd.c
+++ b/lib/netdev-bsd.c
@@ -700,13 +700,22 @@ netdev_bsd_send(struct netdev *netdev_, int qid
OVS_UNUSED,
}
DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
+size_t size = dp_packet_size(packet);
+
+/* TSO not supported in BSD netdev */
+if (dp_packet_is_tso(packet)) {
+VLOG_WARN_RL(, "%s: No TSO enabled on port, TSO packet dropped "
+ "%" PRIu32 " ", name, size);
+
+continue;
+}
+
/* We need the whole data to send the packet on the device */
if (!dp_packet_is_linear(packet)) {
dp_packet_linearize(packet);
}
const void *data = dp_packet_data(packet);
-size_t size = dp_packet_size(packet);
while (!error) {
ssize_t retval;
diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
index 77d04fc..ad7223a 100644
--- a/lib/netdev-dpdk.c
+++ b/lib/netdev-dpdk.c
@@ -1375,14 +1375,16 @@ netdev_dpdk_vhost_construct(struct netdev *netdev)
goto out;
}
-err = rte_vhost_driver_disable_features(dev->vhost_id,
-1ULL << VIRTIO_NET_F_HOST_TSO4
-| 1ULL << VIRTIO_NET_F_HOST_TSO6
-| 1ULL << VIRTIO_NET_F_CSUM);
-if (err) {
-VLOG_ERR("rte_vhost_driver_disable_features failed for vhost user "
- "port: %s\n", name);
-goto out;
+if (!dpdk_multi_segment_mbufs) {
+err = rte_vhost_driver_disable_features(dev->vhost_id,
+1ULL << VIRTIO_NET_F_HOST_TSO4
+| 1ULL << VIRTIO_NET_F_HOST_TSO6
+| 1ULL << VIRTIO_NET_F_CSUM);
+if (err) {
+VLOG_ERR("rte_vhost_driver_disable_features failed for vhost user "
+ "client port: %s\n", dev->up.name);
+goto out;
+}
}
err = rte_vhost_driver_start(dev->vhost_id);
@@ -2027,6 +2029,44 @@ netdev_dpdk_rxq_dealloc(struct netdev_rxq *rxq)
rte_free(rx);
}
+/* Should only be called if PKT_TX_TCP_SEG is set in ol_flags.
+ * Furthermore, it also sets the PKT_TX_TCP_CKSUM and PKT_TX_IP_CKSUM flags,
+ * and PKT_TX_IPV4 and PKT_TX_IPV6 in case the packet is IPv4 or IPv6,
+ * respectiveoly. */
+static void
+netdev_dpdk_prep_tso_packet(struct rte_mbuf *mbuf, int mtu)
+{
+struct dp_packet *pkt;
+struct tcp_header *th;
+
+pkt = CONTAINER_OF(mbuf,
