The DPDK mbuf API specifies 4 status when it comes to L4 checksums:
- RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN: no information about the RX L4 checksum
- RTE_MBUF_F_RX_L4_CKSUM_BAD: the L4 checksum in the packet is wrong
- RTE_MBUF_F_RX_L4_CKSUM_GOOD: the L4 checksum in the packet is valid
- RTE_MBUF_F_RX_L4_CKSUM_NONE: the L4 checksum is not correct in the packet
data, but the integrity of the L4 data is verified.
Similarly to the IP checksum offloads API, revise OVS L4 offloads API.
No information about the L4 protocol is provided by any netdev-*
implementation, so OVS needs to mark this L4 protocol during flow
extraction.
Rename current API for consistency with dp_packet_(inner_)?l4_checksum_.
Signed-off-by: David Marchand <david.march...@redhat.com>
---
Changes since v3:
- L4 partial testing was moved earlier in the series,
Changes since v1:
- fixed tunnel decapsulation checksum validation,
- added partial handling in netdev-dummy and added unit tests,
---
lib/conntrack.c | 11 +-
lib/dp-packet-gso.c | 3 +
lib/dp-packet.c | 85 ++++-----
lib/dp-packet.h | 291 +++++++++++++++----------------
lib/dpif-netdev-extract-avx512.c | 36 +---
lib/flow.c | 42 +----
lib/netdev-dpdk.c | 52 ++++--
lib/netdev-dummy.c | 37 ++--
lib/netdev-linux.c | 104 +++++------
lib/netdev-native-tnl.c | 18 +-
lib/netdev.c | 2 +-
lib/odp-execute-avx512.c | 16 +-
lib/packets.c | 49 ++++--
13 files changed, 353 insertions(+), 393 deletions(-)
diff --git a/lib/conntrack.c b/lib/conntrack.c
index dc7a7535e3..a17be27be2 100644
--- a/lib/conntrack.c
+++ b/lib/conntrack.c
@@ -914,7 +914,6 @@ nat_inner_packet(struct dp_packet *pkt, struct conn_key
*key,
const char *inner_l4 = NULL;
uint16_t orig_l3_ofs = pkt->l3_ofs;
uint16_t orig_l4_ofs = pkt->l4_ofs;
- uint64_t orig_ol_flags = *dp_packet_ol_flags_ptr(pkt);
uint32_t orig_offloads = pkt->offloads;
void *l3 = dp_packet_l3(pkt);
@@ -936,8 +935,8 @@ nat_inner_packet(struct dp_packet *pkt, struct conn_key
*key,
pkt->l4_ofs += inner_l4 - (char *) l4;
/* Drop any offloads to force below helpers to calculate checksums
* if needed. */
- *dp_packet_ol_flags_ptr(pkt) &= ~DP_PACKET_OL_TX_ANY_CKSUM;
dp_packet_ip_checksum_set_unknown(pkt);
+ dp_packet_l4_checksum_set_unknown(pkt);
/* Reverse the key for inner packet. */
struct conn_key rev_key = *key;
@@ -963,7 +962,6 @@ nat_inner_packet(struct dp_packet *pkt, struct conn_key
*key,
pkt->l3_ofs = orig_l3_ofs;
pkt->l4_ofs = orig_l4_ofs;
- *dp_packet_ol_flags_ptr(pkt) = orig_ol_flags;
pkt->offloads = orig_offloads;
}
@@ -2262,8 +2260,7 @@ conn_key_extract(struct conntrack *ct, struct dp_packet
*pkt, ovs_be16 dl_type,
/* Validate the checksum only when hwol is not supported. */
if (extract_l4(&ctx->key, l4, dp_packet_l4_size(pkt),
&ctx->icmp_related, l3,
- !dp_packet_l4_checksum_good(pkt) &&
- !dp_packet_hwol_tx_l4_checksum(pkt),
+ dp_packet_l4_checksum_unknown(pkt),
NULL)) {
ctx->hash = conn_key_hash(&ctx->key, ct->hash_basis);
return true;
@@ -3702,8 +3699,8 @@ handle_ftp_ctl(struct conntrack *ct, const struct
conn_lookup_ctx *ctx,
adj_seqnum(&th->tcp_seq, ec->seq_skew);
}
- if (dp_packet_hwol_tx_l4_checksum(pkt)) {
- dp_packet_ol_reset_l4_csum_good(pkt);
+ if (dp_packet_l4_checksum_valid(pkt)) {
+ dp_packet_l4_checksum_set_partial(pkt);
} else {
th->tcp_csum = 0;
if (ctx->key.dl_type == htons(ETH_TYPE_IPV6)) {
diff --git a/lib/dp-packet-gso.c b/lib/dp-packet-gso.c
index f49a296cd4..5698c3f65a 100644
--- a/lib/dp-packet-gso.c
+++ b/lib/dp-packet-gso.c
@@ -160,6 +160,7 @@ dp_packet_gso(struct dp_packet *p, struct dp_packet_batch
**batches)
tnl_hdr = dp_packet_l4(seg);
tnl_hdr->udp_len = htons(dp_packet_l4_size(seg));
+ dp_packet_l4_checksum_set_partial(seg);
}
if (udp_tnl || gre_tnl) {
@@ -198,8 +199,10 @@ dp_packet_gso(struct dp_packet *p, struct dp_packet_batch
**batches)
/* Update L4 header. */
if (udp_tnl || gre_tnl) {
tcp_hdr = dp_packet_inner_l4(seg);
+ dp_packet_inner_l4_checksum_set_partial(seg);
} else {
tcp_hdr = dp_packet_l4(seg);
+ dp_packet_l4_checksum_set_partial(seg);
}
put_16aligned_be32(&tcp_hdr->tcp_seq, htonl(tcp_seq));
tcp_seq += seg_len;
diff --git a/lib/dp-packet.c b/lib/dp-packet.c
index adcd5e52f1..40246ca420 100644
--- a/lib/dp-packet.c
+++ b/lib/dp-packet.c
@@ -557,8 +557,9 @@ void
dp_packet_ol_send_prepare(struct dp_packet *p, uint64_t flags)
{
if (!dp_packet_ip_checksum_partial(p)
+ && !dp_packet_l4_checksum_partial(p)
&& !dp_packet_inner_ip_checksum_partial(p)
- && !dp_packet_hwol_tx_is_any_csum(p)) {
+ && !dp_packet_inner_l4_checksum_partial(p)) {
/* Only checksumming needs actions. */
return;
}
@@ -569,27 +570,20 @@ dp_packet_ol_send_prepare(struct dp_packet *p, uint64_t
flags)
dp_packet_ip_set_header_csum(p, false);
}
- if (dp_packet_hwol_tx_l4_checksum(p)) {
- if (dp_packet_l4_checksum_good(p)) {
- dp_packet_hwol_reset_tx_l4_csum(p);
- return;
- }
-
- if (dp_packet_hwol_l4_is_tcp(p)
- && !(flags & NETDEV_TX_OFFLOAD_TCP_CKSUM)) {
- packet_tcp_complete_csum(p, false);
- dp_packet_ol_set_l4_csum_good(p);
- dp_packet_hwol_reset_tx_l4_csum(p);
- } else if (dp_packet_hwol_l4_is_udp(p)
- && !(flags & NETDEV_TX_OFFLOAD_UDP_CKSUM)) {
- packet_udp_complete_csum(p, false);
- dp_packet_ol_set_l4_csum_good(p);
- dp_packet_hwol_reset_tx_l4_csum(p);
- } else if (!(flags & NETDEV_TX_OFFLOAD_SCTP_CKSUM)
- && dp_packet_hwol_l4_is_sctp(p)) {
- packet_sctp_complete_csum(p, false);
- dp_packet_ol_set_l4_csum_good(p);
- dp_packet_hwol_reset_tx_l4_csum(p);
+ if (dp_packet_l4_checksum_partial(p)) {
+ if (dp_packet_l4_proto_tcp(p)) {
+ if (!(flags & NETDEV_TX_OFFLOAD_TCP_CKSUM)) {
+ packet_tcp_complete_csum(p, false);
+ }
+ } else if (dp_packet_l4_proto_udp(p)) {
+ if (!(flags & NETDEV_TX_OFFLOAD_UDP_CKSUM)) {
+ packet_udp_complete_csum(p, false);
+ }
+ } else {
+ ovs_assert(dp_packet_l4_proto_sctp(p));
+ if (!(flags & NETDEV_TX_OFFLOAD_SCTP_CKSUM)) {
+ packet_sctp_complete_csum(p, false);
+ }
}
}
@@ -603,8 +597,8 @@ dp_packet_ol_send_prepare(struct dp_packet *p, uint64_t
flags)
* support inner checksum offload and an outer UDP checksum is
* required, then we can't offload inner checksum either. As that would
* invalidate the outer checksum. */
- if (!(flags & NETDEV_TX_OFFLOAD_OUTER_UDP_CKSUM) &&
- dp_packet_hwol_is_outer_udp_cksum(p)) {
+ if (!(flags & NETDEV_TX_OFFLOAD_OUTER_UDP_CKSUM)
+ && dp_packet_l4_checksum_partial(p)) {
flags &= ~(NETDEV_TX_OFFLOAD_TCP_CKSUM |
NETDEV_TX_OFFLOAD_UDP_CKSUM |
NETDEV_TX_OFFLOAD_SCTP_CKSUM |
@@ -617,22 +611,20 @@ dp_packet_ol_send_prepare(struct dp_packet *p, uint64_t
flags)
dp_packet_ip_set_header_csum(p, true);
}
- if (dp_packet_hwol_tx_l4_checksum(p)) {
- if (dp_packet_hwol_l4_is_tcp(p)
- && !(flags & NETDEV_TX_OFFLOAD_TCP_CKSUM)) {
- packet_tcp_complete_csum(p, true);
- dp_packet_ol_set_l4_csum_good(p);
- dp_packet_hwol_reset_tx_l4_csum(p);
- } else if (dp_packet_hwol_l4_is_udp(p)
- && !(flags & NETDEV_TX_OFFLOAD_UDP_CKSUM)) {
- packet_udp_complete_csum(p, true);
- dp_packet_ol_set_l4_csum_good(p);
- dp_packet_hwol_reset_tx_l4_csum(p);
- } else if (!(flags & NETDEV_TX_OFFLOAD_SCTP_CKSUM)
- && dp_packet_hwol_l4_is_sctp(p)) {
- packet_sctp_complete_csum(p, true);
- dp_packet_ol_set_l4_csum_good(p);
- dp_packet_hwol_reset_tx_l4_csum(p);
+ if (dp_packet_inner_l4_checksum_partial(p)) {
+ if (dp_packet_inner_l4_proto_tcp(p)) {
+ if (!(flags & NETDEV_TX_OFFLOAD_TCP_CKSUM)) {
+ packet_tcp_complete_csum(p, true);
+ }
+ } else if (dp_packet_inner_l4_proto_udp(p)) {
+ if (!(flags & NETDEV_TX_OFFLOAD_UDP_CKSUM)) {
+ packet_udp_complete_csum(p, true);
+ }
+ } else {
+ ovs_assert(dp_packet_inner_l4_proto_sctp(p));
+ if (!(flags & NETDEV_TX_OFFLOAD_SCTP_CKSUM)) {
+ packet_sctp_complete_csum(p, true);
+ }
}
}
@@ -641,13 +633,10 @@ dp_packet_ol_send_prepare(struct dp_packet *p, uint64_t
flags)
dp_packet_ip_set_header_csum(p, false);
}
- if (!dp_packet_hwol_is_outer_udp_cksum(p)) {
- return;
- }
-
- if (!(flags & NETDEV_TX_OFFLOAD_OUTER_UDP_CKSUM)) {
- packet_udp_complete_csum(p, false);
- dp_packet_ol_set_l4_csum_good(p);
- dp_packet_hwol_reset_outer_udp_csum(p);
+ if (dp_packet_l4_checksum_partial(p)) {
+ ovs_assert(dp_packet_l4_proto_udp(p));
+ if (!(flags & NETDEV_TX_OFFLOAD_OUTER_UDP_CKSUM)) {
+ packet_udp_complete_csum(p, false);
+ }
}
}
diff --git a/lib/dp-packet.h b/lib/dp-packet.h
index 1ea3cf7761..12aa2d0bfa 100644
--- a/lib/dp-packet.h
+++ b/lib/dp-packet.h
@@ -60,30 +60,11 @@ enum {
/* Value 0 is not used. */
/* TCP Segmentation Offload. */
DEF_OL_FLAG(DP_PACKET_OL_TX_TCP_SEG, RTE_MBUF_F_TX_TCP_SEG, 0x40),
- /* Offload TCP checksum. */
- DEF_OL_FLAG(DP_PACKET_OL_TX_TCP_CKSUM, RTE_MBUF_F_TX_TCP_CKSUM, 0x200),
- /* Offload UDP checksum. */
- DEF_OL_FLAG(DP_PACKET_OL_TX_UDP_CKSUM, RTE_MBUF_F_TX_UDP_CKSUM, 0x400),
- /* Offload SCTP checksum. */
- DEF_OL_FLAG(DP_PACKET_OL_TX_SCTP_CKSUM, RTE_MBUF_F_TX_SCTP_CKSUM, 0x800),
- /* Offload tunnel outer UDP checksum. */
- DEF_OL_FLAG(DP_PACKET_OL_TX_OUTER_UDP_CKSUM,
- RTE_MBUF_F_TX_OUTER_UDP_CKSUM, 0x20000),
/* Adding new field requires adding to DP_PACKET_OL_SUPPORTED_MASK. */
};
-#define DP_PACKET_OL_SUPPORTED_MASK (DP_PACKET_OL_TX_TCP_SEG | \
- DP_PACKET_OL_TX_TCP_CKSUM | \
- DP_PACKET_OL_TX_UDP_CKSUM | \
- DP_PACKET_OL_TX_SCTP_CKSUM | \
- DP_PACKET_OL_TX_OUTER_UDP_CKSUM)
-
-#define DP_PACKET_OL_TX_L4_MASK (DP_PACKET_OL_TX_TCP_CKSUM | \
- DP_PACKET_OL_TX_UDP_CKSUM | \
- DP_PACKET_OL_TX_SCTP_CKSUM)
-#define DP_PACKET_OL_TX_ANY_CKSUM (DP_PACKET_OL_TX_L4_MASK | \
- DP_PACKET_OL_TX_OUTER_UDP_CKSUM)
+#define DP_PACKET_OL_SUPPORTED_MASK DP_PACKET_OL_TX_TCP_SEG
/* Bit masks for the 'offloads' member of the 'dp_packet' structure. */
enum OVS_PACKED_ENUM dp_packet_offload_mask {
@@ -97,6 +78,10 @@ enum OVS_PACKED_ENUM dp_packet_offload_mask {
/* Valid L4 checksum in the packet. */
DP_PACKET_OL_L4_CKSUM_GOOD = UINT16_C(1) << 8,
+ /* Protocol corresponding to above L4 checksums. */
+ DP_PACKET_OL_L4_PROTO_TCP = UINT16_C(1) << 9,
+ DP_PACKET_OL_L4_PROTO_UDP = UINT16_C(1) << 10,
+
/* Bits for marking a packet as tunneled. */
DP_PACKET_OL_TUNNEL_GENEVE = UINT16_C(1) << 11,
DP_PACKET_OL_TUNNEL_VXLAN = UINT16_C(1) << 12,
@@ -108,6 +93,14 @@ enum OVS_PACKED_ENUM dp_packet_offload_mask {
DP_PACKET_OL_IP_CKSUM_BAD << DP_PACKET_OL_SHIFT_COUNT,
DP_PACKET_OL_INNER_IP_CKSUM_GOOD =
DP_PACKET_OL_IP_CKSUM_GOOD << DP_PACKET_OL_SHIFT_COUNT,
+ DP_PACKET_OL_INNER_L4_CKSUM_BAD =
+ DP_PACKET_OL_L4_CKSUM_BAD << DP_PACKET_OL_SHIFT_COUNT,
+ DP_PACKET_OL_INNER_L4_CKSUM_GOOD =
+ DP_PACKET_OL_L4_CKSUM_GOOD << DP_PACKET_OL_SHIFT_COUNT,
+ DP_PACKET_OL_INNER_L4_PROTO_TCP =
+ DP_PACKET_OL_L4_PROTO_TCP << DP_PACKET_OL_SHIFT_COUNT,
+ DP_PACKET_OL_INNER_L4_PROTO_UDP =
+ DP_PACKET_OL_L4_PROTO_UDP << DP_PACKET_OL_SHIFT_COUNT,
};
#ifdef DPDK_NETDEV
@@ -125,9 +118,18 @@ BUILD_ASSERT_DECL(DP_PACKET_OL_L4_CKSUM_GOOD ==
RTE_MBUF_F_RX_L4_CKSUM_GOOD);
#define DP_PACKET_OL_TUNNEL_MASK (DP_PACKET_OL_TUNNEL_GENEVE \
| DP_PACKET_OL_TUNNEL_VXLAN)
+#define DP_PACKET_OL_L4_PROTO_MASK (DP_PACKET_OL_L4_PROTO_TCP \
+ | DP_PACKET_OL_L4_PROTO_UDP)
+
#define DP_PACKET_OL_INNER_IP_CKSUM_MASK (DP_PACKET_OL_INNER_IP_CKSUM_GOOD \
| DP_PACKET_OL_INNER_IP_CKSUM_BAD)
+#define DP_PACKET_OL_INNER_L4_CKSUM_MASK (DP_PACKET_OL_INNER_L4_CKSUM_GOOD \
+ | DP_PACKET_OL_INNER_L4_CKSUM_BAD)
+
+#define DP_PACKET_OL_INNER_L4_PROTO_MASK (DP_PACKET_OL_INNER_L4_PROTO_TCP \
+ | DP_PACKET_OL_INNER_L4_PROTO_UDP)
+
/* Buffer for holding packet data. A dp_packet is automatically reallocated
* as necessary if it grows too large for the available memory.
* By default the packet type is set to Ethernet (PT_ETH).
@@ -1120,20 +1122,6 @@ dp_packet_tunnel(const struct dp_packet *b)
return !!(b->offloads & DP_PACKET_OL_TUNNEL_MASK);
}
-/* Returns the L4 cksum offload bitmask. */
-static inline uint64_t
-dp_packet_hwol_l4_mask(const struct dp_packet *b)
-{
- return *dp_packet_ol_flags_ptr(b) & DP_PACKET_OL_TX_L4_MASK;
-}
-
-/* Return true if the packet 'b' requested L4 checksum offload. */
-static inline bool
-dp_packet_hwol_tx_l4_checksum(const struct dp_packet *b)
-{
- return !!dp_packet_hwol_l4_mask(b);
-}
-
/* Returns 'true' if packet 'b' is marked for TCP segmentation offloading. */
static inline bool
dp_packet_hwol_is_tso(const struct dp_packet *b)
@@ -1141,74 +1129,6 @@ dp_packet_hwol_is_tso(const struct dp_packet *b)
return !!(*dp_packet_ol_flags_ptr(b) & DP_PACKET_OL_TX_TCP_SEG);
}
-/* Returns 'true' if packet 'b' is marked for TCP checksum offloading. */
-static inline bool
-dp_packet_hwol_l4_is_tcp(const struct dp_packet *b)
-{
- return (*dp_packet_ol_flags_ptr(b) & DP_PACKET_OL_TX_L4_MASK) ==
- DP_PACKET_OL_TX_TCP_CKSUM;
-}
-
-/* Returns 'true' if packet 'b' is marked for UDP checksum offloading. */
-static inline bool
-dp_packet_hwol_l4_is_udp(struct dp_packet *b)
-{
- return (*dp_packet_ol_flags_ptr(b) & DP_PACKET_OL_TX_L4_MASK) ==
- DP_PACKET_OL_TX_UDP_CKSUM;
-}
-
-/* Returns 'true' if packet 'b' is marked for SCTP checksum offloading. */
-static inline bool
-dp_packet_hwol_l4_is_sctp(struct dp_packet *b)
-{
- return (*dp_packet_ol_flags_ptr(b) & DP_PACKET_OL_TX_L4_MASK) ==
- DP_PACKET_OL_TX_SCTP_CKSUM;
-}
-
-/* Returns 'true' if packet 'b' is marked for outer UDP checksum offload. */
-static inline bool
-dp_packet_hwol_is_outer_udp_cksum(struct dp_packet *b)
-{
- return !!(*dp_packet_ol_flags_ptr(b) & DP_PACKET_OL_TX_OUTER_UDP_CKSUM);
-}
-
-/* Returns 'true' if packet 'b' is marked for any checksum offload. */
-static inline bool
-dp_packet_hwol_tx_is_any_csum(struct dp_packet *b)
-{
- return !!(*dp_packet_ol_flags_ptr(b) & DP_PACKET_OL_TX_ANY_CKSUM);
-}
-
-static inline void
-dp_packet_hwol_reset_tx_l4_csum(struct dp_packet *p)
-{
- *dp_packet_ol_flags_ptr(p) &= ~DP_PACKET_OL_TX_L4_MASK;
-}
-
-/* Mark packet 'b' for TCP checksum offloading. It implies that either
- * the packet 'b' is marked for IPv4 or IPv6 checksum offloading. */
-static inline void
-dp_packet_hwol_set_csum_tcp(struct dp_packet *b)
-{
- *dp_packet_ol_flags_ptr(b) |= DP_PACKET_OL_TX_TCP_CKSUM;
-}
-
-/* Mark packet 'b' for UDP checksum offloading. It implies that either
- * the packet 'b' is marked for IPv4 or IPv6 checksum offloading. */
-static inline void
-dp_packet_hwol_set_csum_udp(struct dp_packet *b)
-{
- *dp_packet_ol_flags_ptr(b) |= DP_PACKET_OL_TX_UDP_CKSUM;
-}
-
-/* Mark packet 'b' for SCTP checksum offloading. It implies that either
- * the packet 'b' is marked for IPv4 or IPv6 checksum offloading. */
-static inline void
-dp_packet_hwol_set_csum_sctp(struct dp_packet *b)
-{
- *dp_packet_ol_flags_ptr(b) |= DP_PACKET_OL_TX_SCTP_CKSUM;
-}
-
/* Mark packet 'b' for TCP segmentation offloading. It implies that
* either the packet 'b' is marked for IPv4 or IPv6 checksum offloading
* and also for TCP checksum offloading. */
@@ -1218,36 +1138,11 @@ dp_packet_hwol_set_tcp_seg(struct dp_packet *b)
*dp_packet_ol_flags_ptr(b) |= DP_PACKET_OL_TX_TCP_SEG;
}
-static inline void
-dp_packet_hwol_reset_outer_udp_csum(struct dp_packet *p)
-{
- *dp_packet_ol_flags_ptr(p) &= ~DP_PACKET_OL_TX_OUTER_UDP_CKSUM;
-}
-
-/* Mark packet 'b' for csum offloading in outer UDP header. */
-static inline void
-dp_packet_hwol_set_outer_udp_csum(struct dp_packet *b)
-{
- *dp_packet_ol_flags_ptr(b) |= DP_PACKET_OL_TX_OUTER_UDP_CKSUM;
-}
-
-/* Resets TCP Segmentation in packet 'p' and adjust flags to indicate
- * L3 and L4 checksumming is now required. */
+/* Resets TCP Segmentation in packet 'p'. */
static inline void
dp_packet_hwol_reset_tcp_seg(struct dp_packet *p)
{
- uint64_t ol_flags = *dp_packet_ol_flags_ptr(p)
- | DP_PACKET_OL_TX_TCP_CKSUM;
-
- ol_flags &= ~DP_PACKET_OL_TX_TCP_SEG;
- p->offloads &= ~DP_PACKET_OL_L4_CKSUM_GOOD;
-
- if (dp_packet_tunnel_geneve(p)
- || dp_packet_tunnel_vxlan(p)) {
- ol_flags |= DP_PACKET_OL_TX_OUTER_UDP_CKSUM;
- }
-
- *dp_packet_ol_flags_ptr(p) = ol_flags;
+ *dp_packet_ol_flags_ptr(p) &= ~DP_PACKET_OL_TX_TCP_SEG;
}
/* Marks packet 'p' with good IPv4 checksum. */
@@ -1362,26 +1257,82 @@ dp_packet_ip_set_header_csum(struct dp_packet *p, bool
inner)
}
}
+static inline bool OVS_WARN_UNUSED_RESULT
+dp_packet_l4_proto_tcp(const struct dp_packet *b)
+{
+ return (b->offloads & DP_PACKET_OL_L4_PROTO_MASK)
+ == DP_PACKET_OL_L4_PROTO_TCP;
+}
+
+static inline void
+dp_packet_l4_proto_set_tcp(struct dp_packet *b)
+{
+ b->offloads &= ~DP_PACKET_OL_L4_PROTO_UDP;
+ b->offloads |= DP_PACKET_OL_L4_PROTO_TCP;
+}
+
+static inline bool OVS_WARN_UNUSED_RESULT
+dp_packet_l4_proto_udp(const struct dp_packet *b)
+{
+ return (b->offloads & DP_PACKET_OL_L4_PROTO_MASK)
+ == DP_PACKET_OL_L4_PROTO_UDP;
+}
+
+static inline void
+dp_packet_l4_proto_set_udp(struct dp_packet *b)
+{
+ b->offloads &= ~DP_PACKET_OL_L4_PROTO_TCP;
+ b->offloads |= DP_PACKET_OL_L4_PROTO_UDP;
+}
+
+static inline bool OVS_WARN_UNUSED_RESULT
+dp_packet_l4_proto_sctp(const struct dp_packet *b)
+{
+ return (b->offloads & DP_PACKET_OL_L4_PROTO_MASK)
+ == DP_PACKET_OL_L4_PROTO_MASK;
+}
+
+static inline void
+dp_packet_l4_proto_set_sctp(struct dp_packet *b)
+{
+ b->offloads |= DP_PACKET_OL_L4_PROTO_MASK;
+}
+
/* Returns 'true' if the packet 'p' has good integrity and the
* checksum in it is correct. */
-static inline bool
+static inline bool OVS_WARN_UNUSED_RESULT
dp_packet_l4_checksum_good(const struct dp_packet *p)
{
return (p->offloads & DP_PACKET_OL_L4_CKSUM_MASK)
== DP_PACKET_OL_L4_CKSUM_GOOD;
}
-static inline bool
+/* Marks packet 'p' with good L4 checksum. */
+static inline void
+dp_packet_l4_checksum_set_good(struct dp_packet *p)
+{
+ p->offloads &= ~DP_PACKET_OL_L4_CKSUM_BAD;
+ p->offloads |= DP_PACKET_OL_L4_CKSUM_GOOD;
+}
+
+static inline bool OVS_WARN_UNUSED_RESULT
dp_packet_l4_checksum_bad(const struct dp_packet *p)
{
return (p->offloads & DP_PACKET_OL_L4_CKSUM_MASK)
== DP_PACKET_OL_L4_CKSUM_BAD;
}
+static inline void
+dp_packet_l4_checksum_set_bad(struct dp_packet *p)
+{
+ p->offloads &= ~DP_PACKET_OL_L4_CKSUM_GOOD;
+ p->offloads |= DP_PACKET_OL_L4_CKSUM_BAD;
+}
+
/* Returns 'true' if the packet has good integrity though the
* checksum in the packet 'p' is not complete. */
-static inline bool
-dp_packet_ol_l4_csum_partial(const struct dp_packet *p)
+static inline bool OVS_WARN_UNUSED_RESULT
+dp_packet_l4_checksum_partial(const struct dp_packet *p)
{
return (p->offloads & DP_PACKET_OL_L4_CKSUM_MASK)
== DP_PACKET_OL_L4_CKSUM_MASK;
@@ -1390,33 +1341,81 @@ dp_packet_ol_l4_csum_partial(const struct dp_packet *p)
/* Marks packet 'p' with good integrity though the checksum in the
* packet is not complete. */
static inline void
-dp_packet_ol_set_l4_csum_partial(struct dp_packet *p)
+dp_packet_l4_checksum_set_partial(struct dp_packet *p)
{
p->offloads |= DP_PACKET_OL_L4_CKSUM_MASK;
}
-/* Marks packet 'p' with good L4 checksum. */
+static inline bool OVS_WARN_UNUSED_RESULT
+dp_packet_l4_checksum_unknown(const struct dp_packet *p)
+{
+ return !(p->offloads & DP_PACKET_OL_L4_CKSUM_MASK);
+}
+
static inline void
-dp_packet_ol_set_l4_csum_good(struct dp_packet *p)
+dp_packet_l4_checksum_set_unknown(struct dp_packet *p)
{
- p->offloads &= ~DP_PACKET_OL_L4_CKSUM_BAD;
- p->offloads |= DP_PACKET_OL_L4_CKSUM_GOOD;
+ p->offloads &= ~DP_PACKET_OL_L4_CKSUM_MASK;
+}
+
+static inline bool OVS_WARN_UNUSED_RESULT
+dp_packet_l4_checksum_valid(const struct dp_packet *p)
+{
+ return !!(p->offloads & DP_PACKET_OL_L4_CKSUM_GOOD);
+}
+
+static inline bool OVS_WARN_UNUSED_RESULT
+dp_packet_inner_l4_proto_tcp(const struct dp_packet *p)
+{
+ return (p->offloads & DP_PACKET_OL_INNER_L4_PROTO_MASK)
+ == DP_PACKET_OL_INNER_L4_PROTO_TCP;
+}
+
+static inline bool OVS_WARN_UNUSED_RESULT
+dp_packet_inner_l4_proto_udp(const struct dp_packet *p)
+{
+ return (p->offloads & DP_PACKET_OL_INNER_L4_PROTO_MASK)
+ == DP_PACKET_OL_INNER_L4_PROTO_UDP;
+}
+
+static inline bool OVS_WARN_UNUSED_RESULT
+dp_packet_inner_l4_proto_sctp(const struct dp_packet *p)
+{
+ return (p->offloads & DP_PACKET_OL_INNER_L4_PROTO_MASK)
+ == DP_PACKET_OL_INNER_L4_PROTO_MASK;
+}
+
+/* Returns 'true' if the inner L4 header has good integrity and the
+ * checksum in it is complete. */
+static inline bool OVS_WARN_UNUSED_RESULT
+dp_packet_inner_l4_checksum_good(const struct dp_packet *p)
+{
+ return (p->offloads & DP_PACKET_OL_INNER_L4_CKSUM_MASK)
+ == DP_PACKET_OL_INNER_L4_CKSUM_GOOD;
}
-/* Marks packet 'p' with good L4 checksum as modified. */
+/* Marks packet 'p' as having a valid inner l4 header, but no checksum. */
static inline void
-dp_packet_ol_reset_l4_csum_good(struct dp_packet *p)
+dp_packet_inner_l4_checksum_set_good(struct dp_packet *p)
{
- if (!dp_packet_ol_l4_csum_partial(p)) {
- p->offloads &= ~DP_PACKET_OL_L4_CKSUM_GOOD;
- }
+ p->offloads &= ~DP_PACKET_OL_INNER_L4_CKSUM_BAD;
+ p->offloads |= DP_PACKET_OL_INNER_L4_CKSUM_GOOD;
+}
+
+/* Returns 'true' if the inner L4 header has good integrity but the
+ * checksum in it is incomplete. */
+static inline bool OVS_WARN_UNUSED_RESULT
+dp_packet_inner_l4_checksum_partial(const struct dp_packet *p)
+{
+ return (p->offloads & DP_PACKET_OL_INNER_L4_CKSUM_MASK)
+ == DP_PACKET_OL_INNER_L4_CKSUM_MASK;
}
+/* Marks packet 'p' as having a valid inner l4 header, but no checksum. */
static inline void
-dp_packet_ol_set_l4_csum_bad(struct dp_packet *p)
+dp_packet_inner_l4_checksum_set_partial(struct dp_packet *p)
{
- p->offloads &= ~DP_PACKET_OL_L4_CKSUM_GOOD;
- p->offloads |= DP_PACKET_OL_L4_CKSUM_BAD;
+ p->offloads |= DP_PACKET_OL_INNER_L4_CKSUM_MASK;
}
static inline void
diff --git a/lib/dpif-netdev-extract-avx512.c b/lib/dpif-netdev-extract-avx512.c
index 3ad892d375..90bf53a7af 100644
--- a/lib/dpif-netdev-extract-avx512.c
+++ b/lib/dpif-netdev-extract-avx512.c
@@ -758,24 +758,6 @@ mfex_check_tcp_data_offset(const struct tcp_header *tcp)
return ret;
}
-static void
-mfex_tcp_set_hwol(struct dp_packet *pkt)
-{
- if (dp_packet_l4_checksum_good(pkt)
- || dp_packet_ol_l4_csum_partial(pkt)) {
- dp_packet_hwol_set_csum_tcp(pkt);
- }
-}
-
-static void
-mfex_udp_set_hwol(struct dp_packet *pkt)
-{
- if (dp_packet_l4_checksum_good(pkt)
- || dp_packet_ol_l4_csum_partial(pkt)) {
- dp_packet_hwol_set_csum_udp(pkt);
- }
-}
-
/* Generic loop to process any mfex profile. This code is specialized into
* multiple actual MFEX implementation functions. Its marked ALWAYS_INLINE
* to ensure the compiler specializes each instance. The code is marked "hot"
@@ -877,7 +859,7 @@ mfex_avx512_process(struct dp_packet_batch *packets,
const struct tcp_header *tcp = (void *)&pkt[38];
mfex_handle_tcp_flags(tcp, &blocks[7]);
dp_packet_update_rss_hash_ipv4_tcp_udp(packet);
- mfex_tcp_set_hwol(packet);
+ dp_packet_l4_proto_set_tcp(packet);
} break;
case PROFILE_ETH_VLAN_IPV4_UDP: {
@@ -890,7 +872,7 @@ mfex_avx512_process(struct dp_packet_batch *packets,
continue;
}
dp_packet_update_rss_hash_ipv4_tcp_udp(packet);
- mfex_udp_set_hwol(packet);
+ dp_packet_l4_proto_set_udp(packet);
} break;
case PROFILE_ETH_IPV4_TCP: {
@@ -906,7 +888,7 @@ mfex_avx512_process(struct dp_packet_batch *packets,
continue;
}
dp_packet_update_rss_hash_ipv4_tcp_udp(packet);
- mfex_tcp_set_hwol(packet);
+ dp_packet_l4_proto_set_tcp(packet);
} break;
case PROFILE_ETH_IPV4_UDP: {
@@ -918,7 +900,7 @@ mfex_avx512_process(struct dp_packet_batch *packets,
continue;
}
dp_packet_update_rss_hash_ipv4_tcp_udp(packet);
- mfex_udp_set_hwol(packet);
+ dp_packet_l4_proto_set_udp(packet);
} break;
case PROFILE_ETH_IPV6_UDP: {
@@ -937,7 +919,7 @@ mfex_avx512_process(struct dp_packet_batch *packets,
/* Process UDP header. */
mfex_handle_ipv6_l4((void *)&pkt[54], &blocks[9]);
dp_packet_update_rss_hash_ipv6_tcp_udp(packet);
- mfex_udp_set_hwol(packet);
+ dp_packet_l4_proto_set_udp(packet);
} break;
case PROFILE_ETH_IPV6_TCP: {
@@ -961,7 +943,7 @@ mfex_avx512_process(struct dp_packet_batch *packets,
}
mfex_handle_tcp_flags(tcp, &blocks[9]);
dp_packet_update_rss_hash_ipv6_tcp_udp(packet);
- mfex_tcp_set_hwol(packet);
+ dp_packet_l4_proto_set_tcp(packet);
} break;
case PROFILE_ETH_VLAN_IPV6_TCP: {
@@ -988,7 +970,7 @@ mfex_avx512_process(struct dp_packet_batch *packets,
}
mfex_handle_tcp_flags(tcp, &blocks[10]);
dp_packet_update_rss_hash_ipv6_tcp_udp(packet);
- mfex_tcp_set_hwol(packet);
+ dp_packet_l4_proto_set_tcp(packet);
} break;
case PROFILE_ETH_VLAN_IPV6_UDP: {
@@ -1010,7 +992,7 @@ mfex_avx512_process(struct dp_packet_batch *packets,
/* Process UDP header. */
mfex_handle_ipv6_l4((void *)&pkt[58], &blocks[10]);
dp_packet_update_rss_hash_ipv6_tcp_udp(packet);
- mfex_udp_set_hwol(packet);
+ dp_packet_l4_proto_set_udp(packet);
} break;
case PROFILE_ETH_IPV4_NVGRE: {
@@ -1021,7 +1003,7 @@ mfex_avx512_process(struct dp_packet_batch *packets,
continue;
}
dp_packet_update_rss_hash_ipv4(packet);
- mfex_udp_set_hwol(packet);
+ dp_packet_l4_proto_set_udp(packet);
} break;
default:
diff --git a/lib/flow.c b/lib/flow.c
index 6c8ce2e1ad..b522f7f116 100644
--- a/lib/flow.c
+++ b/lib/flow.c
@@ -810,7 +810,6 @@ miniflow_extract(struct dp_packet *packet, struct miniflow
*dst)
uint8_t nw_frag, nw_tos, nw_ttl, nw_proto;
uint8_t *ct_nw_proto_p = NULL;
ovs_be16 ct_tp_src = 0, ct_tp_dst = 0;
- bool tunneling;
/* Metadata. */
if (flow_tnl_dst_is_set(&md->tunnel)) {
@@ -864,8 +863,7 @@ miniflow_extract(struct dp_packet *packet, struct miniflow
*dst)
/* Initialize packet's layer pointer and offsets. */
frame = data;
- tunneling = dp_packet_tunnel(packet);
- if (tunneling) {
+ if (dp_packet_tunnel(packet)) {
/* Preserve inner offsets from previous circulation. */
dp_packet_reset_outer_offsets(packet);
} else {
@@ -1063,10 +1061,7 @@ miniflow_extract(struct dp_packet *packet, struct
miniflow *dst)
} else if (dl_type == htons(ETH_TYPE_IPV6)) {
dp_packet_update_rss_hash_ipv6_tcp_udp(packet);
}
- if (dp_packet_l4_checksum_good(packet)
- || dp_packet_ol_l4_csum_partial(packet)) {
- dp_packet_hwol_set_csum_tcp(packet);
- }
+ dp_packet_l4_proto_set_tcp(packet);
}
}
} else if (OVS_LIKELY(nw_proto == IPPROTO_UDP)) {
@@ -1082,14 +1077,7 @@ miniflow_extract(struct dp_packet *packet, struct
miniflow *dst)
} else if (dl_type == htons(ETH_TYPE_IPV6)) {
dp_packet_update_rss_hash_ipv6_tcp_udp(packet);
}
- if (dp_packet_l4_checksum_good(packet)
- || dp_packet_ol_l4_csum_partial(packet)) {
- if (tunneling) {
- dp_packet_hwol_set_outer_udp_csum(packet);
- } else {
- dp_packet_hwol_set_csum_udp(packet);
- }
- }
+ dp_packet_l4_proto_set_udp(packet);
}
} else if (OVS_LIKELY(nw_proto == IPPROTO_SCTP)) {
if (OVS_LIKELY(size >= SCTP_HEADER_LEN)) {
@@ -1099,10 +1087,7 @@ miniflow_extract(struct dp_packet *packet, struct
miniflow *dst)
miniflow_push_be16(mf, tp_dst, sctp->sctp_dst);
miniflow_push_be16(mf, ct_tp_src, ct_tp_src);
miniflow_push_be16(mf, ct_tp_dst, ct_tp_dst);
- if (dp_packet_l4_checksum_good(packet)
- || dp_packet_ol_l4_csum_partial(packet)) {
- dp_packet_hwol_set_csum_sctp(packet);
- }
+ dp_packet_l4_proto_set_sctp(packet);
}
} else if (OVS_LIKELY(nw_proto == IPPROTO_ICMP)) {
if (OVS_LIKELY(size >= ICMP_HEADER_LEN)) {
@@ -1284,21 +1269,12 @@ parse_tcp_flags(struct dp_packet *packet,
if (nw_proto == IPPROTO_TCP && size >= TCP_HEADER_LEN) {
const struct tcp_header *tcp = data;
- if (dp_packet_l4_checksum_good(packet)
- || dp_packet_ol_l4_csum_partial(packet)) {
- dp_packet_hwol_set_csum_tcp(packet);
- }
+ dp_packet_l4_proto_set_tcp(packet);
return TCP_FLAGS(tcp->tcp_ctl);
} else if (nw_proto == IPPROTO_UDP && size >= UDP_HEADER_LEN) {
- if (dp_packet_l4_checksum_good(packet)
- || dp_packet_ol_l4_csum_partial(packet)) {
- dp_packet_hwol_set_csum_udp(packet);
- }
+ dp_packet_l4_proto_set_udp(packet);
} else if (nw_proto == IPPROTO_SCTP && size >= SCTP_HEADER_LEN) {
- if (dp_packet_l4_checksum_good(packet)
- || dp_packet_ol_l4_csum_partial(packet)) {
- dp_packet_hwol_set_csum_sctp(packet);
- }
+ dp_packet_l4_proto_set_sctp(packet);
}
}
@@ -3245,7 +3221,7 @@ flow_compose_l4_csum(struct dp_packet *p, const struct
flow *flow,
tcp->tcp_csum = 0;
tcp->tcp_csum = csum_finish(csum_continue(pseudo_hdr_csum,
tcp, l4_len));
- dp_packet_ol_set_l4_csum_good(p);
+ dp_packet_l4_checksum_set_good(p);
} else if (flow->nw_proto == IPPROTO_UDP) {
struct udp_header *udp = dp_packet_l4(p);
@@ -3255,7 +3231,7 @@ flow_compose_l4_csum(struct dp_packet *p, const struct
flow *flow,
if (!udp->udp_csum) {
udp->udp_csum = htons(0xffff);
}
- dp_packet_ol_set_l4_csum_good(p);
+ dp_packet_l4_checksum_set_good(p);
} else if (flow->nw_proto == IPPROTO_ICMP) {
struct icmp_header *icmp = dp_packet_l4(p);
diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
index 45a4de9f21..af5ecd5b36 100644
--- a/lib/netdev-dpdk.c
+++ b/lib/netdev-dpdk.c
@@ -2652,19 +2652,22 @@ netdev_dpdk_prep_hwol_packet(struct netdev_dpdk *dev,
struct rte_mbuf *mbuf)
{
struct dp_packet *pkt = CONTAINER_OF(mbuf, struct dp_packet, mbuf);
const struct ip_header *ip;
+ bool is_sctp;
bool l3_csum;
+ bool l4_csum;
+ bool is_tcp;
+ bool is_udp;
void *l2;
void *l3;
void *l4;
- const uint64_t all_inner_requests = (RTE_MBUF_F_TX_L4_MASK
- | RTE_MBUF_F_TX_TCP_SEG);
- const uint64_t all_outer_requests = RTE_MBUF_F_TX_OUTER_UDP_CKSUM;
- const uint64_t all_requests = all_inner_requests | all_outer_requests;
+ const uint64_t all_inner_requests = RTE_MBUF_F_TX_TCP_SEG;
if (!dp_packet_ip_checksum_partial(pkt)
&& !dp_packet_inner_ip_checksum_partial(pkt)
- && !(mbuf->ol_flags & all_requests)) {
+ && !dp_packet_l4_checksum_partial(pkt)
+ && !dp_packet_inner_l4_checksum_partial(pkt)
+ && !(mbuf->ol_flags & all_inner_requests)) {
uint64_t unexpected = mbuf->ol_flags & RTE_MBUF_F_TX_OFFLOAD_MASK;
if (OVS_UNLIKELY(unexpected)) {
@@ -2679,9 +2682,10 @@ netdev_dpdk_prep_hwol_packet(struct netdev_dpdk *dev,
struct rte_mbuf *mbuf)
if (dp_packet_tunnel(pkt)
&& (dp_packet_inner_ip_checksum_partial(pkt)
+ || dp_packet_inner_l4_checksum_partial(pkt)
|| (mbuf->ol_flags & all_inner_requests))) {
if (dp_packet_ip_checksum_partial(pkt)
- || (mbuf->ol_flags & all_outer_requests)) {
+ || dp_packet_l4_checksum_partial(pkt)) {
mbuf->outer_l2_len = (char *) dp_packet_l3(pkt) -
(char *) dp_packet_eth(pkt);
mbuf->outer_l3_len = (char *) dp_packet_l4(pkt) -
@@ -2700,6 +2704,11 @@ netdev_dpdk_prep_hwol_packet(struct netdev_dpdk *dev,
struct rte_mbuf *mbuf)
mbuf->ol_flags |= RTE_MBUF_F_TX_OUTER_IP_CKSUM;
}
+ if (dp_packet_l4_checksum_partial(pkt)) {
+ ovs_assert(dp_packet_l4_proto_udp(pkt));
+ mbuf->ol_flags |= RTE_MBUF_F_TX_OUTER_UDP_CKSUM;
+ }
+
ip = dp_packet_l3(pkt);
mbuf->ol_flags |= IP_VER(ip->ip_ihl_ver) == 4
? RTE_MBUF_F_TX_OUTER_IPV4
@@ -2710,6 +2719,10 @@ netdev_dpdk_prep_hwol_packet(struct netdev_dpdk *dev,
struct rte_mbuf *mbuf)
l3 = dp_packet_inner_l3(pkt);
l3_csum = dp_packet_inner_ip_checksum_partial(pkt);
l4 = dp_packet_inner_l4(pkt);
+ l4_csum = dp_packet_inner_l4_checksum_partial(pkt);
+ is_tcp = dp_packet_inner_l4_proto_tcp(pkt);
+ is_udp = dp_packet_inner_l4_proto_udp(pkt);
+ is_sctp = dp_packet_inner_l4_proto_sctp(pkt);
} else {
mbuf->outer_l2_len = 0;
mbuf->outer_l3_len = 0;
@@ -2719,16 +2732,12 @@ netdev_dpdk_prep_hwol_packet(struct netdev_dpdk *dev,
struct rte_mbuf *mbuf)
l3 = dp_packet_inner_l3(pkt);
l3_csum = dp_packet_inner_ip_checksum_partial(pkt);
l4 = dp_packet_inner_l4(pkt);
+ l4_csum = dp_packet_inner_l4_checksum_partial(pkt);
+ is_tcp = dp_packet_inner_l4_proto_tcp(pkt);
+ is_udp = dp_packet_inner_l4_proto_udp(pkt);
+ is_sctp = dp_packet_inner_l4_proto_sctp(pkt);
}
} else {
- if (dp_packet_tunnel(pkt)) {
- /* No inner offload is requested, fallback to non tunnel
- * checksum offloads. */
- if (mbuf->ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM) {
- mbuf->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM;
- }
- mbuf->ol_flags &= ~all_outer_requests;
- }
mbuf->outer_l2_len = 0;
mbuf->outer_l3_len = 0;
@@ -2736,6 +2745,10 @@ netdev_dpdk_prep_hwol_packet(struct netdev_dpdk *dev,
struct rte_mbuf *mbuf)
l3 = dp_packet_l3(pkt);
l3_csum = dp_packet_ip_checksum_partial(pkt);
l4 = dp_packet_l4(pkt);
+ l4_csum = dp_packet_l4_checksum_partial(pkt);
+ is_tcp = dp_packet_l4_proto_tcp(pkt);
+ is_udp = dp_packet_l4_proto_udp(pkt);
+ is_sctp = dp_packet_l4_proto_sctp(pkt);
}
ovs_assert(l4);
@@ -2748,6 +2761,17 @@ netdev_dpdk_prep_hwol_packet(struct netdev_dpdk *dev,
struct rte_mbuf *mbuf)
mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM;
}
+ if (l4_csum) {
+ if (is_tcp) {
+ mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM;
+ } else if (is_udp) {
+ mbuf->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM;
+ } else {
+ ovs_assert(is_sctp);
+ mbuf->ol_flags |= RTE_MBUF_F_TX_SCTP_CKSUM;
+ }
+ }
+
mbuf->l2_len = (char *) l3 - (char *) l2;
mbuf->l3_len = (char *) l4 - (char *) l3;
diff --git a/lib/netdev-dummy.c b/lib/netdev-dummy.c
index 4e699b7958..8427441ace 100644
--- a/lib/netdev-dummy.c
+++ b/lib/netdev-dummy.c
@@ -1201,8 +1201,6 @@ netdev_dummy_rxq_recv(struct netdev_rxq *rxq_, struct
dp_packet_batch *batch,
netdev->custom_stats[0].value++;
netdev->custom_stats[1].value++;
- *dp_packet_ol_flags_ptr(packet) &= ~DP_PACKET_OL_TX_ANY_CKSUM;
-
if (netdev->ol_ip_rx_csum_set_good) {
dp_packet_ip_checksum_set_good(packet);
} else if (netdev->ol_ip_rx_csum_set_bad) {
@@ -1214,13 +1212,13 @@ netdev_dummy_rxq_recv(struct netdev_rxq *rxq_, struct
dp_packet_batch *batch,
}
if (netdev->ol_l4_rx_csum_set_good) {
- dp_packet_ol_set_l4_csum_good(packet);
+ dp_packet_l4_checksum_set_good(packet);
} else if (netdev->ol_l4_rx_csum_set_bad) {
- dp_packet_ol_set_l4_csum_bad(packet);
+ dp_packet_l4_checksum_set_bad(packet);
} else if (netdev->ol_l4_rx_csum_set_partial) {
- dp_packet_ol_set_l4_csum_partial(packet);
+ dp_packet_l4_checksum_set_partial(packet);
} else {
- dp_packet_ol_reset_l4_csum_good(packet);
+ dp_packet_l4_checksum_set_unknown(packet);
}
if (userspace_tso_enabled() && netdev->ol_tso_segsz) {
@@ -1236,10 +1234,8 @@ netdev_dummy_rxq_recv(struct netdev_rxq *rxq_, struct
dp_packet_batch *batch,
ip_csum_good = !!(packet->offloads & DP_PACKET_OL_IP_CKSUM_GOOD);
ip_csum_bad = !!(packet->offloads & DP_PACKET_OL_IP_CKSUM_BAD);
- l4_csum_good = !!(*dp_packet_ol_flags_ptr(packet)
- & DP_PACKET_OL_L4_CKSUM_GOOD);
- l4_csum_bad = !!(*dp_packet_ol_flags_ptr(packet)
- & DP_PACKET_OL_L4_CKSUM_BAD);
+ l4_csum_good = !!(packet->offloads & DP_PACKET_OL_L4_CKSUM_GOOD);
+ l4_csum_bad = !!(packet->offloads & DP_PACKET_OL_L4_CKSUM_BAD);
VLOG_DBG("Rx: packet with csum IP %s, L4 %s, segsz %"PRIu16,
ip_csum_good ? (ip_csum_bad ? "partial" : "good")
: (ip_csum_bad ? "bad" : "unknown"),
@@ -1352,31 +1348,20 @@ netdev_dummy_send(struct netdev *netdev, int qid,
ip_csum_good = !!(packet->offloads & DP_PACKET_OL_IP_CKSUM_GOOD);
ip_csum_bad = !!(packet->offloads & DP_PACKET_OL_IP_CKSUM_BAD);
- l4_csum_good = !!(*dp_packet_ol_flags_ptr(packet)
- & DP_PACKET_OL_L4_CKSUM_GOOD);
- l4_csum_bad = !!(*dp_packet_ol_flags_ptr(packet)
- & DP_PACKET_OL_L4_CKSUM_BAD);
+ l4_csum_good = !!(packet->offloads & DP_PACKET_OL_L4_CKSUM_GOOD);
+ l4_csum_bad = !!(packet->offloads & DP_PACKET_OL_L4_CKSUM_BAD);
VLOG_DBG("Tx: packet with csum IP %s, L4 %s, segsz %"PRIu16
- ", Tx flags %s, %s",
+ ", Tx flags %s",
ip_csum_good ? (ip_csum_bad ? "partial" : "good")
: (ip_csum_bad ? "bad" : "unknown"),
l4_csum_good ? (l4_csum_bad ? "partial" : "good")
: (l4_csum_bad ? "bad" : "unknown"),
dp_packet_get_tso_segsz(packet),
- dp_packet_hwol_tx_l4_checksum(packet) ? "l4_csum"
- : "none",
dp_packet_hwol_is_tso(packet) ? "tso" : "none");
}
- if (dp_packet_ip_checksum_partial(packet)) {
- dp_packet_ol_send_prepare(packet, flags);
- }
- if (dp_packet_hwol_l4_is_tcp(packet)
- && !dp_packet_l4_checksum_good(packet)) {
- dp_packet_ol_send_prepare(packet, flags);
- }
- if (dp_packet_hwol_l4_is_udp(packet)
- && !dp_packet_l4_checksum_good(packet)) {
+ if (dp_packet_ip_checksum_partial(packet)
+ || dp_packet_l4_checksum_partial(packet)) {
dp_packet_ol_send_prepare(packet, flags);
}
diff --git a/lib/netdev-linux.c b/lib/netdev-linux.c
index 98255290de..e4917310ef 100644
--- a/lib/netdev-linux.c
+++ b/lib/netdev-linux.c
@@ -7066,19 +7066,16 @@ netdev_linux_parse_vnet_hdr(struct dp_packet *b)
return EINVAL;
}
- /* Chicken/egg situation, report L4 checksum as partial so that
- * L4 extraction works. */
- dp_packet_ol_set_l4_csum_partial(b);
parse_tcp_flags(b, NULL, NULL, NULL);
if (csum_start == b->l4_ofs
&& ((csum_offset == offsetof(struct tcp_header, tcp_csum)
- && dp_packet_hwol_l4_is_tcp(b))
+ && dp_packet_l4_proto_tcp(b))
|| (csum_offset == offsetof(struct udp_header, udp_csum)
- && dp_packet_hwol_l4_is_udp(b))
+ && dp_packet_l4_proto_udp(b))
|| (csum_offset == offsetof(struct sctp_header, sctp_csum)
- && dp_packet_hwol_l4_is_sctp(b)))) {
- /* Nothing to do, L4 csum is already marked as partial. */
+ && dp_packet_l4_proto_sctp(b)))) {
+ dp_packet_l4_checksum_set_partial(b);
} else {
ovs_be16 *csum_l4;
void *l4;
@@ -7094,12 +7091,10 @@ netdev_linux_parse_vnet_hdr(struct dp_packet *b)
l4 = dp_packet_at(b, csum_start, dp_packet_size(b) - csum_start);
*csum_l4 = csum(l4, dp_packet_size(b) - csum_start);
- if (dp_packet_hwol_l4_is_tcp(b)
- || dp_packet_hwol_l4_is_udp(b)
- || dp_packet_hwol_l4_is_sctp(b)) {
- dp_packet_ol_set_l4_csum_good(b);
- } else {
- b->offloads &= ~DP_PACKET_OL_L4_CKSUM_MASK;
+ if (dp_packet_l4_proto_tcp(b)
+ || dp_packet_l4_proto_udp(b)
+ || dp_packet_l4_proto_sctp(b)) {
+ dp_packet_l4_checksum_set_good(b);
}
}
}
@@ -7170,8 +7165,9 @@ netdev_linux_prepend_vnet_hdr(struct dp_packet *b, int
mtu)
vnet->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
} else {
VLOG_ERR_RL(&rl, "Unknown gso_type for TSO packet. "
- "Flags: %#"PRIx64,
- (uint64_t) *dp_packet_ol_flags_ptr(b));
+ "Flags: %#"PRIx64", Offloads: %"PRIu32,
+ (uint64_t) *dp_packet_ol_flags_ptr(b),
+ b->offloads);
return EINVAL;
}
} else {
@@ -7180,26 +7176,38 @@ netdev_linux_prepend_vnet_hdr(struct dp_packet *b, int
mtu)
vnet->gso_type = VIRTIO_NET_HDR_GSO_NONE;
}
- bool l4_is_good = dp_packet_l4_checksum_good(b);
-
- if ((dp_packet_tunnel_vxlan(b)
- || dp_packet_tunnel_geneve(b))
- && dp_packet_hwol_tx_l4_checksum(b)) {
- /* This condition is needed because dp-packet doesn't currently track
- * outer and inner checksum statuses seperately. In the case of these
- * two tunnel types we can end up setting outer l4 as good but still
- * need to complete the inner l4. */
- l4_is_good = !(dp_packet_hwol_l4_is_tcp(b) ||
- dp_packet_hwol_l4_is_udp(b));
- }
-
- if (l4_is_good) {
+ if (dp_packet_l4_checksum_good(b)
+ && (!dp_packet_tunnel(b)
+ || dp_packet_inner_l4_checksum_good(b))) {
/* The packet has good L4 checksum. No need to validate again. */
vnet->csum_start = vnet->csum_offset = (OVS_FORCE __virtio16) 0;
vnet->flags = VIRTIO_NET_HDR_F_DATA_VALID;
- } else if (dp_packet_hwol_tx_l4_checksum(b)) {
+ } else if (dp_packet_l4_checksum_partial(b)
+ || dp_packet_inner_l4_checksum_partial(b)) {
+ const struct ip_header *ip_hdr;
+ void *l3_off;
+ void *l4_off;
+ bool is_sctp;
+ bool is_tcp;
+ bool is_udp;
+
+ if (dp_packet_inner_l4_checksum_partial(b)) {
+ l3_off = dp_packet_inner_l3(b);
+ l4_off = dp_packet_inner_l4(b);
+ is_tcp = dp_packet_inner_l4_proto_tcp(b);
+ is_udp = dp_packet_inner_l4_proto_udp(b);
+ is_sctp = dp_packet_inner_l4_proto_sctp(b);
+ } else {
+ l3_off = dp_packet_l3(b);
+ l4_off = dp_packet_l4(b);
+ is_tcp = dp_packet_l4_proto_tcp(b);
+ is_udp = dp_packet_l4_proto_udp(b);
+ is_sctp = dp_packet_l4_proto_sctp(b);
+ }
+ ip_hdr = l3_off;
+
/* The csum calculation is offloaded. */
- if (dp_packet_hwol_l4_is_tcp(b)) {
+ if (is_tcp) {
/* Virtual I/O Device (VIRTIO) Version 1.1
* 5.1.6.2 Packet Transmission
* If the driver negotiated VIRTIO_NET_F_CSUM, it can skip
@@ -7214,17 +7222,9 @@ netdev_linux_prepend_vnet_hdr(struct dp_packet *b, int
mtu)
* the TCP pseudo header, so that replacing it by the ones
* complement checksum of the TCP header and body will give
* the correct result. */
- void *l3_off = dp_packet_inner_l3(b);
- void *l4_off = dp_packet_inner_l4(b);
-
- if (!l3_off || !l4_off) {
- l3_off = dp_packet_l3(b);
- l4_off = dp_packet_l4(b);
- }
-
- const struct ip_header *ip_hdr = l3_off;
struct tcp_header *tcp_hdr = l4_off;
ovs_be16 csum = 0;
+
if (IP_VER(ip_hdr->ip_ihl_ver) == 4) {
csum = ~csum_finish(packet_csum_pseudoheader(ip_hdr));
} else if (IP_VER(ip_hdr->ip_ihl_ver) == 6) {
@@ -7238,19 +7238,10 @@ netdev_linux_prepend_vnet_hdr(struct dp_packet *b, int
mtu)
(char *) dp_packet_data(b));
vnet->csum_offset = (OVS_FORCE __virtio16) __builtin_offsetof(
struct tcp_header, tcp_csum);
- } else if (dp_packet_hwol_l4_is_udp(b)) {
- /* Favour the inner packet when indicating checksum offsets. */
- void *l3_off = dp_packet_inner_l3(b);
- void *l4_off = dp_packet_inner_l4(b);
-
- if (!l3_off || !l4_off) {
- l3_off = dp_packet_l3(b);
- l4_off = dp_packet_l4(b);
- }
-
- const struct ip_header *ip_hdr = l3_off;
+ } else if (is_udp) {
struct udp_header *udp_hdr = l4_off;
ovs_be16 csum = 0;
+
if (IP_VER(ip_hdr->ip_ihl_ver) == 4) {
csum = ~csum_finish(packet_csum_pseudoheader(ip_hdr));
} else if (IP_VER(ip_hdr->ip_ihl_ver) == 6) {
@@ -7264,7 +7255,7 @@ netdev_linux_prepend_vnet_hdr(struct dp_packet *b, int
mtu)
(char *) dp_packet_data(b));;
vnet->csum_offset = (OVS_FORCE __virtio16) __builtin_offsetof(
struct udp_header, udp_csum);
- } else if (dp_packet_hwol_l4_is_sctp(b)) {
+ } else if (is_sctp) {
/* The Linux kernel networking stack only supports csum_start
* and csum_offset when SCTP GSO is enabled. See kernel's
* skb_csum_hwoffload_help(). Currently there is no SCTP
@@ -7272,11 +7263,12 @@ netdev_linux_prepend_vnet_hdr(struct dp_packet *b, int
mtu)
vnet->csum_start = vnet->csum_offset = (OVS_FORCE __virtio16) 0;
vnet->flags = 0;
} else {
- /* This should only happen when DP_PACKET_OL_TX_L4_MASK includes
- * a new flag that is not covered in above checks. */
+ /* This should only happen when a new L4 proto
+ * is not covered in above checks. */
VLOG_WARN_RL(&rl, "Unsupported L4 checksum offload. "
- "Flags: %"PRIu64,
- (uint64_t)*dp_packet_ol_flags_ptr(b));
+ "Flags: %"PRIu64", Offloads: %"PRIu32,
+ (uint64_t)*dp_packet_ol_flags_ptr(b),
+ b->offloads);
vnet->csum_start = vnet->csum_offset = (OVS_FORCE __virtio16) 0;
vnet->flags = 0;
}
diff --git a/lib/netdev-native-tnl.c b/lib/netdev-native-tnl.c
index fde4fe8e87..e070088259 100644
--- a/lib/netdev-native-tnl.c
+++ b/lib/netdev-native-tnl.c
@@ -233,8 +233,7 @@ udp_extract_tnl_md(struct dp_packet *packet, struct
flow_tnl *tnl,
if (udp->udp_csum) {
bool bad_csum = dp_packet_l4_checksum_bad(packet);
- if (OVS_LIKELY(!bad_csum && !dp_packet_ol_l4_csum_partial(packet))
- && OVS_UNLIKELY(!dp_packet_l4_checksum_good(packet))) {
+ if (OVS_UNLIKELY(!bad_csum && dp_packet_l4_checksum_unknown(packet))) {
uint32_t csum;
COVERAGE_INC(native_tnl_l4csum_checked);
if (netdev_tnl_is_header_ipv6(dp_packet_data(packet))) {
@@ -308,16 +307,11 @@ netdev_tnl_push_udp_header(const struct netdev *netdev
OVS_UNUSED,
udp->udp_src = udp_src;
udp->udp_len = htons(ip_tot_size);
+ dp_packet_l4_proto_set_udp(packet);
if (udp->udp_csum) {
- dp_packet_ol_reset_l4_csum_good(packet);
- if (dp_packet_tunnel_geneve(packet)
- || dp_packet_tunnel_vxlan(packet)) {
- dp_packet_hwol_set_outer_udp_csum(packet);
- } else {
- dp_packet_hwol_set_csum_udp(packet);
- }
+ dp_packet_l4_checksum_set_partial(packet);
} else {
- dp_packet_ol_set_l4_csum_good(packet);
+ dp_packet_l4_checksum_set_good(packet);
}
if (l3_ofs != UINT16_MAX) {
@@ -880,8 +874,8 @@ netdev_gtpu_push_header(const struct netdev *netdev,
udp->udp_src = udp_src;
udp->udp_len = htons(ip_tot_size);
/* Postpone checksum to the egress netdev. */
- dp_packet_hwol_set_csum_udp(packet);
- dp_packet_ol_reset_l4_csum_good(packet);
+ dp_packet_l4_proto_set_udp(packet);
+ dp_packet_l4_checksum_set_partial(packet);
gtpuh = ALIGNED_CAST(struct gtpuhdr *, udp + 1);
diff --git a/lib/netdev.c b/lib/netdev.c
index a0aac0dc88..ea147f41f9 100644
--- a/lib/netdev.c
+++ b/lib/netdev.c
@@ -929,7 +929,7 @@ netdev_send(struct netdev *netdev, int qid, struct
dp_packet_batch *batch,
if (dp_packet_hwol_is_tso(packet)
&& (dp_packet_tunnel_vxlan(packet)
|| dp_packet_tunnel_geneve(packet))
- && dp_packet_hwol_is_outer_udp_cksum(packet)) {
+ && dp_packet_l4_checksum_partial(packet)) {
return netdev_send_tso(netdev, qid, batch, concurrent_txq);
}
}
diff --git a/lib/odp-execute-avx512.c b/lib/odp-execute-avx512.c
index 9ed5dff356..13afe0c793 100644
--- a/lib/odp-execute-avx512.c
+++ b/lib/odp-execute-avx512.c
@@ -498,8 +498,8 @@ action_avx512_ipv4_set_addrs(struct dp_packet_batch *batch,
if (nh->ip_proto == IPPROTO_UDP && l4_size >= UDP_HEADER_LEN) {
struct udp_header *uh = dp_packet_l4(packet);
- if (dp_packet_hwol_l4_is_udp(packet)) {
- dp_packet_ol_reset_l4_csum_good(packet);
+ if (dp_packet_l4_checksum_valid(packet)) {
+ dp_packet_l4_checksum_set_partial(packet);
} else if (uh->udp_csum) {
/* New UDP checksum. */
uint16_t old_udp_checksum = ~uh->udp_csum;
@@ -514,8 +514,8 @@ action_avx512_ipv4_set_addrs(struct dp_packet_batch *batch,
}
} else if (nh->ip_proto == IPPROTO_TCP &&
l4_size >= TCP_HEADER_LEN) {
- if (dp_packet_hwol_l4_is_tcp(packet)) {
- dp_packet_ol_reset_l4_csum_good(packet);
+ if (dp_packet_l4_checksum_valid(packet)) {
+ dp_packet_l4_checksum_set_partial(packet);
} else {
/* New TCP checksum. */
struct tcp_header *th = dp_packet_l4(packet);
@@ -691,8 +691,8 @@ action_avx512_set_ipv6(struct dp_packet_batch *batch, const
struct nlattr *a)
if (proto == IPPROTO_UDP && l4_size >= UDP_HEADER_LEN) {
struct udp_header *uh = dp_packet_l4(packet);
- if (dp_packet_hwol_l4_is_udp(packet)) {
- dp_packet_ol_reset_l4_csum_good(packet);
+ if (dp_packet_l4_checksum_valid(packet)) {
+ dp_packet_l4_checksum_set_partial(packet);
} else if (uh->udp_csum) {
delta_checksum = avx512_ipv6_addr_csum_delta(v_packet,
v_new_hdr,
@@ -711,8 +711,8 @@ action_avx512_set_ipv6(struct dp_packet_batch *batch, const
struct nlattr *a)
}
} else if (proto == IPPROTO_TCP && l4_size >= TCP_HEADER_LEN) {
- if (dp_packet_hwol_l4_is_tcp(packet)) {
- dp_packet_ol_reset_l4_csum_good(packet);
+ if (dp_packet_l4_checksum_valid(packet)) {
+ dp_packet_l4_checksum_set_partial(packet);
} else {
delta_checksum = avx512_ipv6_addr_csum_delta(v_packet,
v_new_hdr,
diff --git a/lib/packets.c b/lib/packets.c
index 46e2fb323e..a0bb2ad482 100644
--- a/lib/packets.c
+++ b/lib/packets.c
@@ -1129,15 +1129,15 @@ packet_set_ipv4_addr(struct dp_packet *packet,
pkt_metadata_init_conn(&packet->md);
if (nh->ip_proto == IPPROTO_TCP && l4_size >= TCP_HEADER_LEN) {
- if (dp_packet_hwol_l4_is_tcp(packet)) {
- dp_packet_ol_reset_l4_csum_good(packet);
+ if (dp_packet_l4_checksum_valid(packet)) {
+ dp_packet_l4_checksum_set_partial(packet);
} else {
struct tcp_header *th = dp_packet_l4(packet);
th->tcp_csum = recalc_csum32(th->tcp_csum, old_addr, new_addr);
}
} else if (nh->ip_proto == IPPROTO_UDP && l4_size >= UDP_HEADER_LEN ) {
- if (dp_packet_hwol_l4_is_udp(packet)) {
- dp_packet_ol_reset_l4_csum_good(packet);
+ if (dp_packet_l4_checksum_valid(packet)) {
+ dp_packet_l4_checksum_set_partial(packet);
} else {
struct udp_header *uh = dp_packet_l4(packet);
if (uh->udp_csum) {
@@ -1250,16 +1250,16 @@ packet_update_csum128(struct dp_packet *packet, uint8_t
proto,
size_t l4_size = dp_packet_l4_size(packet);
if (proto == IPPROTO_TCP && l4_size >= TCP_HEADER_LEN) {
- if (dp_packet_hwol_l4_is_tcp(packet)) {
- dp_packet_ol_reset_l4_csum_good(packet);
+ if (dp_packet_l4_checksum_valid(packet)) {
+ dp_packet_l4_checksum_set_partial(packet);
} else {
struct tcp_header *th = dp_packet_l4(packet);
th->tcp_csum = recalc_csum128(th->tcp_csum, addr, new_addr);
}
} else if (proto == IPPROTO_UDP && l4_size >= UDP_HEADER_LEN) {
- if (dp_packet_hwol_l4_is_udp(packet)) {
- dp_packet_ol_reset_l4_csum_good(packet);
+ if (dp_packet_l4_checksum_valid(packet)) {
+ dp_packet_l4_checksum_set_partial(packet);
} else {
struct udp_header *uh = dp_packet_l4(packet);
@@ -1403,8 +1403,8 @@ packet_set_tcp_port(struct dp_packet *packet, ovs_be16
src, ovs_be16 dst)
struct tcp_header *th = dp_packet_l4(packet);
ovs_be16 *csum = NULL;
- if (dp_packet_hwol_l4_is_tcp(packet)) {
- dp_packet_ol_reset_l4_csum_good(packet);
+ if (dp_packet_l4_checksum_valid(packet)) {
+ dp_packet_l4_checksum_set_partial(packet);
} else {
csum = &th->tcp_csum;
}
@@ -1422,8 +1422,8 @@ packet_set_udp_port(struct dp_packet *packet, ovs_be16
src, ovs_be16 dst)
{
struct udp_header *uh = dp_packet_l4(packet);
- if (dp_packet_hwol_l4_is_udp(packet)) {
- dp_packet_ol_reset_l4_csum_good(packet);
+ if (dp_packet_l4_checksum_valid(packet)) {
+ dp_packet_l4_checksum_set_partial(packet);
packet_set_port(&uh->udp_src, src, NULL);
packet_set_port(&uh->udp_dst, dst, NULL);
} else {
@@ -1448,8 +1448,8 @@ packet_set_sctp_port(struct dp_packet *packet, ovs_be16
src, ovs_be16 dst)
{
struct sctp_header *sh = dp_packet_l4(packet);
- if (dp_packet_hwol_l4_is_sctp(packet)) {
- dp_packet_ol_reset_l4_csum_good(packet);
+ if (dp_packet_l4_checksum_valid(packet)) {
+ dp_packet_l4_checksum_set_partial(packet);
sh->sctp_src = src;
sh->sctp_dst = dst;
} else {
@@ -2025,6 +2025,12 @@ packet_tcp_complete_csum(struct dp_packet *p, bool inner)
tcp->tcp_csum = packet_csum_upperlayer6(ip6, tcp, ip6->ip6_nxt,
tcp_sz);
}
+
+ if (inner) {
+ dp_packet_inner_l4_checksum_set_good(p);
+ } else {
+ dp_packet_l4_checksum_set_good(p);
+ }
}
/* Set UDP checksum field in packet 'p' with complete checksum.
@@ -2051,7 +2057,7 @@ packet_udp_complete_csum(struct dp_packet *p, bool inner)
/* Skip csum calculation if the udp_csum is zero. */
if (!udp->udp_csum) {
- return;
+ goto out;
}
udp->udp_csum = 0;
@@ -2070,6 +2076,13 @@ packet_udp_complete_csum(struct dp_packet *p, bool inner)
if (!udp->udp_csum) {
udp->udp_csum = htons(0xffff);
}
+
+out:
+ if (inner) {
+ dp_packet_inner_l4_checksum_set_good(p);
+ } else {
+ dp_packet_l4_checksum_set_good(p);
+ }
}
/* Set SCTP checksum field in packet 'p' with complete checksum.
@@ -2094,4 +2107,10 @@ packet_sctp_complete_csum(struct dp_packet *p, bool
inner)
put_16aligned_be32(&sh->sctp_csum, 0);
csum = crc32c((void *) sh, tp_len);
put_16aligned_be32(&sh->sctp_csum, csum);
+
+ if (inner) {
+ dp_packet_inner_l4_checksum_set_good(p);
+ } else {
+ dp_packet_l4_checksum_set_good(p);
+ }
}
--
2.49.0
_______________________________________________
dev mailing list
d...@openvswitch.org
https://mail.openvswitch.org/mailman/listinfo/ovs-dev
