Delayed checksumming means that the TCP/UDP code merely calculates
the pseudo-header checksum and that we only complete the checksum
in the output path if the interface doesn't support TX checksum
offload.
Currently this is only implemented for IPv4. The diff below adds
the missing pieces for IPv6, so that we will later be able to profit
from checksum offload over IPv6.
This is an update of the work I originally did at n2k10 (mostly
cribbed from NetBSD), with some overlapping parts taken from Henning's
csum diff.
Works for me for basic TCP/UDP use over IPv6.
bridge(4) and pf(4) are completely untested.
Index: sys/mbuf.h
===================================================================
RCS file: /cvs/src/sys/sys/mbuf.h,v
retrieving revision 1.159
diff -u -p -r1.159 mbuf.h
--- sys/mbuf.h 5 Oct 2012 12:27:02 -0000 1.159
+++ sys/mbuf.h 12 Nov 2012 18:28:30 -0000
@@ -182,7 +182,7 @@ struct mbuf {
/* Checksumming flags */
#define M_IPV4_CSUM_OUT 0x0001 /* IPv4 checksum needed */
-#define M_TCP_CSUM_OUT 0x0002 /* TCP checksum needed */
+#define M_TCP_CSUM_OUT 0x0002 /* TCP checksum needed */
#define M_UDP_CSUM_OUT 0x0004 /* UDP checksum needed */
#define M_IPV4_CSUM_IN_OK 0x0008 /* IPv4 checksum verified */
#define M_IPV4_CSUM_IN_BAD 0x0010 /* IPv4 checksum bad */
Index: net/pf.c
===================================================================
RCS file: /cvs/src/sys/net/pf.c,v
retrieving revision 1.816
diff -u -p -r1.816 pf.c
--- net/pf.c 6 Nov 2012 12:32:41 -0000 1.816
+++ net/pf.c 12 Nov 2012 20:37:03 -0000
@@ -6108,6 +6108,7 @@ pf_route6(struct mbuf **m, struct pf_rul
if (IN6_IS_SCOPE_EMBED(&dst->sin6_addr))
dst->sin6_addr.s6_addr16[1] = htons(ifp->if_index);
if ((u_long)m0->m_pkthdr.len <= ifp->if_mtu) {
+ in6_proto_cksum_out(m0, ifp);
nd6_output(ifp, ifp, m0, dst, NULL);
} else {
in6_ifstat_inc(ifp, ifs6_in_toobig);
Index: netinet/tcp_output.c
===================================================================
RCS file: /cvs/src/sys/netinet/tcp_output.c,v
retrieving revision 1.97
diff -u -p -r1.97 tcp_output.c
--- netinet/tcp_output.c 20 Sep 2012 10:25:03 -0000 1.97
+++ netinet/tcp_output.c 12 Nov 2012 18:47:48 -0000
@@ -961,8 +961,11 @@ send:
#endif /* INET */
#ifdef INET6
case AF_INET6:
- th->th_sum = in6_cksum(m, IPPROTO_TCP, sizeof(struct ip6_hdr),
- hdrlen - sizeof(struct ip6_hdr) + len);
+ /* Defer checksumming until later (ip6_output() or hardware) */
+ m->m_pkthdr.csum_flags |= M_TCP_CSUM_OUT;
+ if (len + optlen)
+ th->th_sum = in_cksum_addword(th->th_sum,
+ htons((u_int16_t)(len + optlen)));
break;
#endif /* INET6 */
}
Index: netinet/tcp_subr.c
===================================================================
RCS file: /cvs/src/sys/netinet/tcp_subr.c,v
retrieving revision 1.113
diff -u -p -r1.113 tcp_subr.c
--- netinet/tcp_subr.c 10 Mar 2012 12:03:29 -0000 1.113
+++ netinet/tcp_subr.c 12 Nov 2012 18:49:19 -0000
@@ -283,7 +283,9 @@ tcp_template(tp)
th = (struct tcphdr *)(mtod(m, caddr_t) +
sizeof(struct ip6_hdr));
- th->th_sum = 0;
+ th->th_sum = in6_cksum_phdr(&inp->inp_laddr6,
+ &inp->inp_faddr6, htonl(sizeof(struct tcphdr)),
+ htonl(IPPROTO_TCP));
}
break;
#endif /* INET6 */
Index: netinet6/in6.h
===================================================================
RCS file: /cvs/src/sys/netinet6/in6.h,v
retrieving revision 1.61
diff -u -p -r1.61 in6.h
--- netinet6/in6.h 6 Nov 2012 12:32:42 -0000 1.61
+++ netinet6/in6.h 12 Nov 2012 18:41:19 -0000
@@ -757,7 +757,57 @@ struct ip6_mtuinfo {
#ifdef _KERNEL
struct cmsghdr;
+/*
+ * in6_cksum_phdr:
+ *
+ * Compute significant parts of the IPv6 checksum pseudo-header
+ * for use in a delayed TCP/UDP checksum calculation.
+ *
+ * Args:
+ *
+ * src Source IPv6 address
+ * dst Destination IPv6 address
+ * len htonl(proto-hdr-len)
+ * nxt htonl(next-proto-number)
+ *
+ * NOTE: We expect the src and dst addresses to be 16-bit
+ * aligned!
+ */
+static __inline u_int16_t __attribute__((__unused__))
+in6_cksum_phdr(const struct in6_addr *src, const struct in6_addr *dst,
+ u_int32_t len, u_int32_t nxt)
+{
+ u_int32_t sum = 0;
+ const u_int16_t *w;
+
+ w = (const u_int16_t *) src;
+ sum += w[0];
+ if (!IN6_IS_SCOPE_EMBED(src))
+ sum += w[1];
+ sum += w[2]; sum += w[3]; sum += w[4]; sum += w[5];
+ sum += w[6]; sum += w[7];
+
+ w = (const u_int16_t *) dst;
+ sum += w[0];
+ if (!IN6_IS_SCOPE_EMBED(dst))
+ sum += w[1];
+ sum += w[2]; sum += w[3]; sum += w[4]; sum += w[5];
+ sum += w[6]; sum += w[7];
+
+ sum += (u_int16_t)(len >> 16) + (u_int16_t)(len /*& 0xffff*/);
+
+ sum += (u_int16_t)(nxt >> 16) + (u_int16_t)(nxt /*& 0xffff*/);
+
+ sum = (u_int16_t)(sum >> 16) + (u_int16_t)(sum /*& 0xffff*/);
+
+ if (sum > 0xffff)
+ sum -= 0xffff;
+
+ return (sum);
+}
+
int in6_cksum(struct mbuf *, u_int8_t, u_int32_t, u_int32_t);
+extern void in6_proto_cksum_out(struct mbuf *, struct ifnet *);
int in6_localaddr(struct in6_addr *);
int in6_addrscope(struct in6_addr *);
struct in6_ifaddr *in6_ifawithscope(struct ifnet *, struct in6_addr *, u_int);
Index: netinet6/in6_cksum.c
===================================================================
RCS file: /cvs/src/sys/netinet6/in6_cksum.c,v
retrieving revision 1.15
diff -u -p -r1.15 in6_cksum.c
--- netinet6/in6_cksum.c 11 Jun 2008 19:00:50 -0000 1.15
+++ netinet6/in6_cksum.c 12 Nov 2012 20:22:30 -0000
@@ -115,6 +115,10 @@ in6_cksum(struct mbuf *m, u_int8_t nxt,
m->m_pkthdr.len, off, len);
}
+ /* Skip pseudo-header if nxt == 0. */
+ if (nxt == 0)
+ goto skip_phdr;
+
bzero(&uph, sizeof(uph));
/*
@@ -141,6 +145,7 @@ in6_cksum(struct mbuf *m, u_int8_t nxt,
sum += uph.phs[0]; sum += uph.phs[1];
sum += uph.phs[2]; sum += uph.phs[3];
+skip_phdr:
/*
* Secondly calculate a summary of the first mbuf excluding offset.
*/
Index: netinet6/ip6_output.c
===================================================================
RCS file: /cvs/src/sys/netinet6/ip6_output.c,v
retrieving revision 1.132
diff -u -p -r1.132 ip6_output.c
--- netinet6/ip6_output.c 6 Nov 2012 12:32:42 -0000 1.132
+++ netinet6/ip6_output.c 12 Nov 2012 20:23:14 -0000
@@ -134,6 +134,8 @@ int ip6_splithdr(struct mbuf *, struct i
int ip6_getpmtu(struct route_in6 *, struct route_in6 *,
struct ifnet *, struct in6_addr *, u_long *, int *);
int copypktopts(struct ip6_pktopts *, struct ip6_pktopts *, int);
+void in6_delayed_cksum(struct mbuf *, u_int8_t);
+void in6_proto_cksum_out(struct mbuf *, struct ifnet *);
/* Context for non-repeating IDs */
struct idgen32_ctx ip6_id_ctx;
@@ -297,6 +299,12 @@ ip6_output(struct mbuf *m0, struct ip6_p
sspi = tdb->tdb_spi;
sproto = tdb->tdb_sproto;
splx(s);
+
+ /*
+ * If it needs TCP/UDP hardware-checksumming, do the
+ * computation now.
+ */
+ in6_proto_cksum_out(m, NULL);
}
/* Fall through to the routing/multicast handling code */
@@ -875,6 +883,7 @@ reroute:
* transmit packet without fragmentation
*/
if (dontfrag || (!alwaysfrag && tlen <= mtu)) { /* case 1-a and 2-a */
+ in6_proto_cksum_out(m, ifp);
error = nd6_output(ifp, origifp, m, dst, ro->ro_rt);
goto done;
}
@@ -935,6 +944,8 @@ reroute:
ip6->ip6_nxt = IPPROTO_FRAGMENT;
}
+ in6_proto_cksum_out(m, NULL);
+
m0 = m;
error = ip6_fragment(m0, hlen, nextproto, mtu);
@@ -3213,4 +3224,63 @@ void
ip6_randomid_init(void)
{
idgen32_init(&ip6_id_ctx);
+}
+
+/*
+ * Process a delayed payload checksum calculation.
+ */
+void
+in6_delayed_cksum(struct mbuf *m, u_int8_t nxt)
+{
+ int nxtp, offset;
+ u_int16_t csum;
+
+ offset = ip6_lasthdr(m, 0, IPPROTO_IPV6, &nxtp);
+ if (offset <= 0 || nxtp != nxt)
+ /* If the desired next protocol isn't found, punt. */
+ return;
+
+ csum = (u_int16_t)(in6_cksum(m, 0, offset, m->m_pkthdr.len - offset));
+
+ switch (nxt) {
+ case IPPROTO_TCP:
+ offset += offsetof(struct tcphdr, th_sum);
+ break;
+
+ case IPPROTO_UDP:
+ offset += offsetof(struct udphdr, uh_sum);
+ if (csum == 0)
+ csum = 0xffff;
+ break;
+
+ case IPPROTO_ICMPV6:
+ offset += offsetof(struct icmp6_hdr, icmp6_cksum);
+ break;
+ }
+
+ if ((offset + sizeof(u_int16_t)) > m->m_len)
+ m_copyback(m, offset, sizeof(csum), &csum, M_NOWAIT);
+ else
+ *(u_int16_t *)(mtod(m, caddr_t) + offset) = csum;
+}
+
+void
+in6_proto_cksum_out(struct mbuf *m, struct ifnet *ifp)
+{
+ if (m->m_pkthdr.csum_flags & M_TCP_CSUM_OUT) {
+ if (!ifp || !(ifp->if_capabilities & IFCAP_CSUM_TCPv6) ||
+ ifp->if_bridgeport != NULL) {
+ in6_delayed_cksum(m, IPPROTO_TCP);
+ m->m_pkthdr.csum_flags &= ~M_TCP_CSUM_OUT; /* Clear */
+ }
+ } else if (m->m_pkthdr.csum_flags & M_UDP_CSUM_OUT) {
+ if (!ifp || !(ifp->if_capabilities & IFCAP_CSUM_UDPv6) ||
+ ifp->if_bridgeport != NULL) {
+ in6_delayed_cksum(m, IPPROTO_UDP);
+ m->m_pkthdr.csum_flags &= ~M_UDP_CSUM_OUT; /* Clear */
+ }
+ } else if (m->m_pkthdr.csum_flags & M_ICMP_CSUM_OUT) {
+ in6_delayed_cksum(m, IPPROTO_ICMPV6);
+ m->m_pkthdr.csum_flags &= ~M_ICMP_CSUM_OUT; /* Clear */
+ }
}
Index: netinet6/udp6_output.c
===================================================================
RCS file: /cvs/src/sys/netinet6/udp6_output.c,v
retrieving revision 1.17
diff -u -p -r1.17 udp6_output.c
--- netinet6/udp6_output.c 24 Nov 2011 17:39:55 -0000 1.17
+++ netinet6/udp6_output.c 12 Nov 2012 20:21:23 -0000
@@ -259,10 +259,9 @@ udp6_output(struct in6pcb *in6p, struct
ip6->ip6_src = *laddr;
ip6->ip6_dst = *faddr;
- if ((udp6->uh_sum = in6_cksum(m, IPPROTO_UDP,
- sizeof(struct ip6_hdr), plen)) == 0) {
- udp6->uh_sum = 0xffff;
- }
+ udp6->uh_sum = in6_cksum_phdr(laddr, faddr,
+ htonl(plen), htonl(IPPROTO_UDP));
+ m->m_pkthdr.csum_flags |= M_UDP_CSUM_OUT;
flags = 0;
if (in6p->in6p_flags & IN6P_MINMTU)
--
Christian "naddy" Weisgerber [email protected]
