sepherosa_gmail.com updated this revision to Diff 17347.
sepherosa_gmail.com added a comment.
Address hps's cache pollution concern for queue_mbuf path. Similar approach
was in the patch obtained from rrs.
CHANGES SINCE LAST UPDATE
https://reviews.freebsd.org/D6689?vs=17214&id=17347
REVISION DETAIL
https://reviews.freebsd.org/D6689
AFFECTED FILES
sys/netinet/tcp_lro.c
sys/netinet/tcp_lro.h
EMAIL PREFERENCES
https://reviews.freebsd.org/settings/panel/emailpreferences/
To: sepherosa_gmail.com, rrs, gallatin, glebius, gnn, bz, rwatson, #transport,
hselasky
Cc: freebsd-net-list
diff --git a/sys/netinet/tcp_lro.h b/sys/netinet/tcp_lro.h
--- a/sys/netinet/tcp_lro.h
+++ b/sys/netinet/tcp_lro.h
@@ -40,6 +40,7 @@
struct lro_entry {
LIST_ENTRY(lro_entry) next;
+ LIST_ENTRY(lro_entry) hash_next;
struct mbuf *m_head;
struct mbuf *m_tail;
union {
@@ -95,6 +96,8 @@
unsigned short lro_ackcnt_lim; /* max # of aggregated ACKs */
unsigned lro_length_lim; /* max len of aggregated data */
+ u_long lro_hashsz;
+ struct lro_head *lro_hash;
struct lro_head lro_active;
struct lro_head lro_free;
};
diff --git a/sys/netinet/tcp_lro.c b/sys/netinet/tcp_lro.c
--- a/sys/netinet/tcp_lro.c
+++ b/sys/netinet/tcp_lro.c
@@ -68,19 +68,24 @@
#endif
static void tcp_lro_rx_done(struct lro_ctrl *lc);
+static int tcp_lro_rx2(struct lro_ctrl *lc, struct mbuf *m,
+ uint32_t csum, int use_hash);
static __inline void
-tcp_lro_active_insert(struct lro_ctrl *lc, struct lro_entry *le)
+tcp_lro_active_insert(struct lro_ctrl *lc, struct lro_head *bucket,
+ struct lro_entry *le)
{
LIST_INSERT_HEAD(&lc->lro_active, le, next);
+ LIST_INSERT_HEAD(bucket, le, hash_next);
}
static __inline void
tcp_lro_active_remove(struct lro_entry *le)
{
- LIST_REMOVE(le, next);
+ LIST_REMOVE(le, next); /* active list */
+ LIST_REMOVE(le, hash_next); /* hash bucket */
}
int
@@ -95,7 +100,7 @@
{
struct lro_entry *le;
size_t size;
- unsigned i;
+ unsigned i, elements;
lc->lro_bad_csum = 0;
lc->lro_queued = 0;
@@ -110,6 +115,18 @@
LIST_INIT(&lc->lro_free);
LIST_INIT(&lc->lro_active);
+ /* create hash table to accelerate entry lookup */
+ if (lro_entries > lro_mbufs)
+ elements = lro_entries;
+ else
+ elements = lro_mbufs;
+ lc->lro_hash = phashinit_flags(elements, M_LRO, &lc->lro_hashsz,
+ HASH_NOWAIT);
+ if (lc->lro_hash == NULL) {
+ memset(lc, 0, sizeof(*lc));
+ return (ENOMEM);
+ }
+
/* compute size to allocate */
size = (lro_mbufs * sizeof(struct lro_mbuf_sort)) +
(lro_entries * sizeof(*le));
@@ -147,6 +164,13 @@
m_freem(le->m_head);
}
+ /* free hash table */
+ if (lc->lro_hash != NULL) {
+ free(lc->lro_hash, M_LRO);
+ lc->lro_hash = NULL;
+ }
+ lc->lro_hashsz = 0;
+
/* free mbuf array, if any */
for (x = 0; x != lc->lro_mbuf_count; x++)
m_freem(lc->lro_mbuf_data[x].mb);
@@ -487,7 +511,7 @@
}
/* add packet to LRO engine */
- if (tcp_lro_rx(lc, mb, 0) != 0) {
+ if (tcp_lro_rx2(lc, mb, 0, 0) != 0) {
/* input packet to network layer */
(*lc->ifp->if_input)(lc->ifp, mb);
lc->lro_queued++;
@@ -561,8 +585,8 @@
}
#endif
-int
-tcp_lro_rx(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum)
+static int
+tcp_lro_rx2(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum, int use_hash)
{
struct lro_entry *le;
struct ether_header *eh;
@@ -578,6 +602,8 @@
tcp_seq seq;
int error, ip_len, l;
uint16_t eh_type, tcp_data_len;
+ struct lro_head *bucket;
+ uint32_t hash;
/* We expect a contiguous header [eh, ip, tcp]. */
@@ -670,8 +696,41 @@
seq = ntohl(th->th_seq);
+ if (!use_hash) {
+ bucket = &lc->lro_hash[0];
+ goto find;
+ }
+ if (M_HASHTYPE_ISHASH(m)) {
+ hash = m->m_pkthdr.flowid;
+ } else {
+ switch (eh_type) {
+#ifdef INET
+ case ETHERTYPE_IP:
+ hash = ip4->ip_src.s_addr + ip4->ip_dst.s_addr;
+ break;
+#endif
+#ifdef INET6
+ case ETHERTYPE_IPV6:
+ hash = ip6->ip6_src.s6_addr32[0] +
+ ip6->ip6_dst.s6_addr32[0];
+ hash += ip6->ip6_src.s6_addr32[1] +
+ ip6->ip6_dst.s6_addr32[1];
+ hash += ip6->ip6_src.s6_addr32[2] +
+ ip6->ip6_dst.s6_addr32[2];
+ hash += ip6->ip6_src.s6_addr32[3] +
+ ip6->ip6_dst.s6_addr32[3];
+ break;
+#endif
+ default:
+ hash = 0;
+ break;
+ }
+ hash += th->th_sport + th->th_dport;
+ }
+ bucket = &lc->lro_hash[hash % lc->lro_hashsz];
+find:
/* Try to find a matching previous segment. */
- LIST_FOREACH(le, &lc->lro_active, next) {
+ LIST_FOREACH(le, bucket, hash_next) {
if (le->eh_type != eh_type)
continue;
if (le->source_port != th->th_sport ||
@@ -779,7 +838,7 @@
/* Start a new segment chain. */
le = LIST_FIRST(&lc->lro_free);
LIST_REMOVE(le, next);
- tcp_lro_active_insert(lc, le);
+ tcp_lro_active_insert(lc, bucket, le);
getmicrotime(&le->mtime);
/* Start filling in details. */
@@ -837,6 +896,13 @@
return (0);
}
+int
+tcp_lro_rx(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum)
+{
+
+ return tcp_lro_rx2(lc, m, csum, 1);
+}
+
void
tcp_lro_queue_mbuf(struct lro_ctrl *lc, struct mbuf *mb)
{
_______________________________________________
freebsd-net@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/freebsd-net
To unsubscribe, send any mail to "freebsd-net-unsubscr...@freebsd.org"
