Module Name: src
Committed By: thorpej
Date: Fri Feb 7 12:35:33 UTC 2020
Modified Files:
src/sys/net: bpf.c if_stats.c net_stats.c pktqueue.c
src/sys/net/npf: npf.c
src/sys/netinet: wqinput.c
src/sys/netipsec: key.c
Log Message:
Use percpu_foreach_xcall() to gather volatile per-cpu counters. These
must be serialized against the interrupts / soft-interrupts in which
they're manipulated, as well as protected from non-atomic 64-bit memory
loads on 32-bit platforms.
To generate a diff of this commit:
cvs rdiff -u -r1.234 -r1.235 src/sys/net/bpf.c
cvs rdiff -u -r1.1 -r1.2 src/sys/net/if_stats.c
cvs rdiff -u -r1.5 -r1.6 src/sys/net/net_stats.c
cvs rdiff -u -r1.10 -r1.11 src/sys/net/pktqueue.c
cvs rdiff -u -r1.41 -r1.42 src/sys/net/npf/npf.c
cvs rdiff -u -r1.7 -r1.8 src/sys/netinet/wqinput.c
cvs rdiff -u -r1.269 -r1.270 src/sys/netipsec/key.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/net/bpf.c
diff -u src/sys/net/bpf.c:1.234 src/sys/net/bpf.c:1.235
--- src/sys/net/bpf.c:1.234 Sat Feb 1 02:54:02 2020
+++ src/sys/net/bpf.c Fri Feb 7 12:35:33 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: bpf.c,v 1.234 2020/02/01 02:54:02 riastradh Exp $ */
+/* $NetBSD: bpf.c,v 1.235 2020/02/07 12:35:33 thorpej Exp $ */
/*
* Copyright (c) 1990, 1991, 1993
@@ -39,7 +39,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.234 2020/02/01 02:54:02 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.235 2020/02/07 12:35:33 thorpej Exp $");
#if defined(_KERNEL_OPT)
#include "opt_bpf.h"
@@ -78,6 +78,7 @@ __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.23
#include <sys/percpu.h>
#include <sys/pserialize.h>
#include <sys/lwp.h>
+#include <sys/xcall.h>
#include <net/if.h>
#include <net/slip.h>
@@ -2395,9 +2396,13 @@ bpf_stats(void *p, void *arg, struct cpu
struct bpf_stat *const stats = p;
struct bpf_stat *sum = arg;
+ int s = splnet();
+
sum->bs_recv += stats->bs_recv;
sum->bs_drop += stats->bs_drop;
sum->bs_capt += stats->bs_capt;
+
+ splx(s);
}
static int
@@ -2410,7 +2415,8 @@ bpf_sysctl_gstats_handler(SYSCTLFN_ARGS)
memset(&sum, 0, sizeof(sum));
node = *rnode;
- percpu_foreach(bpf_gstats_percpu, bpf_stats, &sum);
+ percpu_foreach_xcall(bpf_gstats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET),
+ bpf_stats, &sum);
node.sysctl_data = ∑
node.sysctl_size = sizeof(sum);
Index: src/sys/net/if_stats.c
diff -u src/sys/net/if_stats.c:1.1 src/sys/net/if_stats.c:1.2
--- src/sys/net/if_stats.c:1.1 Wed Jan 29 03:16:28 2020
+++ src/sys/net/if_stats.c Fri Feb 7 12:35:33 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: if_stats.c,v 1.1 2020/01/29 03:16:28 thorpej Exp $ */
+/* $NetBSD: if_stats.c,v 1.2 2020/02/07 12:35:33 thorpej Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -30,11 +30,12 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_stats.c,v 1.1 2020/01/29 03:16:28 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_stats.c,v 1.2 2020/02/07 12:35:33 thorpej Exp $");
#include <sys/param.h>
#include <sys/mbuf.h>
#include <sys/systm.h>
+#include <sys/xcall.h>
#include <net/if.h>
@@ -123,7 +124,8 @@ if_stats_to_if_data(ifnet_t * const ifp,
};
memset(ifi, 0, sizeof(*ifi));
- percpu_foreach(ifp->if_stats, if_stats_to_if_data_cb, &ctx);
+ percpu_foreach_xcall(ifp->if_stats, XC_HIGHPRI_IPL(IPL_SOFTNET),
+ if_stats_to_if_data_cb, &ctx);
}
#else /* ! __IF_STATS_PERCPU */
Index: src/sys/net/net_stats.c
diff -u src/sys/net/net_stats.c:1.5 src/sys/net/net_stats.c:1.6
--- src/sys/net/net_stats.c:1.5 Thu Jun 1 02:45:14 2017
+++ src/sys/net/net_stats.c Fri Feb 7 12:35:33 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: net_stats.c,v 1.5 2017/06/01 02:45:14 chs Exp $ */
+/* $NetBSD: net_stats.c,v 1.6 2020/02/07 12:35:33 thorpej Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -30,12 +30,13 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: net_stats.c,v 1.5 2017/06/01 02:45:14 chs Exp $");
+__KERNEL_RCSID(0, "$NetBSD: net_stats.c,v 1.6 2020/02/07 12:35:33 thorpej Exp $");
#include <sys/types.h>
#include <sys/systm.h>
#include <sys/sysctl.h>
#include <sys/kmem.h>
+#include <sys/xcall.h>
#include <net/net_stats.h>
@@ -80,7 +81,8 @@ netstat_sysctl(percpu_t *stat, u_int nco
ctx.ctx_counters = counters;
ctx.ctx_ncounters = ncounters;
- percpu_foreach(stat, netstat_convert_to_user_cb, &ctx);
+ percpu_foreach_xcall(stat, XC_HIGHPRI_IPL(IPL_SOFTNET),
+ netstat_convert_to_user_cb, &ctx);
node = *rnode;
node.sysctl_data = counters;
Index: src/sys/net/pktqueue.c
diff -u src/sys/net/pktqueue.c:1.10 src/sys/net/pktqueue.c:1.11
--- src/sys/net/pktqueue.c:1.10 Fri Aug 10 07:24:09 2018
+++ src/sys/net/pktqueue.c Fri Feb 7 12:35:33 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pktqueue.c,v 1.10 2018/08/10 07:24:09 msaitoh Exp $ */
+/* $NetBSD: pktqueue.c,v 1.11 2020/02/07 12:35:33 thorpej Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -36,7 +36,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.10 2018/08/10 07:24:09 msaitoh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.11 2020/02/07 12:35:33 thorpej Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -48,6 +48,7 @@ __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v
#include <sys/mbuf.h>
#include <sys/proc.h>
#include <sys/percpu.h>
+#include <sys/xcall.h>
#include <net/pktqueue.h>
@@ -161,9 +162,13 @@ pktq_collect_counts(void *mem, void *arg
const pktq_counters_t *c = mem;
pktq_counters_t *sum = arg;
+ int s = splnet();
+
for (u_int i = 0; i < PQCNT_NCOUNTERS; i++) {
sum->count[i] += c->count[i];
}
+
+ splx(s);
}
uint64_t
@@ -173,7 +178,8 @@ pktq_get_count(pktqueue_t *pq, pktq_coun
if (c != PKTQ_MAXLEN) {
memset(&sum, 0, sizeof(sum));
- percpu_foreach(pq->pq_counters, pktq_collect_counts, &sum);
+ percpu_foreach_xcall(pq->pq_counters,
+ XC_HIGHPRI_IPL(IPL_SOFTNET), pktq_collect_counts, &sum);
}
switch (c) {
case PKTQ_NITEMS:
Index: src/sys/net/npf/npf.c
diff -u src/sys/net/npf/npf.c:1.41 src/sys/net/npf/npf.c:1.42
--- src/sys/net/npf/npf.c:1.41 Sun Aug 25 13:21:03 2019
+++ src/sys/net/npf/npf.c Fri Feb 7 12:35:33 2020
@@ -33,7 +33,7 @@
#ifdef _KERNEL
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: npf.c,v 1.41 2019/08/25 13:21:03 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: npf.c,v 1.42 2020/02/07 12:35:33 thorpej Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -41,6 +41,7 @@ __KERNEL_RCSID(0, "$NetBSD: npf.c,v 1.41
#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/percpu.h>
+#include <sys/xcall.h>
#endif
#include "npf_impl.h"
@@ -201,11 +202,13 @@ __dso_public void
npfk_stats(npf_t *npf, uint64_t *buf)
{
memset(buf, 0, NPF_STATS_SIZE);
- percpu_foreach(npf->stats_percpu, npf_stats_collect, buf);
+ percpu_foreach_xcall(npf->stats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET),
+ npf_stats_collect, buf);
}
__dso_public void
npfk_stats_clear(npf_t *npf)
{
- percpu_foreach(npf->stats_percpu, npf_stats_clear_cb, NULL);
+ percpu_foreach_xcall(npf->stats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET),
+ npf_stats_clear_cb, NULL);
}
Index: src/sys/netinet/wqinput.c
diff -u src/sys/netinet/wqinput.c:1.7 src/sys/netinet/wqinput.c:1.8
--- src/sys/netinet/wqinput.c:1.7 Sat Feb 1 12:54:51 2020
+++ src/sys/netinet/wqinput.c Fri Feb 7 12:35:33 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: wqinput.c,v 1.7 2020/02/01 12:54:51 riastradh Exp $ */
+/* $NetBSD: wqinput.c,v 1.8 2020/02/07 12:35:33 thorpej Exp $ */
/*-
* Copyright (c) 2017 Internet Initiative Japan Inc.
@@ -41,6 +41,7 @@
#include <sys/queue.h>
#include <sys/percpu.h>
#include <sys/sysctl.h>
+#include <sys/xcall.h>
#include <net/if.h>
#include <netinet/wqinput.h>
@@ -98,7 +99,8 @@ wqinput_sysctl_drops_handler(SYSCTLFN_AR
node = *rnode;
wqi = node.sysctl_data;
- percpu_foreach(wqi->wqi_worklists, wqinput_drops, &sum);
+ percpu_foreach_xcall(wqi->wqi_worklists, XC_HIGHPRI_IPL(IPL_SOFTNET),
+ wqinput_drops, &sum);
node.sysctl_data = ∑
error = sysctl_lookup(SYSCTLFN_CALL(&node));
Index: src/sys/netipsec/key.c
diff -u src/sys/netipsec/key.c:1.269 src/sys/netipsec/key.c:1.270
--- src/sys/netipsec/key.c:1.269 Thu Nov 14 03:17:08 2019
+++ src/sys/netipsec/key.c Fri Feb 7 12:35:33 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: key.c,v 1.269 2019/11/14 03:17:08 knakahara Exp $ */
+/* $NetBSD: key.c,v 1.270 2020/02/07 12:35:33 thorpej Exp $ */
/* $FreeBSD: key.c,v 1.3.2.3 2004/02/14 22:23:23 bms Exp $ */
/* $KAME: key.c,v 1.191 2001/06/27 10:46:49 sakane Exp $ */
@@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: key.c,v 1.269 2019/11/14 03:17:08 knakahara Exp $");
+__KERNEL_RCSID(0, "$NetBSD: key.c,v 1.270 2020/02/07 12:35:33 thorpej Exp $");
/*
* This code is referred to RFC 2367
@@ -73,6 +73,7 @@ __KERNEL_RCSID(0, "$NetBSD: key.c,v 1.26
#include <sys/localcount.h>
#include <sys/pserialize.h>
#include <sys/hash.h>
+#include <sys/xcall.h>
#include <net/if.h>
#include <net/route.h>
@@ -3961,7 +3962,8 @@ key_setdumpsa(struct secasvar *sav, u_in
time_mono_to_wall(lt.sadb_lifetime_addtime);
lt.sadb_lifetime_usetime =
time_mono_to_wall(lt.sadb_lifetime_usetime);
- percpu_foreach(sav->lft_c_counters_percpu,
+ percpu_foreach_xcall(sav->lft_c_counters_percpu,
+ XC_HIGHPRI_IPL(IPL_SOFTNET),
key_sum_lifetime_counters, sum);
lt.sadb_lifetime_allocations =
sum[LIFETIME_COUNTER_ALLOCATIONS];
@@ -5027,7 +5029,8 @@ restart:
uint64_t lft_c_bytes = 0;
lifetime_counters_t sum = {0};
- percpu_foreach(sav->lft_c_counters_percpu,
+ percpu_foreach_xcall(sav->lft_c_counters_percpu,
+ XC_HIGHPRI_IPL(IPL_SOFTNET),
key_sum_lifetime_counters, sum);
lft_c_bytes = sum[LIFETIME_COUNTER_BYTES];
@@ -5089,7 +5092,8 @@ restart:
uint64_t lft_c_bytes = 0;
lifetime_counters_t sum = {0};
- percpu_foreach(sav->lft_c_counters_percpu,
+ percpu_foreach_xcall(sav->lft_c_counters_percpu,
+ XC_HIGHPRI_IPL(IPL_SOFTNET),
key_sum_lifetime_counters, sum);
lft_c_bytes = sum[LIFETIME_COUNTER_BYTES];
@@ -7425,8 +7429,8 @@ key_expire(struct secasvar *sav)
lt = mtod(m, struct sadb_lifetime *);
lt->sadb_lifetime_len = PFKEY_UNIT64(sizeof(struct sadb_lifetime));
lt->sadb_lifetime_exttype = SADB_EXT_LIFETIME_CURRENT;
- percpu_foreach(sav->lft_c_counters_percpu,
- key_sum_lifetime_counters, sum);
+ percpu_foreach_xcall(sav->lft_c_counters_percpu,
+ XC_HIGHPRI_IPL(IPL_SOFTNET), key_sum_lifetime_counters, sum);
lt->sadb_lifetime_allocations = sum[LIFETIME_COUNTER_ALLOCATIONS];
lt->sadb_lifetime_bytes = sum[LIFETIME_COUNTER_BYTES];
lt->sadb_lifetime_addtime =
