The branch stable/13 has been updated by mjg: URL: https://cgit.FreeBSD.org/src/commit/?id=fff80a6992f1ba13f2261b066d70a3a97d8837e4
commit fff80a6992f1ba13f2261b066d70a3a97d8837e4 Author: Mateusz Guzik <[email protected]> AuthorDate: 2021-07-22 20:47:24 +0000 Commit: Mateusz Guzik <[email protected]> CommitDate: 2021-08-11 13:37:53 +0000 pf: add hybrid 32- an 64- bit counters Numerous counters got migrated from straight uint64_t to the counter(9) API. Unfortunately the implementation comes with a significiant performance hit on some platforms and cannot be easily fixed. Work around the problem by implementing a pf-specific variant. Reviewed by: kp Sponsored by: Rubicon Communications, LLC ("Netgate") (cherry picked from commit defdcdd5648dc1ea789bc54bb45108fcab546a6b) --- sys/net/pfvar.h | 218 ++++++++++++++++++++++++++++++++++++++++++++++++++++ sys/netpfil/pf/pf.c | 28 ++++++- 2 files changed, 245 insertions(+), 1 deletion(-) diff --git a/sys/net/pfvar.h b/sys/net/pfvar.h index 55ecbecf657a..e381279a113f 100644 --- a/sys/net/pfvar.h +++ b/sys/net/pfvar.h @@ -43,9 +43,11 @@ #include <sys/nv.h> #include <sys/refcount.h> #include <sys/sysctl.h> +#include <sys/smp.h> #include <sys/lock.h> #include <sys/rmlock.h> #include <sys/tree.h> +#include <sys/seqc.h> #include <vm/uma.h> #include <net/radix.h> @@ -64,6 +66,222 @@ #ifdef _KERNEL +#if defined(__arm__) +#define PF_WANT_32_TO_64_COUNTER +#endif + +/* + * A hybrid of 32-bit and 64-bit counters which can be used on platforms where + * counter(9) is very expensive. + * + * As 32-bit counters are expected to overflow, a periodic job sums them up to + * a saved 64-bit state. Fetching the value still walks all CPUs to get the most + * current snapshot. + */ +#ifdef PF_WANT_32_TO_64_COUNTER +struct pf_counter_u64_pcpu { + u_int32_t current; + u_int32_t snapshot; +}; + +struct pf_counter_u64 { + struct pf_counter_u64_pcpu *pfcu64_pcpu; + u_int64_t pfcu64_value; + seqc_t pfcu64_seqc; +}; + +static inline int +pf_counter_u64_init(struct pf_counter_u64 *pfcu64, int flags) +{ + + pfcu64->pfcu64_value = 0; + pfcu64->pfcu64_seqc = 0; + pfcu64->pfcu64_pcpu = uma_zalloc_pcpu(pcpu_zone_8, flags | M_ZERO); + if (__predict_false(pfcu64->pfcu64_pcpu == NULL)) + return (ENOMEM); + return (0); +} + +static inline void +pf_counter_u64_deinit(struct pf_counter_u64 *pfcu64) +{ + + uma_zfree_pcpu(pcpu_zone_8, pfcu64->pfcu64_pcpu); +} + +static inline void +pf_counter_u64_critical_enter(void) +{ + + critical_enter(); +} + +static inline void +pf_counter_u64_critical_exit(void) +{ + + critical_exit(); +} + +static inline void +pf_counter_u64_add_protected(struct pf_counter_u64 *pfcu64, uint32_t n) +{ + struct pf_counter_u64_pcpu *pcpu; + u_int32_t val; + + MPASS(curthread->td_critnest > 0); + pcpu = zpcpu_get(pfcu64->pfcu64_pcpu); + val = atomic_load_int(&pcpu->current); + atomic_store_int(&pcpu->current, val + n); +} + +static inline void +pf_counter_u64_add(struct pf_counter_u64 *pfcu64, uint32_t n) +{ + + critical_enter(); + pf_counter_u64_add_protected(pfcu64, n); + critical_exit(); +} + +static inline u_int64_t +pf_counter_u64_periodic(struct pf_counter_u64 *pfcu64) +{ + struct pf_counter_u64_pcpu *pcpu; + u_int64_t sum; + u_int32_t val; + int cpu; + + MPASS(curthread->td_critnest > 0); + seqc_write_begin(&pfcu64->pfcu64_seqc); + sum = pfcu64->pfcu64_value; + CPU_FOREACH(cpu) { + pcpu = zpcpu_get_cpu(pfcu64->pfcu64_pcpu, cpu); + val = atomic_load_int(&pcpu->current); + sum += (uint32_t)(val - pcpu->snapshot); + pcpu->snapshot = val; + } + pfcu64->pfcu64_value = sum; + seqc_write_end(&pfcu64->pfcu64_seqc); + return (sum); +} + +static inline u_int64_t +pf_counter_u64_fetch(struct pf_counter_u64 *pfcu64) +{ + struct pf_counter_u64_pcpu *pcpu; + u_int64_t sum; + seqc_t seqc; + int cpu; + + for (;;) { + seqc = seqc_read(&pfcu64->pfcu64_seqc); + sum = 0; + CPU_FOREACH(cpu) { + pcpu = zpcpu_get_cpu(pfcu64->pfcu64_pcpu, cpu); + sum += (uint32_t)(atomic_load_int(&pcpu->current) -pcpu->snapshot); + } + sum += pfcu64->pfcu64_value; + if (seqc_consistent(&pfcu64->pfcu64_seqc, seqc)) + break; + } + return (sum); +} + +static inline void +pf_counter_u64_zero_protected(struct pf_counter_u64 *pfcu64) +{ + struct pf_counter_u64_pcpu *pcpu; + int cpu; + + MPASS(curthread->td_critnest > 0); + seqc_write_begin(&pfcu64->pfcu64_seqc); + CPU_FOREACH(cpu) { + pcpu = zpcpu_get_cpu(pfcu64->pfcu64_pcpu, cpu); + pcpu->snapshot = atomic_load_int(&pcpu->current); + } + pfcu64->pfcu64_value = 0; + seqc_write_end(&pfcu64->pfcu64_seqc); +} + +static inline void +pf_counter_u64_zero(struct pf_counter_u64 *pfcu64) +{ + + critical_enter(); + pf_counter_u64_zero_protected(pfcu64); + critical_exit(); +} +#else +struct pf_counter_u64 { + counter_u64_t counter; +}; + +static inline int +pf_counter_u64_init(struct pf_counter_u64 *pfcu64, int flags) +{ + + pfcu64->counter = counter_u64_alloc(flags); + if (__predict_false(pfcu64->counter == NULL)) + return (ENOMEM); + return (0); +} + +static inline void +pf_counter_u64_deinit(struct pf_counter_u64 *pfcu64) +{ + + counter_u64_free(pfcu64->counter); +} + +static inline void +pf_counter_u64_critical_enter(void) +{ + +} + +static inline void +pf_counter_u64_critical_exit(void) +{ + +} + +static inline void +pf_counter_u64_add_protected(struct pf_counter_u64 *pfcu64, uint32_t n) +{ + + counter_u64_add(pfcu64->counter, n); +} + +static inline void +pf_counter_u64_add(struct pf_counter_u64 *pfcu64, uint32_t n) +{ + + pf_counter_u64_add_protected(pfcu64, n); +} + +static inline u_int64_t +pf_counter_u64_fetch(struct pf_counter_u64 *pfcu64) +{ + + return (counter_u64_fetch(pfcu64->counter)); +} + +static inline void +pf_counter_u64_zero_protected(struct pf_counter_u64 *pfcu64) +{ + + counter_u64_zero(pfcu64->counter); +} + +static inline void +pf_counter_u64_zero(struct pf_counter_u64 *pfcu64) +{ + + pf_counter_u64_zero_protected(pfcu64); +} +#endif + SYSCTL_DECL(_net_pf); MALLOC_DECLARE(M_PFHASH); diff --git a/sys/netpfil/pf/pf.c b/sys/netpfil/pf/pf.c index 0db9eaed1ac4..b74491bb88d0 100644 --- a/sys/netpfil/pf/pf.c +++ b/sys/netpfil/pf/pf.c @@ -154,6 +154,11 @@ VNET_DECLARE(int, pf_vnet_active); VNET_DEFINE_STATIC(uint32_t, pf_purge_idx); #define V_pf_purge_idx VNET(pf_purge_idx) +#ifdef PF_WANT_32_TO_64_COUNTER +VNET_DEFINE_STATIC(uint32_t, pf_counter_periodic_iter); +#define V_pf_counter_periodic_iter VNET(pf_counter_periodic_iter) +#endif + /* * Queue for pf_intr() sends. */ @@ -1509,6 +1514,25 @@ pf_intr(void *v) CURVNET_RESTORE(); } +#define pf_purge_thread_period (hz / 10) + +#ifdef PF_WANT_32_TO_64_COUNTER +static void +pf_counter_u64_periodic_main(void) +{ + PF_RULES_RLOCK_TRACKER; + + V_pf_counter_periodic_iter++; + + PF_RULES_RLOCK(); + pf_counter_u64_critical_enter(); + pf_counter_u64_critical_exit(); + PF_RULES_RUNLOCK(); +} +#else +#define pf_counter_u64_periodic_main() do { } while (0) +#endif + void pf_purge_thread(void *unused __unused) { @@ -1516,7 +1540,7 @@ pf_purge_thread(void *unused __unused) sx_xlock(&pf_end_lock); while (pf_end_threads == 0) { - sx_sleep(pf_purge_thread, &pf_end_lock, 0, "pftm", hz / 10); + sx_sleep(pf_purge_thread, &pf_end_lock, 0, "pftm", pf_purge_thread_period); VNET_LIST_RLOCK(); VNET_FOREACH(vnet_iter) { @@ -1528,6 +1552,8 @@ pf_purge_thread(void *unused __unused) continue; } + pf_counter_u64_periodic_main(); + /* * Process 1/interval fraction of the state * table every run. _______________________________________________ [email protected] mailing list https://lists.freebsd.org/mailman/listinfo/dev-commits-src-all To unsubscribe, send any mail to "[email protected]"
