Module Name:    src
Committed By:   ozaki-r
Date:           Thu Aug 10 06:33:51 UTC 2017

Modified Files:
        src/sys/netipsec: xform_ah.c xform_esp.c xform_ipcomp.c

Log Message:
Use pool_cache(9) instead of pool(9) for tdb_crypto objects

The change improves network throughput especially on multi-core systems.


To generate a diff of this commit:
cvs rdiff -u -r1.72 -r1.73 src/sys/netipsec/xform_ah.c
cvs rdiff -u -r1.70 -r1.71 src/sys/netipsec/xform_esp.c
cvs rdiff -u -r1.51 -r1.52 src/sys/netipsec/xform_ipcomp.c

Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

Modified files:

Index: src/sys/netipsec/xform_ah.c
diff -u src/sys/netipsec/xform_ah.c:1.72 src/sys/netipsec/xform_ah.c:1.73
--- src/sys/netipsec/xform_ah.c:1.72	Wed Aug  9 09:48:11 2017
+++ src/sys/netipsec/xform_ah.c	Thu Aug 10 06:33:51 2017
@@ -1,4 +1,4 @@
-/*	$NetBSD: xform_ah.c,v 1.72 2017/08/09 09:48:11 ozaki-r Exp $	*/
+/*	$NetBSD: xform_ah.c,v 1.73 2017/08/10 06:33:51 ozaki-r Exp $	*/
 /*	$FreeBSD: src/sys/netipsec/xform_ah.c,v 1.1.4.1 2003/01/24 05:11:36 sam Exp $	*/
 /*	$OpenBSD: ip_ah.c,v 1.63 2001/06/26 06:18:58 angelos Exp $ */
 /*
@@ -39,7 +39,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: xform_ah.c,v 1.72 2017/08/09 09:48:11 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: xform_ah.c,v 1.73 2017/08/10 06:33:51 ozaki-r Exp $");
 
 #if defined(_KERNEL_OPT)
 #include "opt_inet.h"
@@ -123,7 +123,7 @@ static int ah_output_cb(struct cryptop *
 
 const uint8_t ah_stats[256] = { SADB_AALG_STATS_INIT };
 
-static struct pool ah_tdb_crypto_pool;
+static pool_cache_t ah_tdb_crypto_pool_cache;
 static size_t ah_pool_item_size;
 
 /*
@@ -702,7 +702,7 @@ ah_input(struct mbuf *m, struct secasvar
 
 	KASSERTMSG(size <= ah_pool_item_size,
 	    "size=%zu > ah_pool_item_size=%zu\n", size, ah_pool_item_size);
-	tc = pool_get(&ah_tdb_crypto_pool, PR_NOWAIT);
+	tc = pool_cache_get(ah_tdb_crypto_pool_cache, PR_NOWAIT);
 	if (tc == NULL) {
 		DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__));
 		stat = AH_STAT_CRYPTO;
@@ -775,7 +775,7 @@ ah_input(struct mbuf *m, struct secasvar
 
 bad:
 	if (tc != NULL)
-		pool_put(&ah_tdb_crypto_pool, tc);
+		pool_cache_put(ah_tdb_crypto_pool_cache, tc);
 	if (crp != NULL)
 		crypto_freereq(crp);
 	if (m != NULL)
@@ -909,7 +909,7 @@ ah_input_cb(struct cryptop *crp)
 	/* Copyback the saved (uncooked) network headers. */
 	m_copyback(m, 0, skip, ptr);
 
-	pool_put(&ah_tdb_crypto_pool, tc);
+	pool_cache_put(ah_tdb_crypto_pool_cache, tc);
 	tc = NULL;
 
 	/*
@@ -957,7 +957,7 @@ bad:
 	if (m != NULL)
 		m_freem(m);
 	if (tc != NULL)
-		pool_put(&ah_tdb_crypto_pool, tc);
+		pool_cache_put(ah_tdb_crypto_pool_cache, tc);
 	if (crp != NULL)
 		crypto_freereq(crp);
 	return error;
@@ -1117,7 +1117,7 @@ ah_output(
 	crda->crd_klen = _KEYBITS(sav->key_auth);
 
 	/* Allocate IPsec-specific opaque crypto info. */
-	tc = pool_get(&ah_tdb_crypto_pool, PR_NOWAIT);
+	tc = pool_cache_get(ah_tdb_crypto_pool_cache, PR_NOWAIT);
 	if (tc == NULL) {
 		crypto_freereq(crp);
 		DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__));
@@ -1151,7 +1151,7 @@ ah_output(
 	    skip, ahx->type, 1);
 	if (error != 0) {
 		m = NULL;	/* mbuf was free'd by ah_massage_headers. */
-		pool_put(&ah_tdb_crypto_pool, tc);
+		pool_cache_put(ah_tdb_crypto_pool_cache, tc);
 		crypto_freereq(crp);
 		goto bad;
 	}
@@ -1165,7 +1165,7 @@ ah_output(
 	if (__predict_false(isr->sp->state == IPSEC_SPSTATE_DEAD ||
 	    sav->state == SADB_SASTATE_DEAD)) {
 		pserialize_read_exit(s);
-		pool_put(&ah_tdb_crypto_pool, tc);
+		pool_cache_put(ah_tdb_crypto_pool_cache, tc);
 		crypto_freereq(crp);
 		AH_STATINC(AH_STAT_NOTDB);
 		error = ENOENT;
@@ -1269,7 +1269,7 @@ ah_output_cb(struct cryptop *crp)
 	m_copyback(m, 0, skip, ptr);
 
 	/* No longer needed. */
-	pool_put(&ah_tdb_crypto_pool, tc);
+	pool_cache_put(ah_tdb_crypto_pool_cache, tc);
 	crypto_freereq(crp);
 
 #ifdef IPSEC_DEBUG
@@ -1299,7 +1299,7 @@ bad:
 	IPSEC_RELEASE_GLOBAL_LOCKS();
 	if (m)
 		m_freem(m);
-	pool_put(&ah_tdb_crypto_pool, tc);
+	pool_cache_put(ah_tdb_crypto_pool_cache, tc);
 	crypto_freereq(crp);
 	return error;
 }
@@ -1350,8 +1350,9 @@ ah_attach(void)
 	ah_pool_item_size = sizeof(struct tdb_crypto) +
 	    sizeof(struct ip) + MAX_IPOPTLEN +
 	    sizeof(struct ah) + sizeof(uint32_t) + ah_max_authsize;
-	pool_init(&ah_tdb_crypto_pool, ah_pool_item_size,
-	    0, 0, 0, "ah_tdb_crypto", NULL, IPL_SOFTNET);
+	ah_tdb_crypto_pool_cache = pool_cache_init(ah_pool_item_size,
+	    coherency_unit, 0, 0, "ah_tdb_crypto", NULL, IPL_SOFTNET,
+	    NULL, NULL, NULL);
 
 	xform_register(&ah_xformsw);
 }

Index: src/sys/netipsec/xform_esp.c
diff -u src/sys/netipsec/xform_esp.c:1.70 src/sys/netipsec/xform_esp.c:1.71
--- src/sys/netipsec/xform_esp.c:1.70	Wed Aug  9 09:48:11 2017
+++ src/sys/netipsec/xform_esp.c	Thu Aug 10 06:33:51 2017
@@ -1,4 +1,4 @@
-/*	$NetBSD: xform_esp.c,v 1.70 2017/08/09 09:48:11 ozaki-r Exp $	*/
+/*	$NetBSD: xform_esp.c,v 1.71 2017/08/10 06:33:51 ozaki-r Exp $	*/
 /*	$FreeBSD: src/sys/netipsec/xform_esp.c,v 1.2.2.1 2003/01/24 05:11:36 sam Exp $	*/
 /*	$OpenBSD: ip_esp.c,v 1.69 2001/06/26 06:18:59 angelos Exp $ */
 
@@ -39,7 +39,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: xform_esp.c,v 1.70 2017/08/09 09:48:11 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: xform_esp.c,v 1.71 2017/08/10 06:33:51 ozaki-r Exp $");
 
 #if defined(_KERNEL_OPT)
 #include "opt_inet.h"
@@ -104,7 +104,7 @@ static int esp_output_cb(struct cryptop 
 
 const uint8_t esp_stats[256] = { SADB_EALG_STATS_INIT };
 
-static struct pool esp_tdb_crypto_pool;
+static pool_cache_t esp_tdb_crypto_pool_cache;
 static size_t esp_pool_item_size;
 
 /*
@@ -383,7 +383,7 @@ esp_input(struct mbuf *m, struct secasva
 	KASSERTMSG(sizeof(*tc) + extra <= esp_pool_item_size,
 	    "sizeof(*tc) + extra=%zu > esp_pool_item_size=%zu\n",
 	    sizeof(*tc) + extra, esp_pool_item_size);
-	tc = pool_get(&esp_tdb_crypto_pool, PR_NOWAIT);
+	tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT);
 	if (tc == NULL) {
 		DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__));
 		error = ENOBUFS;
@@ -437,7 +437,7 @@ esp_input(struct mbuf *m, struct secasva
 	 */
 	if (__predict_false(sav->state == SADB_SASTATE_DEAD)) {
 		pserialize_read_exit(s);
-		pool_put(&esp_tdb_crypto_pool, tc);
+		pool_cache_put(esp_tdb_crypto_pool_cache, tc);
 		crypto_freereq(crp);
 		ESP_STATINC(ESP_STAT_NOTDB);
 		return ENOENT;
@@ -481,7 +481,7 @@ esp_input(struct mbuf *m, struct secasva
 	return crypto_dispatch(crp);
 
 out2:
-	pool_put(&esp_tdb_crypto_pool, tc);
+	pool_cache_put(esp_tdb_crypto_pool_cache, tc);
 out1:
 	crypto_freereq(crp);
 out:
@@ -608,7 +608,7 @@ esp_input_cb(struct cryptop *crp)
 	}
 
 	/* Release the crypto descriptors */
-	pool_put(&esp_tdb_crypto_pool, tc);
+	pool_cache_put(esp_tdb_crypto_pool_cache, tc);
 	tc = NULL;
 	crypto_freereq(crp), crp = NULL;
 
@@ -699,7 +699,7 @@ bad:
 	if (m != NULL)
 		m_freem(m);
 	if (tc != NULL)
-		pool_put(&esp_tdb_crypto_pool, tc);
+		pool_cache_put(esp_tdb_crypto_pool_cache, tc);
 	if (crp != NULL)
 		crypto_freereq(crp);
 	return error;
@@ -905,7 +905,7 @@ esp_output(
 		crda = crp->crp_desc;
 
 	/* IPsec-specific opaque crypto info. */
-	tc = pool_get(&esp_tdb_crypto_pool, PR_NOWAIT);
+	tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT);
 	if (tc == NULL) {
 		crypto_freereq(crp);
 		DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__));
@@ -923,7 +923,7 @@ esp_output(
 	if (__predict_false(isr->sp->state == IPSEC_SPSTATE_DEAD ||
 	    sav->state == SADB_SASTATE_DEAD)) {
 		pserialize_read_exit(s);
-		pool_put(&esp_tdb_crypto_pool, tc);
+		pool_cache_put(esp_tdb_crypto_pool_cache, tc);
 		crypto_freereq(crp);
 		ESP_STATINC(ESP_STAT_NOTDB);
 		error = ENOENT;
@@ -1043,7 +1043,7 @@ esp_output_cb(struct cryptop *crp)
 		AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]);
 
 	/* Release crypto descriptors. */
-	pool_put(&esp_tdb_crypto_pool, tc);
+	pool_cache_put(esp_tdb_crypto_pool_cache, tc);
 	crypto_freereq(crp);
 
 #ifdef IPSEC_DEBUG
@@ -1077,7 +1077,7 @@ bad:
 	IPSEC_RELEASE_GLOBAL_LOCKS();
 	if (m)
 		m_freem(m);
-	pool_put(&esp_tdb_crypto_pool, tc);
+	pool_cache_put(esp_tdb_crypto_pool_cache, tc);
 	crypto_freereq(crp);
 	return error;
 }
@@ -1102,8 +1102,9 @@ esp_attach(void)
 	extern int ah_max_authsize;
 	KASSERT(ah_max_authsize != 0);
 	esp_pool_item_size = sizeof(struct tdb_crypto) + ah_max_authsize;
-	pool_init(&esp_tdb_crypto_pool, esp_pool_item_size,
-	    0, 0, 0, "esp_tdb_crypto", NULL, IPL_SOFTNET);
+	esp_tdb_crypto_pool_cache = pool_cache_init(esp_pool_item_size,
+	    coherency_unit, 0, 0, "esp_tdb_crypto", NULL, IPL_SOFTNET,
+	    NULL, NULL, NULL);
 
 #define	MAXIV(xform)					\
 	if (xform.ivsize > esp_max_ivlen)		\

Index: src/sys/netipsec/xform_ipcomp.c
diff -u src/sys/netipsec/xform_ipcomp.c:1.51 src/sys/netipsec/xform_ipcomp.c:1.52
--- src/sys/netipsec/xform_ipcomp.c:1.51	Wed Aug  9 09:48:11 2017
+++ src/sys/netipsec/xform_ipcomp.c	Thu Aug 10 06:33:51 2017
@@ -1,4 +1,4 @@
-/*	$NetBSD: xform_ipcomp.c,v 1.51 2017/08/09 09:48:11 ozaki-r Exp $	*/
+/*	$NetBSD: xform_ipcomp.c,v 1.52 2017/08/10 06:33:51 ozaki-r Exp $	*/
 /*	$FreeBSD: src/sys/netipsec/xform_ipcomp.c,v 1.1.4.1 2003/01/24 05:11:36 sam Exp $	*/
 /* $OpenBSD: ip_ipcomp.c,v 1.1 2001/07/05 12:08:52 jjbg Exp $ */
 
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: xform_ipcomp.c,v 1.51 2017/08/09 09:48:11 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: xform_ipcomp.c,v 1.52 2017/08/10 06:33:51 ozaki-r Exp $");
 
 /* IP payload compression protocol (IPComp), see RFC 2393 */
 #if defined(_KERNEL_OPT)
@@ -89,7 +89,7 @@ static int ipcomp_output_cb(struct crypt
 
 const uint8_t ipcomp_stats[256] = { SADB_CALG_STATS_INIT };
 
-static struct pool ipcomp_tdb_crypto_pool;
+static pool_cache_t ipcomp_tdb_crypto_pool_cache;
 
 const struct comp_algo *
 ipcomp_algorithm_lookup(int alg)
@@ -165,7 +165,7 @@ ipcomp_input(struct mbuf *m, struct seca
 		return ENOBUFS;
 	}
 	/* Get IPsec-specific opaque pointer */
-	tc = pool_get(&ipcomp_tdb_crypto_pool, PR_NOWAIT);
+	tc = pool_cache_get(ipcomp_tdb_crypto_pool_cache, PR_NOWAIT);
 	if (tc == NULL) {
 		m_freem(m);
 		crypto_freereq(crp);
@@ -178,7 +178,7 @@ ipcomp_input(struct mbuf *m, struct seca
 	if (error) {
 		DPRINTF(("%s: m_makewritable failed\n", __func__));
 		m_freem(m);
-		pool_put(&ipcomp_tdb_crypto_pool, tc);
+		pool_cache_put(ipcomp_tdb_crypto_pool_cache, tc);
 		crypto_freereq(crp);
 		IPCOMP_STATINC(IPCOMP_STAT_CRYPTO);
 		return error;
@@ -192,7 +192,7 @@ ipcomp_input(struct mbuf *m, struct seca
 	 */
 	if (__predict_false(sav->state == SADB_SASTATE_DEAD)) {
 		pserialize_read_exit(s);
-		pool_put(&ipcomp_tdb_crypto_pool, tc);
+		pool_cache_put(ipcomp_tdb_crypto_pool_cache, tc);
 		crypto_freereq(crp);
 		IPCOMP_STATINC(IPCOMP_STAT_NOTDB);
 		return ENOENT;
@@ -318,7 +318,7 @@ ipcomp_input_cb(struct cryptop *crp)
 	clen = crp->crp_olen;		/* Length of data after processing */
 
 	/* Release the crypto descriptors */
-	pool_put(&ipcomp_tdb_crypto_pool, tc);
+	pool_cache_put(ipcomp_tdb_crypto_pool_cache, tc);
 	tc = NULL;
 	crypto_freereq(crp), crp = NULL;
 
@@ -374,7 +374,7 @@ bad:
 	if (m)
 		m_freem(m);
 	if (tc != NULL)
-		pool_put(&ipcomp_tdb_crypto_pool, tc);
+		pool_cache_put(ipcomp_tdb_crypto_pool_cache, tc);
 	if (crp)
 		crypto_freereq(crp);
 	return error;
@@ -487,7 +487,7 @@ ipcomp_output(
 	crdc->crd_alg = ipcompx->type;
 
 	/* IPsec-specific opaque crypto info */
-	tc = pool_get(&ipcomp_tdb_crypto_pool, PR_NOWAIT);
+	tc = pool_cache_get(ipcomp_tdb_crypto_pool_cache, PR_NOWAIT);
 	if (tc == NULL) {
 		IPCOMP_STATINC(IPCOMP_STAT_CRYPTO);
 		DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__));
@@ -505,7 +505,7 @@ ipcomp_output(
 	if (__predict_false(isr->sp->state == IPSEC_SPSTATE_DEAD ||
 	    sav->state == SADB_SASTATE_DEAD)) {
 		pserialize_read_exit(s);
-		pool_put(&ipcomp_tdb_crypto_pool, tc);
+		pool_cache_put(ipcomp_tdb_crypto_pool_cache, tc);
 		crypto_freereq(crp);
 		IPCOMP_STATINC(IPCOMP_STAT_NOTDB);
 		error = ENOENT;
@@ -675,7 +675,7 @@ ipcomp_output_cb(struct cryptop *crp)
 
 
 	/* Release the crypto descriptor */
-	pool_put(&ipcomp_tdb_crypto_pool, tc);
+	pool_cache_put(ipcomp_tdb_crypto_pool_cache, tc);
 	crypto_freereq(crp);
 
 	/* NB: m is reclaimed by ipsec_process_done. */
@@ -691,7 +691,7 @@ bad:
 	IPSEC_RELEASE_GLOBAL_LOCKS();
 	if (m)
 		m_freem(m);
-	pool_put(&ipcomp_tdb_crypto_pool, tc);
+	pool_cache_put(ipcomp_tdb_crypto_pool_cache, tc);
 	crypto_freereq(crp);
 	return error;
 }
@@ -711,7 +711,8 @@ void
 ipcomp_attach(void)
 {
 	ipcompstat_percpu = percpu_alloc(sizeof(uint64_t) * IPCOMP_NSTATS);
-	pool_init(&ipcomp_tdb_crypto_pool, sizeof(struct tdb_crypto),
-	    0, 0, 0, "ipcomp_tdb_crypto", NULL, IPL_SOFTNET);
+	ipcomp_tdb_crypto_pool_cache = pool_cache_init(sizeof(struct tdb_crypto),
+	    coherency_unit, 0, 0, "ipcomp_tdb_crypto", NULL, IPL_SOFTNET,
+	    NULL, NULL, NULL);
 	xform_register(&ipcomp_xformsw);
 }

Reply via email to