On Sat, Oct 23, 2021 at 03:08:29PM +0200, Tobias Heider wrote:
> Now that we have removed all the legacy crypto offloading drivers we can
> simplify the crypto framework API by ripping out the async callbacks.
>
> The diff below removes crypto_dispatch() and crypto_done() and replaces
> them with crypto_invoke() which is blocking and only returns after the
> operation has completed or an error occured.
> The former callback functions are invoked directly by the consumer
> (e.g. IPsec and softraid) instead of the crypto driver.
>
> The plan is to clean up the callers in a follow-up diff to remove
> the callback functions alltogether.
>
> ok?
OK bluhm@
> diff --git a/sys/arch/amd64/amd64/aesni.c b/sys/arch/amd64/amd64/aesni.c
> index 12ed40b37a8..86b3b1947ed 100644
> --- a/sys/arch/amd64/amd64/aesni.c
> +++ b/sys/arch/amd64/amd64/aesni.c
> @@ -699,7 +699,6 @@ aesni_process(struct cryptop *crp)
> out:
> smr_read_leave();
> crp->crp_etype = err;
> - crypto_done(crp);
> return (err);
> }
>
> diff --git a/sys/arch/amd64/amd64/via.c b/sys/arch/amd64/amd64/via.c
> index d6f04c3a88c..bb677399e01 100644
> --- a/sys/arch/amd64/amd64/via.c
> +++ b/sys/arch/amd64/amd64/via.c
> @@ -460,7 +460,6 @@ viac3_crypto_process(struct cryptop *crp)
> }
> out:
> crp->crp_etype = err;
> - crypto_done(crp);
> return (err);
> }
>
> diff --git a/sys/arch/arm64/arm64/cryptox.c b/sys/arch/arm64/arm64/cryptox.c
> index 0b4ff183f04..82def0b8702 100644
> --- a/sys/arch/arm64/arm64/cryptox.c
> +++ b/sys/arch/arm64/arm64/cryptox.c
> @@ -486,6 +486,5 @@ cryptox_process(struct cryptop *crp)
> out:
> smr_read_leave();
> crp->crp_etype = err;
> - crypto_done(crp);
> return (err);
> }
> diff --git a/sys/arch/i386/i386/via.c b/sys/arch/i386/i386/via.c
> index 9822bec8519..a627c9cb19c 100644
> --- a/sys/arch/i386/i386/via.c
> +++ b/sys/arch/i386/i386/via.c
> @@ -468,7 +468,6 @@ viac3_crypto_process(struct cryptop *crp)
> }
> out:
> crp->crp_etype = err;
> - crypto_done(crp);
> return (err);
> }
>
> diff --git a/sys/arch/i386/pci/glxsb.c b/sys/arch/i386/pci/glxsb.c
> index 2dca0de55fb..212a343ecc2 100644
> --- a/sys/arch/i386/pci/glxsb.c
> +++ b/sys/arch/i386/pci/glxsb.c
> @@ -825,7 +825,6 @@ glxsb_crypto_process(struct cryptop *crp)
>
> out:
> crp->crp_etype = err;
> - crypto_done(crp);
> splx(s);
> return (err);
> }
> diff --git a/sys/arch/octeon/dev/octcrypto.c b/sys/arch/octeon/dev/octcrypto.c
> index 26449679790..dbfdc2a0ac1 100644
> --- a/sys/arch/octeon/dev/octcrypto.c
> +++ b/sys/arch/octeon/dev/octcrypto.c
> @@ -658,7 +658,6 @@ out:
> smr_read_leave();
>
> crp->crp_etype = error;
> - crypto_done(crp);
> return error;
> }
>
> diff --git a/sys/crypto/crypto.c b/sys/crypto/crypto.c
> index c4bc6064a9b..dc1a34f0c38 100644
> --- a/sys/crypto/crypto.c
> +++ b/sys/crypto/crypto.c
> @@ -376,33 +376,6 @@ crypto_unregister(u_int32_t driverid, int alg)
> return 0;
> }
>
> -/*
> - * Add crypto request to a queue, to be processed by a kernel thread.
> - */
> -void
> -crypto_dispatch(struct cryptop *crp)
> -{
> - int lock = 1, s;
> - u_int32_t hid;
> -
> - s = splvm();
> - hid = (crp->crp_sid >> 32) & 0xffffffff;
> - if (hid < crypto_drivers_num) {
> - if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_MPSAFE)
> - lock = 0;
> - }
> - splx(s);
> -
> - /* XXXSMP crypto_invoke() is not MP safe */
> - lock = 1;
> -
> - if (lock)
> - KERNEL_LOCK();
> - crypto_invoke(crp);
> - if (lock)
> - KERNEL_UNLOCK();
> -}
> -
> /*
> * Dispatch a crypto request to the appropriate crypto devices.
> */
> @@ -416,7 +389,6 @@ crypto_invoke(struct cryptop *crp)
>
> /* Sanity checks. */
> KASSERT(crp != NULL);
> - KASSERT(crp->crp_callback != NULL);
>
> KERNEL_ASSERT_LOCKED();
>
> @@ -466,7 +438,6 @@ crypto_invoke(struct cryptop *crp)
>
> crp->crp_etype = EAGAIN;
> done:
> - crypto_done(crp);
> splx(s);
> }
>
> @@ -518,14 +489,3 @@ crypto_init(void)
> pool_init(&cryptop_pool, sizeof(struct cryptop), 0, IPL_VM, 0,
> "cryptop", NULL);
> }
> -
> -/*
> - * Invoke the callback on behalf of the driver.
> - */
> -void
> -crypto_done(struct cryptop *crp)
> -{
> - crp->crp_flags |= CRYPTO_F_DONE;
> -
> - crp->crp_callback(crp);
> -}
> diff --git a/sys/crypto/cryptodev.h b/sys/crypto/cryptodev.h
> index 5caa5125a46..7a7b6ed78ca 100644
> --- a/sys/crypto/cryptodev.h
> +++ b/sys/crypto/cryptodev.h
> @@ -169,18 +169,14 @@ struct cryptop {
> #define CRYPTO_F_IMBUF 0x0001 /* Input/output are mbuf chains,
> otherwise contig */
> #define CRYPTO_F_IOV 0x0002 /* Input/output are uio */
> #define CRYPTO_F_MPSAFE 0x0004 /* Do not use kernel lock for callback
> */
> -#define CRYPTO_F_DONE 0x0010 /* request completed */
>
> void *crp_buf; /* Data to be processed */
> - void *crp_opaque; /* Opaque pointer, passed along */
>
> struct cryptodesc *crp_desc; /* List of processing descriptors */
> struct cryptodesc crp_sdesc[2]; /* Static array for small ops */
> int crp_ndesc; /* Amount of descriptors to use */
> int crp_ndescalloc;/* Amount of descriptors allocated */
>
> - void (*crp_callback)(struct cryptop *); /* Callback function */
> -
> caddr_t crp_mac;
> };
>
> @@ -214,14 +210,12 @@ void crypto_init(void);
>
> int crypto_newsession(u_int64_t *, struct cryptoini *, int);
> int crypto_freesession(u_int64_t);
> -void crypto_dispatch(struct cryptop *);
> int crypto_register(u_int32_t, int *,
> int (*)(u_int32_t *, struct cryptoini *), int (*)(u_int64_t),
> int (*)(struct cryptop *));
> int crypto_unregister(u_int32_t, int);
> int32_t crypto_get_driverid(u_int8_t);
> void crypto_invoke(struct cryptop *);
> -void crypto_done(struct cryptop *);
>
> void cuio_copydata(struct uio *, int, int, caddr_t);
> void cuio_copyback(struct uio *, int, int, const void *);
> diff --git a/sys/crypto/cryptosoft.c b/sys/crypto/cryptosoft.c
> index 3fc7e3ca8f0..fe60a1b5da2 100644
> --- a/sys/crypto/cryptosoft.c
> +++ b/sys/crypto/cryptosoft.c
> @@ -1129,7 +1129,6 @@ swcr_process(struct cryptop *crp)
> }
>
> done:
> - crypto_done(crp);
> return 0;
> }
>
> diff --git a/sys/dev/softraid_crypto.c b/sys/dev/softraid_crypto.c
> index 867e9f61df5..1bded7b9ffb 100644
> --- a/sys/dev/softraid_crypto.c
> +++ b/sys/dev/softraid_crypto.c
> @@ -87,13 +87,13 @@ int
> sr_crypto_meta_opt_handler_internal(struct sr_discipline *,
> struct sr_crypto *, struct sr_meta_opt_hdr *);
> int sr_crypto_meta_opt_handler(struct sr_discipline *,
> struct sr_meta_opt_hdr *);
> -void sr_crypto_write(struct cryptop *);
> +void sr_crypto_write(struct sr_crypto_wu *);
> int sr_crypto_rw(struct sr_workunit *);
> int sr_crypto_dev_rw(struct sr_workunit *, struct sr_crypto_wu *);
> void sr_crypto_done_internal(struct sr_workunit *,
> struct sr_crypto *);
> void sr_crypto_done(struct sr_workunit *);
> -void sr_crypto_read(struct cryptop *);
> +void sr_crypto_read(struct sr_crypto_wu *);
> void sr_crypto_calculate_check_hmac_sha1(u_int8_t *, int,
> u_int8_t *, int, u_char *);
> void sr_crypto_hotplug(struct sr_discipline *, struct disk *, int);
> @@ -322,7 +322,6 @@ sr_crypto_prepare(struct sr_workunit *wu, struct
> sr_crypto *mdd_crypto,
> keyndx = blkno >> SR_CRYPTO_KEY_BLKSHIFT;
> crwu->cr_crp->crp_sid = mdd_crypto->scr_sid[keyndx];
>
> - crwu->cr_crp->crp_opaque = crwu;
> crwu->cr_crp->crp_ilen = xs->datalen;
> crwu->cr_crp->crp_alloctype = M_DEVBUF;
> crwu->cr_crp->crp_flags = CRYPTO_F_IOV;
> @@ -1168,8 +1167,10 @@ sr_crypto_rw(struct sr_workunit *wu)
> if (wu->swu_xs->flags & SCSI_DATA_OUT) {
> mdd_crypto = &wu->swu_dis->mds.mdd_crypto;
> crwu = sr_crypto_prepare(wu, mdd_crypto, 1);
> - crwu->cr_crp->crp_callback = sr_crypto_write;
> - crypto_dispatch(crwu->cr_crp);
> + KERNEL_LOCK();
> + crypto_invoke(crwu->cr_crp);
> + KERNEL_UNLOCK();
> + sr_crypto_write(crwu);
> rv = crwu->cr_crp->crp_etype;
> } else
> rv = sr_crypto_dev_rw(wu, NULL);
> @@ -1178,16 +1179,15 @@ sr_crypto_rw(struct sr_workunit *wu)
> }
>
> void
> -sr_crypto_write(struct cryptop *crp)
> +sr_crypto_write(struct sr_crypto_wu *crwu)
> {
> - struct sr_crypto_wu *crwu = crp->crp_opaque;
> struct sr_workunit *wu = &crwu->cr_wu;
> int s;
>
> DNPRINTF(SR_D_INTR, "%s: sr_crypto_write: wu %p xs: %p\n",
> DEVNAME(wu->swu_dis->sd_sc), wu, wu->swu_xs);
>
> - if (crp->crp_etype) {
> + if (crwu->cr_crp->crp_etype) {
> /* fail io */
> wu->swu_xs->error = XS_DRIVER_STUFFUP;
> s = splbio();
> @@ -1246,10 +1246,12 @@ sr_crypto_done_internal(struct sr_workunit *wu,
> struct sr_crypto *mdd_crypto)
> /* If this was a successful read, initiate decryption of the data. */
> if (ISSET(xs->flags, SCSI_DATA_IN) && xs->error == XS_NOERROR) {
> crwu = sr_crypto_prepare(wu, mdd_crypto, 0);
> - crwu->cr_crp->crp_callback = sr_crypto_read;
> - DNPRINTF(SR_D_INTR, "%s: sr_crypto_done: crypto_dispatch %p\n",
> + DNPRINTF(SR_D_INTR, "%s: sr_crypto_done: crypto_invoke %p\n",
> DEVNAME(wu->swu_dis->sd_sc), crwu->cr_crp);
> - crypto_dispatch(crwu->cr_crp);
> + KERNEL_LOCK();
> + crypto_invoke(crwu->cr_crp);
> + KERNEL_UNLOCK();
> + sr_crypto_read(crwu);
> return;
> }
>
> @@ -1266,16 +1268,15 @@ sr_crypto_done(struct sr_workunit *wu)
> }
>
> void
> -sr_crypto_read(struct cryptop *crp)
> +sr_crypto_read(struct sr_crypto_wu *crwu)
> {
> - struct sr_crypto_wu *crwu = crp->crp_opaque;
> struct sr_workunit *wu = &crwu->cr_wu;
> int s;
>
> DNPRINTF(SR_D_INTR, "%s: sr_crypto_read: wu %p xs: %p\n",
> DEVNAME(wu->swu_dis->sd_sc), wu, wu->swu_xs);
>
> - if (crp->crp_etype)
> + if (crwu->cr_crp->crp_etype)
> wu->swu_xs->error = XS_DRIVER_STUFFUP;
>
> s = splbio();
> diff --git a/sys/dev/softraid_raid1c.c b/sys/dev/softraid_raid1c.c
> index d23323bc5e4..77b2f1142b2 100644
> --- a/sys/dev/softraid_raid1c.c
> +++ b/sys/dev/softraid_raid1c.c
> @@ -58,7 +58,7 @@ void sr_raid1c_free_resources(struct sr_discipline
> *sd);
> int sr_raid1c_ioctl(struct sr_discipline *sd, struct bioc_discipline *bd);
> int sr_raid1c_meta_opt_handler(struct sr_discipline *,
> struct sr_meta_opt_hdr *);
> -void sr_raid1c_write(struct cryptop *);
> +void sr_raid1c_write(struct sr_crypto_wu *);
> int sr_raid1c_rw(struct sr_workunit *);
> int sr_raid1c_dev_rw(struct sr_workunit *, struct sr_crypto_wu *);
> void sr_raid1c_done(struct sr_workunit *wu);
> @@ -312,16 +312,15 @@ bad:
> }
>
> void
> -sr_raid1c_write(struct cryptop *crp)
> +sr_raid1c_write(struct sr_crypto_wu *crwu)
> {
> - struct sr_crypto_wu *crwu = crp->crp_opaque;
> struct sr_workunit *wu = &crwu->cr_wu;
> int s;
>
> DNPRINTF(SR_D_INTR, "%s: sr_raid1c_write: wu %p xs: %p\n",
> DEVNAME(wu->swu_dis->sd_sc), wu, wu->swu_xs);
>
> - if (crp->crp_etype) {
> + if (crwu->cr_crp->crp_etype) {
> /* fail io */
> wu->swu_xs->error = XS_DRIVER_STUFFUP;
> s = splbio();
> @@ -358,8 +357,10 @@ sr_raid1c_rw(struct sr_workunit *wu)
> !ISSET(wu->swu_flags, SR_WUF_REBUILD)) {
> mdd_raid1c = &wu->swu_dis->mds.mdd_raid1c;
> crwu = sr_crypto_prepare(wu, &mdd_raid1c->sr1c_crypto, 1);
> - crwu->cr_crp->crp_callback = sr_raid1c_write;
> - crypto_dispatch(crwu->cr_crp);
> + KERNEL_LOCK();
> + crypto_invoke(crwu->cr_crp);
> + KERNEL_UNLOCK();
> + sr_raid1c_write(crwu);
> rv = crwu->cr_crp->crp_etype;
> } else
> rv = sr_raid1c_dev_rw(wu, NULL);
> diff --git a/sys/netinet/ip_ah.c b/sys/netinet/ip_ah.c
> index a10c2a7b7ba..7ce83a9730e 100644
> --- a/sys/netinet/ip_ah.c
> +++ b/sys/netinet/ip_ah.c
> @@ -531,7 +531,7 @@ ah_input(struct mbuf *m, struct tdb *tdb, int skip, int
> protoff)
> struct tdb_crypto *tc = NULL;
> u_int32_t btsx, esn;
> u_int8_t hl;
> - int error, rplen;
> + int error, rplen, clen;
> u_int64_t ibytes;
> #ifdef ENCDEBUG
> char buf[INET6_ADDRSTRLEN];
> @@ -686,9 +686,7 @@ ah_input(struct mbuf *m, struct tdb *tdb, int skip, int
> protoff)
> crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
> crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
> crp->crp_buf = (caddr_t)m;
> - crp->crp_callback = ipsec_input_cb;
> crp->crp_sid = tdb->tdb_cryptoid;
> - crp->crp_opaque = (caddr_t)tc;
>
> /* These are passed as-is to the callback. */
> tc->tc_skip = skip;
> @@ -699,7 +697,36 @@ ah_input(struct mbuf *m, struct tdb *tdb, int skip, int
> protoff)
> memcpy(&tc->tc_dst, &tdb->tdb_dst, sizeof(union sockaddr_union));
> tc->tc_rpl = tdb->tdb_rpl;
>
> - crypto_dispatch(crp);
> + KERNEL_LOCK();
> + crypto_invoke(crp);
> + while (crp->crp_etype == EAGAIN) {
> + /* Reset the session ID */
> + if (tdb->tdb_cryptoid != 0)
> + tdb->tdb_cryptoid = crp->crp_sid;
> + crypto_invoke(crp);
> + }
> + KERNEL_UNLOCK();
> + if (crp->crp_etype) {
> + DPRINTF("crypto error %d", crp->crp_etype);
> + ipsecstat_inc(ipsec_noxform);
> + ipsecstat_inc(ipsec_idrops);
> + tdb->tdb_idrops++;
> + error = crp->crp_etype;
> + goto drop;
> + }
> +
> + /* Length of data after processing */
> + clen = crp->crp_olen;
> +
> + /* Release the crypto descriptors */
> + crypto_freereq(crp);
> +
> + error = ah_input_cb(tdb, tc, m, clen);
> + if (error) {
> + ipsecstat_inc(ipsec_idrops);
> + tdb->tdb_idrops++;
> + }
> +
> return 0;
>
> drop:
> @@ -893,7 +920,7 @@ ah_output(struct mbuf *m, struct tdb *tdb, int skip, int
> protoff)
> struct cryptop *crp = NULL;
> u_int64_t replay64;
> u_int16_t iplen;
> - int error, rplen, roff;
> + int error, rplen, roff, ilen, olen;
> u_int8_t prot;
> struct ah *ah;
> #if NBPFILTER > 0
> @@ -1133,9 +1160,7 @@ ah_output(struct mbuf *m, struct tdb *tdb, int skip,
> int protoff)
> crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
> crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
> crp->crp_buf = (caddr_t)m;
> - crp->crp_callback = ipsec_output_cb;
> crp->crp_sid = tdb->tdb_cryptoid;
> - crp->crp_opaque = (caddr_t)tc;
>
> /* These are passed as-is to the callback. */
> tc->tc_skip = skip;
> @@ -1145,7 +1170,36 @@ ah_output(struct mbuf *m, struct tdb *tdb, int skip,
> int protoff)
> tc->tc_rdomain = tdb->tdb_rdomain;
> memcpy(&tc->tc_dst, &tdb->tdb_dst, sizeof(union sockaddr_union));
>
> - crypto_dispatch(crp);
> + KERNEL_LOCK();
> + crypto_invoke(crp);
> + while (crp->crp_etype == EAGAIN) {
> + /* Reset the session ID */
> + if (tdb->tdb_cryptoid != 0)
> + tdb->tdb_cryptoid = crp->crp_sid;
> + crypto_invoke(crp);
> + }
> + KERNEL_UNLOCK();
> + if (crp->crp_etype) {
> + DPRINTF("crypto error %d", crp->crp_etype);
> + ipsecstat_inc(ipsec_noxform);
> + ipsecstat_inc(ipsec_odrops);
> + tdb->tdb_odrops++;
> + error = crp->crp_etype;
> + goto drop;
> + }
> +
> + ilen = crp->crp_ilen;
> + olen = crp->crp_olen;
> +
> + /* Release the crypto descriptors */
> + crypto_freereq(crp);
> +
> + error = ah_output_cb(tdb, tc, m, ilen, olen);
> + if (error) {
> + ipsecstat_inc(ipsec_odrops);
> + tdb->tdb_odrops++;
> + }
> +
> return 0;
>
> drop:
> diff --git a/sys/netinet/ip_esp.c b/sys/netinet/ip_esp.c
> index 1b96a2b942d..1e4cfaafdcf 100644
> --- a/sys/netinet/ip_esp.c
> +++ b/sys/netinet/ip_esp.c
> @@ -347,7 +347,7 @@ esp_input(struct mbuf *m, struct tdb *tdb, int skip, int
> protoff)
> struct cryptodesc *crde = NULL, *crda = NULL;
> struct cryptop *crp = NULL;
> struct tdb_crypto *tc = NULL;
> - int plen, alen, hlen, error;
> + int plen, alen, hlen, error, clen;
> u_int32_t btsx, esn;
> #ifdef ENCDEBUG
> char buf[INET6_ADDRSTRLEN];
> @@ -498,9 +498,7 @@ esp_input(struct mbuf *m, struct tdb *tdb, int skip, int
> protoff)
> crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
> crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
> crp->crp_buf = (caddr_t)m;
> - crp->crp_callback = ipsec_input_cb;
> crp->crp_sid = tdb->tdb_cryptoid;
> - crp->crp_opaque = (caddr_t)tc;
>
> /* These are passed as-is to the callback */
> tc->tc_skip = skip;
> @@ -526,7 +524,35 @@ esp_input(struct mbuf *m, struct tdb *tdb, int skip, int
> protoff)
> crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
> }
>
> - crypto_dispatch(crp);
> + KERNEL_LOCK();
> + crypto_invoke(crp);
> + while (crp->crp_etype == EAGAIN) {
> + /* Reset the session ID */
> + if (tdb->tdb_cryptoid != 0)
> + tdb->tdb_cryptoid = crp->crp_sid;
> + crypto_invoke(crp);
> + }
> + KERNEL_UNLOCK();
> + if (crp->crp_etype) {
> + DPRINTF("crypto error %d", crp->crp_etype);
> + ipsecstat_inc(ipsec_noxform);
> + ipsecstat_inc(ipsec_idrops);
> + tdb->tdb_idrops++;
> + error = crp->crp_etype;
> + goto drop;
> + }
> +
> + clen = crp->crp_olen;
> +
> + /* Release the crypto descriptors */
> + crypto_freereq(crp);
> +
> + error = esp_input_cb(tdb, tc, m, clen);
> + if (error) {
> + ipsecstat_inc(ipsec_idrops);
> + tdb->tdb_idrops++;
> + }
> +
> return 0;
>
> drop:
> @@ -742,7 +768,7 @@ esp_output(struct mbuf *m, struct tdb *tdb, int skip, int
> protoff)
> {
> const struct enc_xform *espx = tdb->tdb_encalgxform;
> const struct auth_hash *esph = tdb->tdb_authalgxform;
> - int ilen, hlen, rlen, padding, blks, alen, roff, error;
> + int ilen, olen, hlen, rlen, padding, blks, alen, roff, error;
> u_int64_t replay64;
> u_int32_t replay;
> struct mbuf *mi, *mo = (struct mbuf *) NULL;
> @@ -980,8 +1006,6 @@ esp_output(struct mbuf *m, struct tdb *tdb, int skip,
> int protoff)
> crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
> crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
> crp->crp_buf = (caddr_t)m;
> - crp->crp_callback = ipsec_output_cb;
> - crp->crp_opaque = (caddr_t)tc;
> crp->crp_sid = tdb->tdb_cryptoid;
>
> if (esph) {
> @@ -1010,7 +1034,36 @@ esp_output(struct mbuf *m, struct tdb *tdb, int skip,
> int protoff)
> crda->crd_len = m->m_pkthdr.len - (skip + alen);
> }
>
> - crypto_dispatch(crp);
> + KERNEL_LOCK();
> + crypto_invoke(crp);
> + while (crp->crp_etype == EAGAIN) {
> + /* Reset the session ID */
> + if (tdb->tdb_cryptoid != 0)
> + tdb->tdb_cryptoid = crp->crp_sid;
> + crypto_invoke(crp);
> + }
> + KERNEL_UNLOCK();
> + if (crp->crp_etype) {
> + DPRINTF("crypto error %d", crp->crp_etype);
> + ipsecstat_inc(ipsec_noxform);
> + ipsecstat_inc(ipsec_odrops);
> + tdb->tdb_odrops++;
> + error = crp->crp_etype;
> + goto drop;
> + }
> +
> + ilen = crp->crp_ilen;
> + olen = crp->crp_olen;
> +
> + /* Release the crypto descriptors */
> + crypto_freereq(crp);
> +
> + error = esp_output_cb(tdb, tc, m, ilen, olen);
> + if (error) {
> + ipsecstat_inc(ipsec_odrops);
> + tdb->tdb_odrops++;
> + }
> +
> return 0;
>
> drop:
> diff --git a/sys/netinet/ip_ipcomp.c b/sys/netinet/ip_ipcomp.c
> index 902955c9d6e..4837429fee6 100644
> --- a/sys/netinet/ip_ipcomp.c
> +++ b/sys/netinet/ip_ipcomp.c
> @@ -135,7 +135,7 @@ ipcomp_input(struct mbuf *m, struct tdb *tdb, int skip,
> int protoff)
> {
> const struct comp_algo *ipcompx = tdb->tdb_compalgxform;
> struct tdb_crypto *tc;
> - int hlen;
> + int hlen, error, clen;
>
> struct cryptodesc *crdc = NULL;
> struct cryptop *crp;
> @@ -172,9 +172,7 @@ ipcomp_input(struct mbuf *m, struct tdb *tdb, int skip,
> int protoff)
> crp->crp_ilen = m->m_pkthdr.len - (skip + hlen);
> crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
> crp->crp_buf = (caddr_t)m;
> - crp->crp_callback = ipsec_input_cb;
> crp->crp_sid = tdb->tdb_cryptoid;
> - crp->crp_opaque = (caddr_t)tc;
>
> /* These are passed as-is to the callback */
> tc->tc_skip = skip;
> @@ -184,7 +182,37 @@ ipcomp_input(struct mbuf *m, struct tdb *tdb, int skip,
> int protoff)
> tc->tc_rdomain = tdb->tdb_rdomain;
> tc->tc_dst = tdb->tdb_dst;
>
> - crypto_dispatch(crp);
> + KERNEL_LOCK();
> + crypto_invoke(crp);
> + while (crp->crp_etype == EAGAIN) {
> + /* Reset the session ID */
> + if (tdb->tdb_cryptoid != 0)
> + tdb->tdb_cryptoid = crp->crp_sid;
> + crypto_invoke(crp);
> + }
> + KERNEL_UNLOCK();
> + if (crp->crp_etype) {
> + DPRINTF("crypto error %d", crp->crp_etype);
> + ipsecstat_inc(ipsec_noxform);
> + ipsecstat_inc(ipsec_idrops);
> + tdb->tdb_idrops++;
> + free(tc, M_XDATA, 0);
> + m_freem(m);
> + crypto_freereq(crp);
> + return crp->crp_etype;
> + }
> +
> + clen = crp->crp_olen;
> +
> + /* Release the crypto descriptors */
> + crypto_freereq(crp);
> +
> + error = ipcomp_input_cb(tdb, tc, m, clen);
> + if (error) {
> + ipsecstat_inc(ipsec_idrops);
> + tdb->tdb_idrops++;
> + }
> +
> return 0;
> }
>
> @@ -319,7 +347,7 @@ int
> ipcomp_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
> {
> const struct comp_algo *ipcompx = tdb->tdb_compalgxform;
> - int error, hlen;
> + int error, hlen, ilen, olen;
> struct cryptodesc *crdc = NULL;
> struct cryptop *crp = NULL;
> struct tdb_crypto *tc;
> @@ -474,11 +502,40 @@ ipcomp_output(struct mbuf *m, struct tdb *tdb, int
> skip, int protoff)
> crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
> crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
> crp->crp_buf = (caddr_t)m;
> - crp->crp_callback = ipsec_output_cb;
> - crp->crp_opaque = (caddr_t)tc;
> crp->crp_sid = tdb->tdb_cryptoid;
>
> - crypto_dispatch(crp);
> + KERNEL_LOCK();
> + crypto_invoke(crp);
> + while (crp->crp_etype == EAGAIN) {
> + /* Reset the session ID */
> + if (tdb->tdb_cryptoid != 0)
> + tdb->tdb_cryptoid = crp->crp_sid;
> + crypto_invoke(crp);
> + }
> + KERNEL_UNLOCK();
> + if (crp->crp_etype) {
> + DPRINTF("crypto error %d", crp->crp_etype);
> + ipsecstat_inc(ipsec_noxform);
> + ipsecstat_inc(ipsec_odrops);
> + free(tc, M_XDATA, 0);
> + tdb->tdb_odrops++;
> + m_freem(m);
> + crypto_freereq(crp);
> + return crp->crp_etype;
> + }
> +
> + ilen = crp->crp_ilen;
> + olen = crp->crp_olen;
> +
> + /* Release the crypto descriptors */
> + crypto_freereq(crp);
> +
> + error = ipcomp_output_cb(tdb, tc, m, crp->crp_ilen, crp->crp_olen);
> + if (error) {
> + ipsecstat_inc(ipsec_odrops);
> + tdb->tdb_odrops++;
> + }
> +
> return 0;
>
> drop:
> diff --git a/sys/netinet/ip_ipsp.h b/sys/netinet/ip_ipsp.h
> index 9bb39465b57..63aea100e5d 100644
> --- a/sys/netinet/ip_ipsp.h
> +++ b/sys/netinet/ip_ipsp.h
> @@ -648,8 +648,6 @@ void ipsp_init(void);
> void ipsec_init(void);
> int ipsec_sysctl(int *, u_int, void *, size_t *, void *, size_t);
> int ipsec_common_input(struct mbuf *, int, int, int, int, int);
> -void ipsec_input_cb(struct cryptop *);
> -void ipsec_output_cb(struct cryptop *);
> int ipsec_common_input_cb(struct mbuf *, struct tdb *, int, int);
> int ipsec_delete_policy(struct ipsec_policy *);
> ssize_t ipsec_hdrsz(struct tdb *);
> diff --git a/sys/netinet/ipsec_input.c b/sys/netinet/ipsec_input.c
> index 2a9a378bde7..6fda85f24b7 100644
> --- a/sys/netinet/ipsec_input.c
> +++ b/sys/netinet/ipsec_input.c
> @@ -366,78 +366,6 @@ ipsec_common_input(struct mbuf *m, int skip, int
> protoff, int af, int sproto,
> return error;
> }
>
> -void
> -ipsec_input_cb(struct cryptop *crp)
> -{
> - struct tdb_crypto *tc = (struct tdb_crypto *) crp->crp_opaque;
> - struct mbuf *m = (struct mbuf *) crp->crp_buf;
> - struct tdb *tdb = NULL;
> - int clen, error;
> -
> - NET_ASSERT_LOCKED();
> -
> - if (m == NULL) {
> - DPRINTF("bogus returned buffer from crypto");
> - ipsecstat_inc(ipsec_crypto);
> - goto drop;
> - }
> -
> - tdb = gettdb(tc->tc_rdomain, tc->tc_spi, &tc->tc_dst, tc->tc_proto);
> - if (tdb == NULL) {
> - DPRINTF("TDB is expired while in crypto");
> - ipsecstat_inc(ipsec_notdb);
> - goto drop;
> - }
> -
> - /* Check for crypto errors */
> - if (crp->crp_etype) {
> - if (crp->crp_etype == EAGAIN) {
> - /* Reset the session ID */
> - if (tdb->tdb_cryptoid != 0)
> - tdb->tdb_cryptoid = crp->crp_sid;
> - crypto_dispatch(crp);
> - return;
> - }
> - DPRINTF("crypto error %d", crp->crp_etype);
> - ipsecstat_inc(ipsec_noxform);
> - goto drop;
> - }
> -
> - /* Length of data after processing */
> - clen = crp->crp_olen;
> -
> - /* Release the crypto descriptors */
> - crypto_freereq(crp);
> -
> - switch (tdb->tdb_sproto) {
> - case IPPROTO_ESP:
> - error = esp_input_cb(tdb, tc, m, clen);
> - break;
> - case IPPROTO_AH:
> - error = ah_input_cb(tdb, tc, m, clen);
> - break;
> - case IPPROTO_IPCOMP:
> - error = ipcomp_input_cb(tdb, tc, m, clen);
> - break;
> - default:
> - panic("%s: unknown/unsupported security protocol %d",
> - __func__, tdb->tdb_sproto);
> - }
> - if (error) {
> - ipsecstat_inc(ipsec_idrops);
> - tdb->tdb_idrops++;
> - }
> - return;
> -
> - drop:
> - m_freem(m);
> - free(tc, M_XDATA, 0);
> - crypto_freereq(crp);
> - ipsecstat_inc(ipsec_idrops);
> - if (tdb != NULL)
> - tdb->tdb_idrops++;
> -}
> -
> /*
> * IPsec input callback, called by the transform callback. Takes care of
> * filtering and other sanity checks on the processed packet.
> diff --git a/sys/netinet/ipsec_output.c b/sys/netinet/ipsec_output.c
> index ed11171b543..9962e521dad 100644
> --- a/sys/netinet/ipsec_output.c
> +++ b/sys/netinet/ipsec_output.c
> @@ -377,81 +377,6 @@ ipsp_process_packet(struct mbuf *m, struct tdb *tdb, int
> af, int tunalready)
> return error;
> }
>
> -/*
> - * IPsec output callback, called directly by the crypto driver.
> - */
> -void
> -ipsec_output_cb(struct cryptop *crp)
> -{
> - struct tdb_crypto *tc = (struct tdb_crypto *) crp->crp_opaque;
> - struct mbuf *m = (struct mbuf *) crp->crp_buf;
> - struct tdb *tdb = NULL;
> - int error, ilen, olen;
> -
> - NET_ASSERT_LOCKED();
> -
> - if (m == NULL) {
> - DPRINTF("bogus returned buffer from crypto");
> - ipsecstat_inc(ipsec_crypto);
> - goto drop;
> - }
> -
> - tdb = gettdb(tc->tc_rdomain, tc->tc_spi, &tc->tc_dst, tc->tc_proto);
> - if (tdb == NULL) {
> - DPRINTF("TDB is expired while in crypto");
> - ipsecstat_inc(ipsec_notdb);
> - goto drop;
> - }
> -
> - /* Check for crypto errors. */
> - if (crp->crp_etype) {
> - if (crp->crp_etype == EAGAIN) {
> - /* Reset the session ID */
> - if (tdb->tdb_cryptoid != 0)
> - tdb->tdb_cryptoid = crp->crp_sid;
> - crypto_dispatch(crp);
> - return;
> - }
> - DPRINTF("crypto error %d", crp->crp_etype);
> - ipsecstat_inc(ipsec_noxform);
> - goto drop;
> - }
> -
> - olen = crp->crp_olen;
> - ilen = crp->crp_ilen;
> -
> - /* Release crypto descriptors. */
> - crypto_freereq(crp);
> -
> - switch (tdb->tdb_sproto) {
> - case IPPROTO_ESP:
> - error = esp_output_cb(tdb, tc, m, ilen, olen);
> - break;
> - case IPPROTO_AH:
> - error = ah_output_cb(tdb, tc, m, ilen, olen);
> - break;
> - case IPPROTO_IPCOMP:
> - error = ipcomp_output_cb(tdb, tc, m, ilen, olen);
> - break;
> - default:
> - panic("%s: unhandled security protocol %d",
> - __func__, tdb->tdb_sproto);
> - }
> - if (error) {
> - ipsecstat_inc(ipsec_odrops);
> - tdb->tdb_odrops++;
> - }
> - return;
> -
> - drop:
> - m_freem(m);
> - free(tc, M_XDATA, 0);
> - crypto_freereq(crp);
> - ipsecstat_inc(ipsec_odrops);
> - if (tdb != NULL)
> - tdb->tdb_odrops++;
> -}
> -
> /*
> * Called by the IPsec output transform callbacks, to transmit the packet
> * or do further processing, as necessary.
