Module Name: src
Committed By: mlelstv
Date: Mon Mar 9 08:33:15 UTC 2020
Modified Files:
src/sys/dev: cgd.c cgdvar.h
Log Message:
Defer crypto operations to a workqueue and make it utilize all CPUs.
Make device mpsafe.
Some code cleanup.
To generate a diff of this commit:
cvs rdiff -u -r1.121 -r1.122 src/sys/dev/cgd.c
cvs rdiff -u -r1.18 -r1.19 src/sys/dev/cgdvar.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/dev/cgd.c
diff -u src/sys/dev/cgd.c:1.121 src/sys/dev/cgd.c:1.122
--- src/sys/dev/cgd.c:1.121 Mon Mar 2 16:01:56 2020
+++ src/sys/dev/cgd.c Mon Mar 9 08:33:15 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cgd.c,v 1.121 2020/03/02 16:01:56 riastradh Exp $ */
+/* $NetBSD: cgd.c,v 1.122 2020/03/09 08:33:15 mlelstv Exp $ */
/*-
* Copyright (c) 2002 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cgd.c,v 1.121 2020/03/02 16:01:56 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cgd.c,v 1.122 2020/03/09 08:33:15 mlelstv Exp $");
#include <sys/types.h>
#include <sys/param.h>
@@ -39,6 +39,7 @@ __KERNEL_RCSID(0, "$NetBSD: cgd.c,v 1.12
#include <sys/errno.h>
#include <sys/buf.h>
#include <sys/bufq.h>
+#include <sys/kmem.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/pool.h>
@@ -51,6 +52,8 @@ __KERNEL_RCSID(0, "$NetBSD: cgd.c,v 1.12
#include <sys/vnode.h>
#include <sys/conf.h>
#include <sys/syslog.h>
+#include <sys/workqueue.h>
+#include <sys/cpu.h>
#include <dev/dkvar.h>
#include <dev/cgdvar.h>
@@ -90,7 +93,7 @@ const struct bdevsw cgd_bdevsw = {
.d_dump = cgddump,
.d_psize = cgdsize,
.d_discard = nodiscard,
- .d_flag = D_DISK
+ .d_flag = D_DISK | D_MPSAFE
};
const struct cdevsw cgd_cdevsw = {
@@ -105,7 +108,7 @@ const struct cdevsw cgd_cdevsw = {
.d_mmap = nommap,
.d_kqfilter = nokqfilter,
.d_discard = nodiscard,
- .d_flag = D_DISK
+ .d_flag = D_DISK | D_MPSAFE
};
/*
@@ -207,12 +210,20 @@ static int cgd_match(device_t, cfdata_t,
static void cgd_attach(device_t, device_t, void *);
static int cgd_detach(device_t, int);
static struct cgd_softc *cgd_spawn(int);
+static struct cgd_worker *cgd_create_one_worker(void);
+static void cgd_destroy_one_worker(struct cgd_worker *);
+static struct cgd_worker *cgd_create_worker(void);
+static void cgd_destroy_worker(struct cgd_worker *);
static int cgd_destroy(device_t);
/* Internal Functions */
static int cgd_diskstart(device_t, struct buf *);
+static void cgd_diskstart2(struct cgd_softc *, struct cgd_xfer *);
static void cgdiodone(struct buf *);
+static void cgd_iodone2(struct cgd_softc *, struct cgd_xfer *);
+static void cgd_enqueue(struct cgd_softc *, struct cgd_xfer *);
+static void cgd_process(struct work *, void *);
static int cgd_dumpblocks(device_t, void *, daddr_t, int);
static int cgd_ioctl_set(struct cgd_softc *, void *, struct lwp *);
@@ -264,25 +275,49 @@ static void hexprint(const char *, void
/* Global variables */
+static kmutex_t cgd_spawning_mtx;
+static kcondvar_t cgd_spawning_cv;
+static bool cgd_spawning;
+static struct cgd_worker *cgd_worker;
+static u_int cgd_refcnt; /* number of users of cgd_worker */
+
/* Utility Functions */
#define CGDUNIT(x) DISKUNIT(x)
-#define GETCGD_SOFTC(_cs, x) if (!((_cs) = getcgd_softc(x))) return ENXIO
/* The code */
-static struct cgd_softc *
-getcgd_softc(dev_t dev)
+static int
+cgd_lock(bool intr)
{
- int unit = CGDUNIT(dev);
- struct cgd_softc *sc;
+ int error = 0;
- DPRINTF_FOLLOW(("getcgd_softc(0x%"PRIx64"): unit = %d\n", dev, unit));
+ mutex_enter(&cgd_spawning_mtx);
+ while (cgd_spawning) {
+ if (intr)
+ error = cv_wait_sig(&cgd_spawning_cv, &cgd_spawning_mtx);
+ else
+ cv_wait(&cgd_spawning_cv, &cgd_spawning_mtx);
+ }
+ if (error == 0)
+ cgd_spawning = true;
+ mutex_exit(&cgd_spawning_mtx);
+ return error;
+}
- sc = device_lookup_private(&cgd_cd, unit);
- if (sc == NULL)
- sc = cgd_spawn(unit);
- return sc;
+static void
+cgd_unlock(void)
+{
+ mutex_enter(&cgd_spawning_mtx);
+ cgd_spawning = false;
+ cv_broadcast(&cgd_spawning_cv);
+ mutex_exit(&cgd_spawning_mtx);
+}
+
+static struct cgd_softc *
+getcgd_softc(dev_t dev)
+{
+ return device_lookup_private(&cgd_cd, CGDUNIT(dev));
}
static int
@@ -298,6 +333,7 @@ cgd_attach(device_t parent, device_t sel
struct cgd_softc *sc = device_private(self);
mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_BIO);
+ cv_init(&sc->sc_cv, "cgdcv");
dk_init(&sc->sc_dksc, self, DKTYPE_CGD);
disk_init(&sc->sc_dksc.sc_dkdev, sc->sc_dksc.sc_xname, &cgddkdriver);
@@ -323,6 +359,7 @@ cgd_detach(device_t self, int flags)
return ret;
disk_destroy(&dksc->sc_dkdev);
+ cv_destroy(&sc->sc_cv);
mutex_destroy(&sc->sc_lock);
return 0;
@@ -331,89 +368,224 @@ cgd_detach(device_t self, int flags)
void
cgdattach(int num)
{
+#ifndef _MODULE
int error;
+ mutex_init(&cgd_spawning_mtx, MUTEX_DEFAULT, IPL_NONE);
+ cv_init(&cgd_spawning_cv, "cgspwn");
+
error = config_cfattach_attach(cgd_cd.cd_name, &cgd_ca);
if (error != 0)
aprint_error("%s: unable to register cfattach\n",
cgd_cd.cd_name);
+#endif
}
static struct cgd_softc *
cgd_spawn(int unit)
{
cfdata_t cf;
+ struct cgd_worker *cw;
+ struct cgd_softc *sc;
- cf = malloc(sizeof(*cf), M_DEVBUF, M_WAITOK);
+ cf = kmem_alloc(sizeof(*cf), KM_SLEEP);
cf->cf_name = cgd_cd.cd_name;
cf->cf_atname = cgd_cd.cd_name;
cf->cf_unit = unit;
cf->cf_fstate = FSTATE_STAR;
- return device_private(config_attach_pseudo(cf));
+ cw = cgd_create_one_worker();
+ if (cw == NULL) {
+ kmem_free(cf, sizeof(*cf));
+ return NULL;
+ }
+
+ sc = device_private(config_attach_pseudo(cf));
+ if (sc == NULL) {
+ cgd_destroy_one_worker(cw);
+ return NULL;
+ }
+
+ sc->sc_worker = cw;
+
+ return sc;
}
static int
cgd_destroy(device_t dev)
{
- int error;
+ struct cgd_softc *sc = device_private(dev);
+ struct cgd_worker *cw = sc->sc_worker;
cfdata_t cf;
+ int error;
cf = device_cfdata(dev);
error = config_detach(dev, DETACH_QUIET);
if (error)
return error;
- free(cf, M_DEVBUF);
+
+ cgd_destroy_one_worker(cw);
+
+ kmem_free(cf, sizeof(*cf));
return 0;
}
+static void
+cgd_busy(struct cgd_softc *sc)
+{
+
+ mutex_enter(&sc->sc_lock);
+ while (sc->sc_busy)
+ cv_wait(&sc->sc_cv, &sc->sc_lock);
+ sc->sc_busy = true;
+ mutex_exit(&sc->sc_lock);
+}
+
+static void
+cgd_unbusy(struct cgd_softc *sc)
+{
+
+ mutex_enter(&sc->sc_lock);
+ sc->sc_busy = false;
+ cv_broadcast(&sc->sc_cv);
+ mutex_exit(&sc->sc_lock);
+}
+
+static struct cgd_worker *
+cgd_create_one_worker(void)
+{
+ KASSERT(cgd_spawning);
+
+ if (cgd_refcnt++ == 0) {
+ KASSERT(cgd_worker == NULL);
+ cgd_worker = cgd_create_worker();
+ }
+
+ KASSERT(cgd_worker != NULL);
+ return cgd_worker;
+}
+
+static void
+cgd_destroy_one_worker(struct cgd_worker *cw)
+{
+ KASSERT(cgd_spawning);
+ KASSERT(cw == cgd_worker);
+
+ if (--cgd_refcnt == 0) {
+ cgd_destroy_worker(cgd_worker);
+ cgd_worker = NULL;
+ }
+}
+
+static struct cgd_worker *
+cgd_create_worker(void)
+{
+ struct cgd_worker *cw;
+ struct workqueue *wq;
+ struct pool *cp;
+ int error;
+
+ cw = kmem_alloc(sizeof(struct cgd_worker), KM_SLEEP);
+ cp = kmem_alloc(sizeof(struct pool), KM_SLEEP);
+
+ error = workqueue_create(&wq, "cgd", cgd_process, NULL,
+ PRI_BIO, IPL_BIO, WQ_MPSAFE | WQ_PERCPU);
+ if (error) {
+ kmem_free(cp, sizeof(struct pool));
+ kmem_free(cw, sizeof(struct cgd_worker));
+ return NULL;
+ }
+
+ cw->cw_cpool = cp;
+ cw->cw_wq = wq;
+ pool_init(cw->cw_cpool, sizeof(struct cgd_xfer), 0,
+ 0, 0, "cgdcpl", NULL, IPL_BIO);
+
+ mutex_init(&cw->cw_lock, MUTEX_DEFAULT, IPL_BIO);
+
+ return cw;
+}
+
+static void
+cgd_destroy_worker(struct cgd_worker *cw)
+{
+ mutex_destroy(&cw->cw_lock);
+
+ if (cw->cw_cpool) {
+ pool_destroy(cw->cw_cpool);
+ kmem_free(cw->cw_cpool, sizeof(struct pool));
+ }
+ if (cw->cw_wq)
+ workqueue_destroy(cw->cw_wq);
+
+ kmem_free(cw, sizeof(struct cgd_worker));
+}
+
static int
cgdopen(dev_t dev, int flags, int fmt, struct lwp *l)
{
- struct cgd_softc *cs;
+ struct cgd_softc *sc;
+ int error;
DPRINTF_FOLLOW(("cgdopen(0x%"PRIx64", %d)\n", dev, flags));
- GETCGD_SOFTC(cs, dev);
- return dk_open(&cs->sc_dksc, dev, flags, fmt, l);
+
+ error = cgd_lock(true);
+ if (error)
+ return error;
+ sc = getcgd_softc(dev);
+ if (sc == NULL)
+ sc = cgd_spawn(CGDUNIT(dev));
+ cgd_unlock();
+ if (sc == NULL)
+ return ENXIO;
+
+ return dk_open(&sc->sc_dksc, dev, flags, fmt, l);
}
static int
cgdclose(dev_t dev, int flags, int fmt, struct lwp *l)
{
- int error;
- struct cgd_softc *cs;
+ struct cgd_softc *sc;
struct dk_softc *dksc;
+ int error;
DPRINTF_FOLLOW(("cgdclose(0x%"PRIx64", %d)\n", dev, flags));
- GETCGD_SOFTC(cs, dev);
- dksc = &cs->sc_dksc;
- if ((error = dk_close(dksc, dev, flags, fmt, l)) != 0)
+
+ error = cgd_lock(false);
+ if (error)
return error;
+ sc = getcgd_softc(dev);
+ if (sc == NULL) {
+ error = ENXIO;
+ goto done;
+ }
+
+ dksc = &sc->sc_dksc;
+ if ((error = dk_close(dksc, dev, flags, fmt, l)) != 0)
+ goto done;
if (!DK_ATTACHED(dksc)) {
- if ((error = cgd_destroy(cs->sc_dksc.sc_dev)) != 0) {
- aprint_error_dev(dksc->sc_dev,
+ if ((error = cgd_destroy(sc->sc_dksc.sc_dev)) != 0) {
+ device_printf(dksc->sc_dev,
"unable to detach instance\n");
- return error;
+ goto done;
}
}
- return 0;
+
+done:
+ cgd_unlock();
+
+ return error;
}
static void
cgdstrategy(struct buf *bp)
{
- struct cgd_softc *cs;
+ struct cgd_softc *sc = getcgd_softc(bp->b_dev);
DPRINTF_FOLLOW(("cgdstrategy(%p): b_bcount = %ld\n", bp,
(long)bp->b_bcount));
- cs = getcgd_softc(bp->b_dev);
- if (!cs) {
- bp->b_error = ENXIO;
- goto bail;
- }
-
/*
* Reject unaligned writes.
*/
@@ -422,7 +594,7 @@ cgdstrategy(struct buf *bp)
goto bail;
}
- dk_strategy(&cs->sc_dksc, bp);
+ dk_strategy(&sc->sc_dksc, bp);
return;
bail:
@@ -434,66 +606,62 @@ bail:
static int
cgdsize(dev_t dev)
{
- struct cgd_softc *cs = getcgd_softc(dev);
+ struct cgd_softc *sc = getcgd_softc(dev);
DPRINTF_FOLLOW(("cgdsize(0x%"PRIx64")\n", dev));
- if (!cs)
+ if (!sc)
return -1;
- return dk_size(&cs->sc_dksc, dev);
+ return dk_size(&sc->sc_dksc, dev);
}
/*
* cgd_{get,put}data are functions that deal with getting a buffer
- * for the new encrypted data. We have a buffer per device so that
- * we can ensure that we can always have a transaction in flight.
- * We use this buffer first so that we have one less piece of
- * malloc'ed data at any given point.
+ * for the new encrypted data.
+ * We can no longer have a buffer per device, we need a buffer per
+ * work queue...
*/
static void *
-cgd_getdata(struct dk_softc *dksc, unsigned long size)
+cgd_getdata(struct cgd_softc *sc, unsigned long size)
{
- struct cgd_softc *cs = (struct cgd_softc *)dksc;
- void * data = NULL;
+ void *data = NULL;
- mutex_enter(&cs->sc_lock);
- if (cs->sc_data_used == 0) {
- cs->sc_data_used = 1;
- data = cs->sc_data;
+ mutex_enter(&sc->sc_lock);
+ if (!sc->sc_data_used) {
+ sc->sc_data_used = true;
+ data = sc->sc_data;
}
- mutex_exit(&cs->sc_lock);
+ mutex_exit(&sc->sc_lock);
if (data)
return data;
- return malloc(size, M_DEVBUF, M_NOWAIT);
+ return malloc(size, M_DEVBUF, M_WAITOK);
}
static void
-cgd_putdata(struct dk_softc *dksc, void *data)
+cgd_putdata(struct cgd_softc *sc, void *data)
{
- struct cgd_softc *cs = (struct cgd_softc *)dksc;
- if (data == cs->sc_data) {
- mutex_enter(&cs->sc_lock);
- cs->sc_data_used = 0;
- mutex_exit(&cs->sc_lock);
- } else {
+ if (data == sc->sc_data) {
+ mutex_enter(&sc->sc_lock);
+ sc->sc_data_used = false;
+ mutex_exit(&sc->sc_lock);
+ } else
free(data, M_DEVBUF);
- }
}
static int
cgd_diskstart(device_t dev, struct buf *bp)
{
- struct cgd_softc *cs = device_private(dev);
- struct dk_softc *dksc = &cs->sc_dksc;
+ struct cgd_softc *sc = device_private(dev);
+ struct cgd_worker *cw = sc->sc_worker;
+ struct dk_softc *dksc = &sc->sc_dksc;
struct disk_geom *dg = &dksc->sc_dkdev.dk_geom;
+ struct cgd_xfer *cx;
struct buf *nbp;
- void * addr;
void * newaddr;
daddr_t bn;
- struct vnode *vp;
DPRINTF_FOLLOW(("cgd_diskstart(%p, %p)\n", dksc, bp));
@@ -503,34 +671,66 @@ cgd_diskstart(device_t dev, struct buf *
* We attempt to allocate all of our resources up front, so that
* we can fail quickly if they are unavailable.
*/
- nbp = getiobuf(cs->sc_tvn, false);
+ nbp = getiobuf(sc->sc_tvn, false);
if (nbp == NULL)
return EAGAIN;
+ cx = pool_get(cw->cw_cpool, PR_NOWAIT);
+ if (cx == NULL) {
+ putiobuf(nbp);
+ return EAGAIN;
+ }
+
+ cx->cx_sc = sc;
+ cx->cx_obp = bp;
+ cx->cx_nbp = nbp;
+ cx->cx_srcv = cx->cx_dstv = bp->b_data;
+ cx->cx_blkno = bn;
+ cx->cx_secsize = dg->dg_secsize;
+
/*
* If we are writing, then we need to encrypt the outgoing
* block into a new block of memory.
*/
- newaddr = addr = bp->b_data;
if ((bp->b_flags & B_READ) == 0) {
- newaddr = cgd_getdata(dksc, bp->b_bcount);
+ newaddr = cgd_getdata(sc, bp->b_bcount);
if (!newaddr) {
+ pool_put(cw->cw_cpool, cx);
putiobuf(nbp);
return EAGAIN;
}
- cgd_cipher(cs, newaddr, addr, bp->b_bcount, bn,
- dg->dg_secsize, CGD_CIPHER_ENCRYPT);
+
+ cx->cx_dstv = newaddr;
+ cx->cx_len = bp->b_bcount;
+ cx->cx_dir = CGD_CIPHER_ENCRYPT;
+
+ cgd_enqueue(sc, cx);
+ return 0;
}
- nbp->b_data = newaddr;
+ cgd_diskstart2(sc, cx);
+ return 0;
+}
+
+static void
+cgd_diskstart2(struct cgd_softc *sc, struct cgd_xfer *cx)
+{
+ struct vnode *vp;
+ struct buf *bp;
+ struct buf *nbp;
+
+ bp = cx->cx_obp;
+ nbp = cx->cx_nbp;
+
+ nbp->b_data = cx->cx_dstv;
nbp->b_flags = bp->b_flags;
nbp->b_oflags = bp->b_oflags;
nbp->b_cflags = bp->b_cflags;
nbp->b_iodone = cgdiodone;
nbp->b_proc = bp->b_proc;
- nbp->b_blkno = btodb(bn * dg->dg_secsize);
+ nbp->b_blkno = btodb(cx->cx_blkno * cx->cx_secsize);
nbp->b_bcount = bp->b_bcount;
- nbp->b_private = bp;
+ nbp->b_private = cx;
BIO_COPYPRIO(nbp, bp);
@@ -540,21 +740,20 @@ cgd_diskstart(device_t dev, struct buf *
vp->v_numoutput++;
mutex_exit(vp->v_interlock);
}
- VOP_STRATEGY(cs->sc_tvn, nbp);
-
- return 0;
+ VOP_STRATEGY(sc->sc_tvn, nbp);
}
static void
cgdiodone(struct buf *nbp)
{
- struct buf *obp = nbp->b_private;
- struct cgd_softc *cs = getcgd_softc(obp->b_dev);
- struct dk_softc *dksc = &cs->sc_dksc;
+ struct cgd_xfer *cx = nbp->b_private;
+ struct buf *obp = cx->cx_obp;
+ struct cgd_softc *sc = getcgd_softc(obp->b_dev);
+ struct dk_softc *dksc = &sc->sc_dksc;
struct disk_geom *dg = &dksc->sc_dkdev.dk_geom;
daddr_t bn;
- KDASSERT(cs);
+ KDASSERT(sc);
DPRINTF_FOLLOW(("cgdiodone(%p)\n", nbp));
DPRINTF(CGDB_IO, ("cgdiodone: bp %p bcount %d resid %d\n",
@@ -576,13 +775,36 @@ cgdiodone(struct buf *nbp)
if (nbp->b_flags & B_READ) {
bn = dbtob(nbp->b_blkno) / dg->dg_secsize;
- cgd_cipher(cs, obp->b_data, obp->b_data, obp->b_bcount,
- bn, dg->dg_secsize, CGD_CIPHER_DECRYPT);
+
+ cx->cx_obp = obp;
+ cx->cx_nbp = nbp;
+ cx->cx_dstv = obp->b_data;
+ cx->cx_srcv = obp->b_data;
+ cx->cx_len = obp->b_bcount;
+ cx->cx_blkno = bn;
+ cx->cx_secsize = dg->dg_secsize;
+ cx->cx_dir = CGD_CIPHER_DECRYPT;
+
+ cgd_enqueue(sc, cx);
+ return;
}
+ cgd_iodone2(sc, cx);
+}
+
+static void
+cgd_iodone2(struct cgd_softc *sc, struct cgd_xfer *cx)
+{
+ struct cgd_worker *cw = sc->sc_worker;
+ struct buf *obp = cx->cx_obp;
+ struct buf *nbp = cx->cx_nbp;
+ struct dk_softc *dksc = &sc->sc_dksc;
+
+ pool_put(cw->cw_cpool, cx);
+
/* If we allocated memory, free it now... */
if (nbp->b_data != obp->b_data)
- cgd_putdata(dksc, nbp->b_data);
+ cgd_putdata(sc, nbp->b_data);
putiobuf(nbp);
@@ -591,10 +813,8 @@ cgdiodone(struct buf *nbp)
if (obp->b_error != 0)
obp->b_resid = obp->b_bcount;
- KERNEL_LOCK(1, NULL); /* XXXSMP */
dk_done(dksc, obp);
dk_start(dksc, NULL);
- KERNEL_UNLOCK_ONE(NULL); /* XXXSMP */
}
static int
@@ -625,7 +845,7 @@ cgd_dumpblocks(device_t dev, void *va, d
nbytes = nblk*blksize;
/* Try to acquire a buffer to store the ciphertext. */
- buf = cgd_getdata(dksc, nbytes);
+ buf = cgd_getdata(sc, nbytes);
if (buf == NULL)
/* Out of memory: give up. */
return ENOMEM;
@@ -637,7 +857,7 @@ cgd_dumpblocks(device_t dev, void *va, d
error = bdev_dump(sc->sc_tdev, blkno, buf, nbytes);
/* Release the buffer. */
- cgd_putdata(dksc, buf);
+ cgd_putdata(sc, buf);
/* Return any error from the underlying disk device. */
return error;
@@ -647,13 +867,15 @@ cgd_dumpblocks(device_t dev, void *va, d
static int
cgdread(dev_t dev, struct uio *uio, int flags)
{
- struct cgd_softc *cs;
+ struct cgd_softc *sc;
struct dk_softc *dksc;
DPRINTF_FOLLOW(("cgdread(0x%llx, %p, %d)\n",
(unsigned long long)dev, uio, flags));
- GETCGD_SOFTC(cs, dev);
- dksc = &cs->sc_dksc;
+ sc = getcgd_softc(dev);
+ if (sc == NULL)
+ return ENXIO;
+ dksc = &sc->sc_dksc;
if (!DK_ATTACHED(dksc))
return ENXIO;
return physio(cgdstrategy, NULL, dev, B_READ, minphys, uio);
@@ -663,12 +885,14 @@ cgdread(dev_t dev, struct uio *uio, int
static int
cgdwrite(dev_t dev, struct uio *uio, int flags)
{
- struct cgd_softc *cs;
+ struct cgd_softc *sc;
struct dk_softc *dksc;
DPRINTF_FOLLOW(("cgdwrite(0x%"PRIx64", %p, %d)\n", dev, uio, flags));
- GETCGD_SOFTC(cs, dev);
- dksc = &cs->sc_dksc;
+ sc = getcgd_softc(dev);
+ if (sc == NULL)
+ return ENXIO;
+ dksc = &sc->sc_dksc;
if (!DK_ATTACHED(dksc))
return ENXIO;
return physio(cgdstrategy, NULL, dev, B_WRITE, minphys, uio);
@@ -677,10 +901,11 @@ cgdwrite(dev_t dev, struct uio *uio, int
static int
cgdioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
{
- struct cgd_softc *cs;
+ struct cgd_softc *sc;
struct dk_softc *dksc;
int part = DISKPART(dev);
int pmask = 1 << part;
+ int error;
DPRINTF_FOLLOW(("cgdioctl(0x%"PRIx64", %ld, %p, %d, %p)\n",
dev, cmd, data, flag, l));
@@ -694,39 +919,60 @@ cgdioctl(dev_t dev, u_long cmd, void *da
return EBADF;
/* FALLTHROUGH */
default:
- GETCGD_SOFTC(cs, dev);
- dksc = &cs->sc_dksc;
+ sc = getcgd_softc(dev);
+ if (sc == NULL)
+ return ENXIO;
+ dksc = &sc->sc_dksc;
break;
}
switch (cmd) {
case CGDIOCSET:
+ cgd_busy(sc);
if (DK_ATTACHED(dksc))
- return EBUSY;
- return cgd_ioctl_set(cs, data, l);
+ error = EBUSY;
+ else
+ error = cgd_ioctl_set(sc, data, l);
+ cgd_unbusy(sc);
+ break;
case CGDIOCCLR:
- if (DK_BUSY(&cs->sc_dksc, pmask))
- return EBUSY;
- return cgd_ioctl_clr(cs, l);
+ cgd_busy(sc);
+ if (DK_BUSY(&sc->sc_dksc, pmask))
+ error = EBUSY;
+ else
+ error = cgd_ioctl_clr(sc, l);
+ cgd_unbusy(sc);
+ break;
case DIOCGCACHE:
case DIOCCACHESYNC:
- if (!DK_ATTACHED(dksc))
- return ENOENT;
+ cgd_busy(sc);
+ if (!DK_ATTACHED(dksc)) {
+ cgd_unbusy(sc);
+ error = ENOENT;
+ break;
+ }
/*
* We pass this call down to the underlying disk.
*/
- return VOP_IOCTL(cs->sc_tvn, cmd, data, flag, l->l_cred);
+ error = VOP_IOCTL(sc->sc_tvn, cmd, data, flag, l->l_cred);
+ cgd_unbusy(sc);
+ break;
case DIOCGSECTORALIGN: {
struct disk_sectoralign *dsa = data;
- int error;
- if (!DK_ATTACHED(dksc))
- return ENOENT;
+ cgd_busy(sc);
+ if (!DK_ATTACHED(dksc)) {
+ cgd_unbusy(sc);
+ error = ENOENT;
+ break;
+ }
/* Get the underlying disk's sector alignment. */
- error = VOP_IOCTL(cs->sc_tvn, cmd, data, flag, l->l_cred);
- if (error)
- return error;
+ error = VOP_IOCTL(sc->sc_tvn, cmd, data, flag, l->l_cred);
+ if (error) {
+ cgd_unbusy(sc);
+ break;
+ }
/* Adjust for the disklabel partition if necessary. */
if (part != RAW_PART) {
@@ -741,30 +987,38 @@ cgdioctl(dev_t dev, u_long cmd, void *da
dsa->dsa_firstaligned = (dsa->dsa_firstaligned
+ dsa->dsa_alignment) - r;
}
- return 0;
+ cgd_unbusy(sc);
+ break;
}
case DIOCGSTRATEGY:
case DIOCSSTRATEGY:
- if (!DK_ATTACHED(dksc))
- return ENOENT;
+ if (!DK_ATTACHED(dksc)) {
+ error = ENOENT;
+ break;
+ }
/*FALLTHROUGH*/
default:
- return dk_ioctl(dksc, dev, cmd, data, flag, l);
+ error = dk_ioctl(dksc, dev, cmd, data, flag, l);
+ break;
case CGDIOCGET:
KASSERT(0);
- return EINVAL;
+ error = EINVAL;
}
+
+ return error;
}
static int
cgddump(dev_t dev, daddr_t blkno, void *va, size_t size)
{
- struct cgd_softc *cs;
+ struct cgd_softc *sc;
DPRINTF_FOLLOW(("cgddump(0x%"PRIx64", %" PRId64 ", %p, %lu)\n",
dev, blkno, va, (unsigned long)size));
- GETCGD_SOFTC(cs, dev);
- return dk_dump(&cs->sc_dksc, dev, blkno, va, size, DK_DUMP_RECURSIVE);
+ sc = getcgd_softc(dev);
+ if (sc == NULL)
+ return ENXIO;
+ return dk_dump(&sc->sc_dksc, dev, blkno, va, size, DK_DUMP_RECURSIVE);
}
/*
@@ -785,7 +1039,7 @@ static const struct {
/* ARGSUSED */
static int
-cgd_ioctl_set(struct cgd_softc *cs, void *data, struct lwp *l)
+cgd_ioctl_set(struct cgd_softc *sc, void *data, struct lwp *l)
{
struct cgd_ioctl *ci = data;
struct vnode *vp;
@@ -795,7 +1049,7 @@ cgd_ioctl_set(struct cgd_softc *cs, void
const char *cp;
struct pathbuf *pb;
char *inbuf;
- struct dk_softc *dksc = &cs->sc_dksc;
+ struct dk_softc *dksc = &sc->sc_dksc;
cp = ci->ci_disk;
@@ -809,17 +1063,17 @@ cgd_ioctl_set(struct cgd_softc *cs, void
return ret;
}
- inbuf = malloc(MAX_KEYSIZE, M_TEMP, M_WAITOK);
+ inbuf = kmem_alloc(MAX_KEYSIZE, KM_SLEEP);
- if ((ret = cgdinit(cs, cp, vp, l)) != 0)
+ if ((ret = cgdinit(sc, cp, vp, l)) != 0)
goto bail;
(void)memset(inbuf, 0, MAX_KEYSIZE);
ret = copyinstr(ci->ci_alg, inbuf, 256, NULL);
if (ret)
goto bail;
- cs->sc_cfuncs = cryptfuncs_find(inbuf);
- if (!cs->sc_cfuncs) {
+ sc->sc_cfuncs = cryptfuncs_find(inbuf);
+ if (!sc->sc_cfuncs) {
ret = EINVAL;
goto bail;
}
@@ -849,15 +1103,15 @@ cgd_ioctl_set(struct cgd_softc *cs, void
if (ret)
goto bail;
- cs->sc_cdata.cf_blocksize = ci->ci_blocksize;
- cs->sc_cdata.cf_mode = encblkno[i].v;
- cs->sc_cdata.cf_keylen = ci->ci_keylen;
- cs->sc_cdata.cf_priv = cs->sc_cfuncs->cf_init(ci->ci_keylen, inbuf,
- &cs->sc_cdata.cf_blocksize);
- if (cs->sc_cdata.cf_blocksize > CGD_MAXBLOCKSIZE) {
+ sc->sc_cdata.cf_blocksize = ci->ci_blocksize;
+ sc->sc_cdata.cf_mode = encblkno[i].v;
+ sc->sc_cdata.cf_keylen = ci->ci_keylen;
+ sc->sc_cdata.cf_priv = sc->sc_cfuncs->cf_init(ci->ci_keylen, inbuf,
+ &sc->sc_cdata.cf_blocksize);
+ if (sc->sc_cdata.cf_blocksize > CGD_MAXBLOCKSIZE) {
log(LOG_WARNING, "cgd: Disallowed cipher with blocksize %zu > %u\n",
- cs->sc_cdata.cf_blocksize, CGD_MAXBLOCKSIZE);
- cs->sc_cdata.cf_priv = NULL;
+ sc->sc_cdata.cf_blocksize, CGD_MAXBLOCKSIZE);
+ sc->sc_cdata.cf_priv = NULL;
}
/*
@@ -865,18 +1119,18 @@ cgd_ioctl_set(struct cgd_softc *cs, void
* it was expressed in bits. For compatibility we maintain encblkno
* and encblkno8.
*/
- cs->sc_cdata.cf_blocksize /= encblkno[i].d;
+ sc->sc_cdata.cf_blocksize /= encblkno[i].d;
(void)explicit_memset(inbuf, 0, MAX_KEYSIZE);
- if (!cs->sc_cdata.cf_priv) {
+ if (!sc->sc_cdata.cf_priv) {
ret = EINVAL; /* XXX is this the right error? */
goto bail;
}
- free(inbuf, M_TEMP);
+ kmem_free(inbuf, MAX_KEYSIZE);
bufq_alloc(&dksc->sc_bufq, "fcfs", 0);
- cs->sc_data = malloc(MAXPHYS, M_DEVBUF, M_WAITOK);
- cs->sc_data_used = 0;
+ sc->sc_data = malloc(MAXPHYS, M_DEVBUF, M_WAITOK);
+ sc->sc_data_used = false;
/* Attach the disk. */
dk_attach(dksc);
@@ -890,16 +1144,16 @@ cgd_ioctl_set(struct cgd_softc *cs, void
return 0;
bail:
- free(inbuf, M_TEMP);
+ kmem_free(inbuf, MAX_KEYSIZE);
(void)vn_close(vp, FREAD|FWRITE, l->l_cred);
return ret;
}
/* ARGSUSED */
static int
-cgd_ioctl_clr(struct cgd_softc *cs, struct lwp *l)
+cgd_ioctl_clr(struct cgd_softc *sc, struct lwp *l)
{
- struct dk_softc *dksc = &cs->sc_dksc;
+ struct dk_softc *dksc = &sc->sc_dksc;
if (!DK_ATTACHED(dksc))
return ENXIO;
@@ -911,11 +1165,11 @@ cgd_ioctl_clr(struct cgd_softc *cs, stru
dk_drain(dksc);
bufq_free(dksc->sc_bufq);
- (void)vn_close(cs->sc_tvn, FREAD|FWRITE, l->l_cred);
- cs->sc_cfuncs->cf_destroy(cs->sc_cdata.cf_priv);
- free(cs->sc_tpath, M_DEVBUF);
- free(cs->sc_data, M_DEVBUF);
- cs->sc_data_used = 0;
+ (void)vn_close(sc->sc_tvn, FREAD|FWRITE, l->l_cred);
+ sc->sc_cfuncs->cf_destroy(sc->sc_cdata.cf_priv);
+ kmem_free(sc->sc_tpath, sc->sc_tpathlen);
+ free(sc->sc_data, M_DEVBUF);
+ sc->sc_data_used = false;
dk_detach(dksc);
disk_detach(&dksc->sc_dkdev);
@@ -925,10 +1179,9 @@ cgd_ioctl_clr(struct cgd_softc *cs, stru
static int
cgd_ioctl_get(dev_t dev, void *data, struct lwp *l)
{
- struct cgd_softc *cs = getcgd_softc(dev);
+ struct cgd_softc *sc;
struct cgd_user *cgu;
- int unit;
- struct dk_softc *dksc = &cs->sc_dksc;
+ int unit, error;
unit = CGDUNIT(dev);
cgu = (struct cgd_user *)data;
@@ -936,14 +1189,21 @@ cgd_ioctl_get(dev_t dev, void *data, str
DPRINTF_FOLLOW(("cgd_ioctl_get(0x%"PRIx64", %d, %p, %p)\n",
dev, unit, data, l));
+ /* XXX, we always return this units data, so if cgu_unit is
+ * not -1, that field doesn't match the rest
+ */
if (cgu->cgu_unit == -1)
cgu->cgu_unit = unit;
if (cgu->cgu_unit < 0)
return EINVAL; /* XXX: should this be ENXIO? */
- cs = device_lookup_private(&cgd_cd, unit);
- if (cs == NULL || !DK_ATTACHED(dksc)) {
+ error = cgd_lock(false);
+ if (error)
+ return error;
+
+ sc = device_lookup_private(&cgd_cd, unit);
+ if (sc == NULL || !DK_ATTACHED(&sc->sc_dksc)) {
cgu->cgu_dev = 0;
cgu->cgu_alg[0] = '\0';
cgu->cgu_blocksize = 0;
@@ -951,18 +1211,22 @@ cgd_ioctl_get(dev_t dev, void *data, str
cgu->cgu_keylen = 0;
}
else {
- cgu->cgu_dev = cs->sc_tdev;
- strlcpy(cgu->cgu_alg, cs->sc_cfuncs->cf_name,
+ mutex_enter(&sc->sc_lock);
+ cgu->cgu_dev = sc->sc_tdev;
+ strncpy(cgu->cgu_alg, sc->sc_cfuncs->cf_name,
sizeof(cgu->cgu_alg));
- cgu->cgu_blocksize = cs->sc_cdata.cf_blocksize;
- cgu->cgu_mode = cs->sc_cdata.cf_mode;
- cgu->cgu_keylen = cs->sc_cdata.cf_keylen;
+ cgu->cgu_blocksize = sc->sc_cdata.cf_blocksize;
+ cgu->cgu_mode = sc->sc_cdata.cf_mode;
+ cgu->cgu_keylen = sc->sc_cdata.cf_keylen;
+ mutex_exit(&sc->sc_lock);
}
+
+ cgd_unlock();
return 0;
}
static int
-cgdinit(struct cgd_softc *cs, const char *cpath, struct vnode *vp,
+cgdinit(struct cgd_softc *sc, const char *cpath, struct vnode *vp,
struct lwp *l)
{
struct disk_geom *dg;
@@ -970,19 +1234,19 @@ cgdinit(struct cgd_softc *cs, const char
char *tmppath;
uint64_t psize;
unsigned secsize;
- struct dk_softc *dksc = &cs->sc_dksc;
+ struct dk_softc *dksc = &sc->sc_dksc;
- cs->sc_tvn = vp;
- cs->sc_tpath = NULL;
+ sc->sc_tvn = vp;
+ sc->sc_tpath = NULL;
- tmppath = malloc(MAXPATHLEN, M_TEMP, M_WAITOK);
- ret = copyinstr(cpath, tmppath, MAXPATHLEN, &cs->sc_tpathlen);
+ tmppath = kmem_alloc(MAXPATHLEN, KM_SLEEP);
+ ret = copyinstr(cpath, tmppath, MAXPATHLEN, &sc->sc_tpathlen);
if (ret)
goto bail;
- cs->sc_tpath = malloc(cs->sc_tpathlen, M_DEVBUF, M_WAITOK);
- memcpy(cs->sc_tpath, tmppath, cs->sc_tpathlen);
+ sc->sc_tpath = kmem_alloc(sc->sc_tpathlen, KM_SLEEP);
+ memcpy(sc->sc_tpath, tmppath, sc->sc_tpathlen);
- cs->sc_tdev = vp->v_rdev;
+ sc->sc_tdev = vp->v_rdev;
if ((ret = getdisksize(vp, &psize, &secsize)) != 0)
goto bail;
@@ -1007,9 +1271,9 @@ cgdinit(struct cgd_softc *cs, const char
dg->dg_ncylinders = dg->dg_secperunit / dg->dg_nsectors;
bail:
- free(tmppath, M_TEMP);
- if (ret && cs->sc_tpath)
- free(cs->sc_tpath, M_DEVBUF);
+ kmem_free(tmppath, MAXPATHLEN);
+ if (ret && sc->sc_tpath)
+ kmem_free(sc->sc_tpath, sc->sc_tpathlen);
return ret;
}
@@ -1050,19 +1314,80 @@ blkno2blkno_buf(char *sbuf, daddr_t blkn
}
}
+static struct cpu_info *
+cgd_cpu(struct cgd_softc *sc)
+{
+ struct cgd_worker *cw = sc->sc_worker;
+ struct cpu_info *ci = NULL;
+ u_int cidx, i;
+
+ if (cw->cw_busy == 0) {
+ cw->cw_last = cpu_index(curcpu());
+ return NULL;
+ }
+
+ for (i=0, cidx = cw->cw_last+1; i<maxcpus; ++i, ++cidx) {
+ if (cidx >= maxcpus)
+ cidx = 0;
+ ci = cpu_lookup(cidx);
+ if (ci) {
+ cw->cw_last = cidx;
+ break;
+ }
+ }
+
+ return ci;
+}
+
+static void
+cgd_enqueue(struct cgd_softc *sc, struct cgd_xfer *cx)
+{
+ struct cgd_worker *cw = sc->sc_worker;
+ struct cpu_info *ci;
+
+ mutex_enter(&cw->cw_lock);
+ ci = cgd_cpu(sc);
+ cw->cw_busy++;
+ mutex_exit(&cw->cw_lock);
+
+ workqueue_enqueue(cw->cw_wq, &cx->cx_work, ci);
+}
+
+static void
+cgd_process(struct work *wk, void *arg)
+{
+ struct cgd_xfer *cx = (struct cgd_xfer *)wk;
+ struct cgd_softc *sc = cx->cx_sc;
+ struct cgd_worker *cw = sc->sc_worker;
+
+ cgd_cipher(sc, cx->cx_dstv, cx->cx_srcv, cx->cx_len,
+ cx->cx_blkno, cx->cx_secsize, cx->cx_dir);
+
+ if (cx->cx_dir == CGD_CIPHER_ENCRYPT) {
+ cgd_diskstart2(sc, cx);
+ } else {
+ cgd_iodone2(sc, cx);
+ }
+
+ mutex_enter(&cw->cw_lock);
+ if (cw->cw_busy > 0)
+ cw->cw_busy--;
+ mutex_exit(&cw->cw_lock);
+}
+
static void
-cgd_cipher(struct cgd_softc *cs, void *dstv, void *srcv,
+cgd_cipher(struct cgd_softc *sc, void *dstv, void *srcv,
size_t len, daddr_t blkno, size_t secsize, int dir)
{
char *dst = dstv;
char *src = srcv;
- cfunc_cipher_prep *ciprep = cs->sc_cfuncs->cf_cipher_prep;
- cfunc_cipher *cipher = cs->sc_cfuncs->cf_cipher;
+ cfunc_cipher_prep *ciprep = sc->sc_cfuncs->cf_cipher_prep;
+ cfunc_cipher *cipher = sc->sc_cfuncs->cf_cipher;
struct uio dstuio;
struct uio srcuio;
struct iovec dstiov[2];
struct iovec srciov[2];
- size_t blocksize = cs->sc_cdata.cf_blocksize;
+ size_t blocksize = sc->sc_cdata.cf_blocksize;
size_t todo;
char blkno_buf[CGD_MAXBLOCKSIZE], *iv;
@@ -1102,10 +1427,10 @@ cgd_cipher(struct cgd_softc *cs, void *d
* can convert blkno_buf in-place.
*/
iv = blkno_buf;
- ciprep(cs->sc_cdata.cf_priv, iv, blkno_buf, blocksize, dir);
+ ciprep(sc->sc_cdata.cf_priv, iv, blkno_buf, blocksize, dir);
IFDEBUG(CGDB_CRYPTO, hexprint("step 2: iv", iv, blocksize));
- cipher(cs->sc_cdata.cf_priv, &dstuio, &srcuio, iv, dir);
+ cipher(sc->sc_cdata.cf_priv, &dstuio, &srcuio, iv, dir);
dst += todo;
src += todo;
@@ -1129,7 +1454,7 @@ hexprint(const char *start, void *buf, i
static void
selftest(void)
{
- struct cgd_softc cs;
+ struct cgd_softc sc;
void *buf;
printf("running cgd selftest ");
@@ -1142,40 +1467,40 @@ selftest(void)
printf("%s-%d ", alg, keylen);
- memset(&cs, 0, sizeof(cs));
+ memset(&sc, 0, sizeof(sc));
- cs.sc_cfuncs = cryptfuncs_find(alg);
- if (cs.sc_cfuncs == NULL)
+ sc.sc_cfuncs = cryptfuncs_find(alg);
+ if (sc.sc_cfuncs == NULL)
panic("%s not implemented", alg);
- cs.sc_cdata.cf_blocksize = 8 * selftests[i].blocksize;
- cs.sc_cdata.cf_mode = CGD_CIPHER_CBC_ENCBLKNO1;
- cs.sc_cdata.cf_keylen = keylen;
-
- cs.sc_cdata.cf_priv = cs.sc_cfuncs->cf_init(keylen,
- key, &cs.sc_cdata.cf_blocksize);
- if (cs.sc_cdata.cf_priv == NULL)
+ sc.sc_cdata.cf_blocksize = 8 * selftests[i].blocksize;
+ sc.sc_cdata.cf_mode = CGD_CIPHER_CBC_ENCBLKNO1;
+ sc.sc_cdata.cf_keylen = keylen;
+
+ sc.sc_cdata.cf_priv = sc.sc_cfuncs->cf_init(keylen,
+ key, &sc.sc_cdata.cf_blocksize);
+ if (sc.sc_cdata.cf_priv == NULL)
panic("cf_priv is NULL");
- if (cs.sc_cdata.cf_blocksize > CGD_MAXBLOCKSIZE)
- panic("bad block size %zu", cs.sc_cdata.cf_blocksize);
+ if (sc.sc_cdata.cf_blocksize > CGD_MAXBLOCKSIZE)
+ panic("bad block size %zu", sc.sc_cdata.cf_blocksize);
- cs.sc_cdata.cf_blocksize /= 8;
+ sc.sc_cdata.cf_blocksize /= 8;
buf = malloc(txtlen, M_DEVBUF, M_WAITOK);
memcpy(buf, selftests[i].ptxt, txtlen);
- cgd_cipher(&cs, buf, buf, txtlen, selftests[i].blkno,
+ cgd_cipher(&sc, buf, buf, txtlen, selftests[i].blkno,
selftests[i].secsize, CGD_CIPHER_ENCRYPT);
if (memcmp(buf, selftests[i].ctxt, txtlen) != 0)
panic("encryption is broken");
- cgd_cipher(&cs, buf, buf, txtlen, selftests[i].blkno,
+ cgd_cipher(&sc, buf, buf, txtlen, selftests[i].blkno,
selftests[i].secsize, CGD_CIPHER_DECRYPT);
if (memcmp(buf, selftests[i].ptxt, txtlen) != 0)
panic("decryption is broken");
free(buf, M_DEVBUF);
- cs.sc_cfuncs->cf_destroy(cs.sc_cdata.cf_priv);
+ sc.sc_cfuncs->cf_destroy(sc.sc_cdata.cf_priv);
}
printf("done\n");
@@ -1198,6 +1523,9 @@ cgd_modcmd(modcmd_t cmd, void *arg)
case MODULE_CMD_INIT:
selftest();
#ifdef _MODULE
+ mutex_init(&cgd_spawning_mtx, MUTEX_DEFAULT, IPL_NONE);
+ cv_init(&cgd_spawning_cv, "cgspwn");
+
error = config_cfdriver_attach(&cgd_cd);
if (error)
break;
@@ -1255,6 +1583,9 @@ cgd_modcmd(modcmd_t cmd, void *arg)
"error %d\n", __func__, cgd_cd.cd_name, error);
break;
}
+
+ cv_destroy(&cgd_spawning_cv);
+ mutex_destroy(&cgd_spawning_mtx);
#endif
break;
Index: src/sys/dev/cgdvar.h
diff -u src/sys/dev/cgdvar.h:1.18 src/sys/dev/cgdvar.h:1.19
--- src/sys/dev/cgdvar.h:1.18 Sun Sep 6 06:00:59 2015
+++ src/sys/dev/cgdvar.h Mon Mar 9 08:33:15 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cgdvar.h,v 1.18 2015/09/06 06:00:59 dholland Exp $ */
+/* $NetBSD: cgdvar.h,v 1.19 2020/03/09 08:33:15 mlelstv Exp $ */
/*-
* Copyright (c) 2002 The NetBSD Foundation, Inc.
@@ -80,17 +80,41 @@ struct cryptdata {
void *cf_priv; /* enc alg private data */
};
+struct cgd_xfer {
+ struct work cx_work;
+ struct cgd_softc *cx_sc;
+ struct buf *cx_obp;
+ struct buf *cx_nbp;
+ void *cx_dstv;
+ void *cx_srcv;
+ size_t cx_len;
+ daddr_t cx_blkno;
+ size_t cx_secsize;
+ int cx_dir;
+};
+
+struct cgd_worker {
+ struct workqueue *cw_wq; /* work queue */
+ struct pool *cw_cpool; /* cgd_xfer contexts */
+ u_int cw_busy; /* number of busy contexts */
+ u_int cw_last; /* index of last CPU used */
+ kmutex_t cw_lock;
+};
+
struct cgd_softc {
struct dk_softc sc_dksc; /* generic disk interface */
struct vnode *sc_tvn; /* target device's vnode */
dev_t sc_tdev; /* target device */
char *sc_tpath; /* target device's path */
- void * sc_data; /* emergency buffer */
- int sc_data_used; /* Really lame, we'll change */
+ void *sc_data; /* emergency buffer */
+ bool sc_data_used; /* Really lame, we'll change */
size_t sc_tpathlen; /* length of prior string */
struct cryptdata sc_cdata; /* crypto data */
const struct cryptfuncs *sc_cfuncs; /* encryption functions */
- kmutex_t sc_lock; /* our lock */
+ kmutex_t sc_lock;
+ kcondvar_t sc_cv;
+ bool sc_busy;
+ struct cgd_worker *sc_worker; /* shared worker data */
};
#endif
