Module Name: src
Committed By: yamaguchi
Date: Mon Jan 14 14:55:37 UTC 2019
Modified Files:
src/sys/dev/pci: virtio.c virtio_pci.c virtiovar.h
Log Message:
Add multiqueue support, virtio(4)
To generate a diff of this commit:
cvs rdiff -u -r1.36 -r1.37 src/sys/dev/pci/virtio.c
cvs rdiff -u -r1.5 -r1.6 src/sys/dev/pci/virtio_pci.c
cvs rdiff -u -r1.12 -r1.13 src/sys/dev/pci/virtiovar.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/dev/pci/virtio.c
diff -u src/sys/dev/pci/virtio.c:1.36 src/sys/dev/pci/virtio.c:1.37
--- src/sys/dev/pci/virtio.c:1.36 Sun Sep 30 15:44:22 2018
+++ src/sys/dev/pci/virtio.c Mon Jan 14 14:55:37 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: virtio.c,v 1.36 2018/09/30 15:44:22 jmcneill Exp $ */
+/* $NetBSD: virtio.c,v 1.37 2019/01/14 14:55:37 yamaguchi Exp $ */
/*
* Copyright (c) 2010 Minoura Makoto.
@@ -26,7 +26,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.36 2018/09/30 15:44:22 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.37 2019/01/14 14:55:37 yamaguchi Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -235,6 +235,54 @@ vq_sync_indirect(struct virtio_softc *sc
ops);
}
+static void
+virtio_vq_soft_intr(void *arg)
+{
+ struct virtqueue *vq = arg;
+
+ KASSERT(vq->vq_intrhand != NULL);
+
+ (vq->vq_intrhand)(vq);
+}
+
+static int
+virtio_vq_softint_establish(struct virtio_softc *sc)
+{
+ struct virtqueue *vq;
+ int qid;
+ u_int flags;
+
+ flags = SOFTINT_NET;
+ if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
+ flags |= SOFTINT_MPSAFE;
+
+ for (qid = 0; qid < sc->sc_nvqs; qid++) {
+ vq = &sc->sc_vqs[qid];
+ vq->vq_soft_ih =
+ softint_establish(flags, virtio_vq_soft_intr, vq);
+ if (vq->vq_soft_ih == NULL)
+ return -1;
+ }
+
+ return 0;
+}
+
+static void
+virtio_vq_softint_disestablish(struct virtio_softc *sc)
+{
+ struct virtqueue *vq;
+ int qid;
+
+ for (qid = 0; qid < sc->sc_nvqs; qid++) {
+ vq = &sc->sc_vqs[qid];
+ if (vq->vq_soft_ih == NULL)
+ continue;
+
+ softint_disestablish(vq->vq_soft_ih);
+ vq->vq_soft_ih = NULL;
+ }
+}
+
/*
* Can be used as sc_intrhand.
*/
@@ -242,6 +290,26 @@ vq_sync_indirect(struct virtio_softc *sc
* Scan vq, bus_dmamap_sync for the vqs (not for the payload),
* and calls (*vq_done)() if some entries are consumed.
*/
+static int
+virtio_vq_intr_common(struct virtqueue *vq)
+{
+ struct virtio_softc *sc = vq->vq_owner;
+ int r = 0;
+
+ if (vq->vq_queued) {
+ vq->vq_queued = 0;
+ vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE);
+ }
+ vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD);
+ membar_consumer();
+ if (vq->vq_used_idx != vq->vq_used->idx) {
+ if (vq->vq_done)
+ r |= (vq->vq_done)(vq);
+ }
+
+ return r;
+}
+
int
virtio_vq_intr(struct virtio_softc *sc)
{
@@ -250,21 +318,19 @@ virtio_vq_intr(struct virtio_softc *sc)
for (i = 0; i < sc->sc_nvqs; i++) {
vq = &sc->sc_vqs[i];
- if (vq->vq_queued) {
- vq->vq_queued = 0;
- vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE);
- }
- vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD);
- membar_consumer();
- if (vq->vq_used_idx != vq->vq_used->idx) {
- if (vq->vq_done)
- r |= (vq->vq_done)(vq);
- }
+ r |= virtio_vq_intr_common(vq);
}
return r;
}
+static int
+virtio_vq_mq_intr(struct virtqueue *vq)
+{
+
+ return virtio_vq_intr_common(vq);
+}
+
/*
* Start/stop vq interrupt. No guarantee.
*/
@@ -409,6 +475,7 @@ virtio_alloc_vq(struct virtio_softc *sc,
/* remember addresses and offsets for later use */
vq->vq_owner = sc;
+ vq->vq_intrhand = virtio_vq_mq_intr;
vq->vq_num = vq_size;
vq->vq_index = index;
vq->vq_desc = vq->vq_vaddr;
@@ -844,6 +911,16 @@ virtio_child_attach_start(struct virtio_
aprint_naive("\n");
}
+void
+virtio_child_attach_set_vqs(struct virtio_softc *sc,
+ struct virtqueue *vqs, int nvq_pairs)
+{
+ if (nvq_pairs > 1)
+ sc->sc_child_mq = true;
+
+ sc->sc_vqs = vqs;
+}
+
int
virtio_child_attach_finish(struct virtio_softc *sc)
{
@@ -868,12 +945,26 @@ virtio_child_attach_finish(struct virtio
"failed to establish soft interrupt\n");
goto fail;
}
+
+ if (sc->sc_child_mq) {
+ r = virtio_vq_softint_establish(sc);
+ aprint_error_dev(sc->sc_dev,
+ "failed to establish softint interrupt\n");
+ goto fail;
+ }
}
virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
return 0;
fail:
+ if (sc->sc_soft_ih) {
+ softint_disestablish(sc->sc_soft_ih);
+ sc->sc_soft_ih = NULL;
+ }
+
+ virtio_vq_softint_disestablish(sc);
+
virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
return 1;
}
Index: src/sys/dev/pci/virtio_pci.c
diff -u src/sys/dev/pci/virtio_pci.c:1.5 src/sys/dev/pci/virtio_pci.c:1.6
--- src/sys/dev/pci/virtio_pci.c:1.5 Wed Jun 6 16:11:36 2018
+++ src/sys/dev/pci/virtio_pci.c Mon Jan 14 14:55:37 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: virtio_pci.c,v 1.5 2018/06/06 16:11:36 jakllsch Exp $ */
+/* $NetBSD: virtio_pci.c,v 1.6 2019/01/14 14:55:37 yamaguchi Exp $ */
/*
* Copyright (c) 2010 Minoura Makoto.
@@ -26,12 +26,13 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.5 2018/06/06 16:11:36 jakllsch Exp $");
+__KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.6 2019/01/14 14:55:37 yamaguchi Exp $");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kmem.h>
#include <sys/module.h>
+#include <sys/interrupt.h>
#include <sys/device.h>
@@ -79,6 +80,7 @@ static void virtio_pci_free_interrupts(s
static int virtio_pci_intr(void *arg);
static int virtio_pci_msix_queue_intr(void *);
+static int virtio_pci_msix_vq_intr(void *);
static int virtio_pci_msix_config_intr(void *);
static int virtio_pci_setup_msix_vectors(struct virtio_softc *);
static int virtio_pci_setup_msix_interrupts(struct virtio_softc *,
@@ -428,7 +430,7 @@ virtio_pci_setup_queue(struct virtio_sof
if (psc->sc_ihs_num > 1) {
int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
- if (false) /* (for per-vq vectors) */
+ if (sc->sc_child_mq)
vec += idx;
bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
VIRTIO_CONFIG_MSI_QUEUE_VECTOR, vec);
@@ -488,6 +490,9 @@ virtio_pci_setup_msix_vectors(struct vir
offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR;
vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
+ if (sc->sc_child_mq)
+ vector += qid;
+
bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, vector);
ret = bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, offset);
aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n",
@@ -507,29 +512,57 @@ virtio_pci_setup_msix_interrupts(struct
device_t self = sc->sc_dev;
pci_chipset_tag_t pc = pa->pa_pc;
char intrbuf[PCI_INTRSTR_LEN];
+ char intr_xname[INTRDEVNAMEBUF];
char const *intrstr;
- int idx;
+ int idx, qid, n;
idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
+ snprintf(intr_xname, sizeof(intr_xname), "%s config",
+ device_xname(sc->sc_dev));
+
psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
- sc->sc_ipl, virtio_pci_msix_config_intr, sc, device_xname(sc->sc_dev));
+ sc->sc_ipl, virtio_pci_msix_config_intr, sc, intr_xname);
if (psc->sc_ihs[idx] == NULL) {
aprint_error_dev(self, "couldn't establish MSI-X for config\n");
goto error;
}
idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
- if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
- pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
+ if (sc->sc_child_mq) {
+ for (qid = 0; qid < sc->sc_nvqs; qid++) {
+ n = idx + qid;
+
+ snprintf(intr_xname, sizeof(intr_xname), "%s vq#%d",
+ device_xname(sc->sc_dev), qid);
+
+ if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) {
+ pci_intr_setattr(pc, &psc->sc_ihp[n],
+ PCI_INTR_MPSAFE, true);
+ }
+
+ psc->sc_ihs[n] = pci_intr_establish_xname(pc, psc->sc_ihp[n],
+ sc->sc_ipl, virtio_pci_msix_vq_intr, &sc->sc_vqs[qid],
+ intr_xname);
+ if (psc->sc_ihs[n] == NULL) {
+ aprint_error_dev(self, "couldn't establish MSI-X for a vq\n");
+ goto error;
+ }
+ }
+ } else {
+ if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
+ pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
- psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
- sc->sc_ipl, virtio_pci_msix_queue_intr, sc, device_xname(sc->sc_dev));
- if (psc->sc_ihs[idx] == NULL) {
- aprint_error_dev(self, "couldn't establish MSI-X for queues\n");
- goto error;
+ snprintf(intr_xname, sizeof(intr_xname), "%s queues",
+ device_xname(sc->sc_dev));
+ psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
+ sc->sc_ipl, virtio_pci_msix_queue_intr, sc, intr_xname);
+ if (psc->sc_ihs[idx] == NULL) {
+ aprint_error_dev(self, "couldn't establish MSI-X for queues\n");
+ goto error;
+ }
}
if (virtio_pci_setup_msix_vectors(sc) != 0) {
@@ -541,8 +574,38 @@ virtio_pci_setup_msix_interrupts(struct
intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
aprint_normal_dev(self, "config interrupting at %s\n", intrstr);
idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
- intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
- aprint_normal_dev(self, "queues interrupting at %s\n", intrstr);
+ if (sc->sc_child_mq) {
+ kcpuset_t *affinity;
+ int affinity_to, r;
+
+ kcpuset_create(&affinity, false);
+
+ for (qid = 0; qid < sc->sc_nvqs; qid++) {
+ n = idx + qid;
+ affinity_to = (qid / 2) % ncpu;
+
+ intrstr = pci_intr_string(pc, psc->sc_ihp[n],
+ intrbuf, sizeof(intrbuf));
+
+ kcpuset_zero(affinity);
+ kcpuset_set(affinity, affinity_to);
+ r = interrupt_distribute(psc->sc_ihs[n], affinity, NULL);
+ if (r == 0) {
+ aprint_normal_dev(self,
+ "for vq #%d interrupting at %s affinity to %u\n",
+ qid, intrstr, affinity_to);
+ } else {
+ aprint_normal_dev(self,
+ "for vq #%d interrupting at %s\n",
+ qid, intrstr);
+ }
+ }
+
+ kcpuset_destroy(affinity);
+ } else {
+ intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
+ aprint_normal_dev(self, "queues interrupting at %s\n", intrstr);
+ }
return 0;
@@ -551,8 +614,18 @@ error:
if (psc->sc_ihs[idx] != NULL)
pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
- if (psc->sc_ihs[idx] != NULL)
- pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
+ if (sc->sc_child_mq) {
+ for (qid = 0; qid < sc->sc_nvqs; qid++) {
+ n = idx + qid;
+ if (psc->sc_ihs[n] == NULL)
+ continue;
+ pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[n]);
+ }
+
+ } else {
+ if (psc->sc_ihs[idx] != NULL)
+ pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
+ }
return -1;
}
@@ -604,8 +677,14 @@ virtio_pci_setup_interrupts(struct virti
counts[PCI_INTR_TYPE_INTX] = 1;
} else {
/* Try MSI-X first and INTx second */
+ if (sc->sc_child_mq &&
+ sc->sc_nvqs > (nmsix - VIRTIO_MSIX_QUEUE_VECTOR_INDEX)) {
+ nmsix = 2;
+ sc->sc_child_mq = false;
+ }
+
max_type = PCI_INTR_TYPE_MSIX;
- counts[PCI_INTR_TYPE_MSIX] = 2;
+ counts[PCI_INTR_TYPE_MSIX] = nmsix;
counts[PCI_INTR_TYPE_MSI] = 0;
counts[PCI_INTR_TYPE_INTX] = 1;
}
@@ -618,13 +697,13 @@ retry:
}
if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) {
- psc->sc_ihs = kmem_alloc(sizeof(*psc->sc_ihs) * 2,
+ psc->sc_ihs = kmem_alloc(sizeof(*psc->sc_ihs) * nmsix,
KM_SLEEP);
error = virtio_pci_setup_msix_interrupts(sc, &psc->sc_pa);
if (error != 0) {
- kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 2);
- pci_intr_release(pc, psc->sc_ihp, 2);
+ kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * nmsix);
+ pci_intr_release(pc, psc->sc_ihp, nmsix);
/* Retry INTx */
max_type = PCI_INTR_TYPE_INTX;
@@ -632,7 +711,7 @@ retry:
goto retry;
}
- psc->sc_ihs_num = 2;
+ psc->sc_ihs_num = nmsix;
psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI;
} else if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) {
psc->sc_ihs = kmem_alloc(sizeof(*psc->sc_ihs) * 1,
@@ -719,6 +798,22 @@ virtio_pci_msix_queue_intr(void *arg)
}
static int
+virtio_pci_msix_vq_intr(void *arg)
+{
+ struct virtqueue *vq = arg;
+ int r = 0;
+
+ if (vq->vq_intrhand != NULL) {
+ if (vq->vq_soft_ih)
+ softint_schedule(vq->vq_soft_ih);
+ else
+ r |= vq->vq_intrhand(vq);
+ }
+
+ return r;
+}
+
+static int
virtio_pci_msix_config_intr(void *arg)
{
struct virtio_softc *sc = arg;
Index: src/sys/dev/pci/virtiovar.h
diff -u src/sys/dev/pci/virtiovar.h:1.12 src/sys/dev/pci/virtiovar.h:1.13
--- src/sys/dev/pci/virtiovar.h:1.12 Fri Jun 15 17:13:43 2018
+++ src/sys/dev/pci/virtiovar.h Mon Jan 14 14:55:37 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: virtiovar.h,v 1.12 2018/06/15 17:13:43 jakllsch Exp $ */
+/* $NetBSD: virtiovar.h,v 1.13 2019/01/14 14:55:37 yamaguchi Exp $ */
/*
* Copyright (c) 2010 Minoura Makoto.
@@ -116,6 +116,9 @@ struct virtqueue {
/* interrupt handler */
int (*vq_done)(struct virtqueue*);
+ void *vq_done_ctx;
+ void *vq_soft_ih;
+ int (*vq_intrhand)(struct virtqueue*);
};
struct virtio_attach_args {
@@ -161,6 +164,7 @@ struct virtio_softc {
int sc_childdevid;
device_t sc_child; /* set by child */
+ bool sc_child_mq;
virtio_callback sc_config_change; /* set by child */
virtio_callback sc_intrhand; /* set by child */
};
@@ -196,6 +200,8 @@ void virtio_child_attach_start(struct vi
struct virtqueue *,
virtio_callback, virtio_callback, int,
int, const char *);
+void virtio_child_attach_set_vqs(struct virtio_softc *,
+ struct virtqueue *, int);
int virtio_child_attach_finish(struct virtio_softc *);
void virtio_child_attach_failed(struct virtio_softc *);
void virtio_child_detach(struct virtio_softc *);
