Re: move vmd vioblk handling to another thread
On 12 Nov 01:52, David Gwynne wrote:
> this updates a diff i had from a few years ago to move the vioblk
> handling in vmd into a separate thread.
>
> basically disk io in your virtual machine should not block the vcpu from
> running now.
>
> just throwing this out so people can give it a go and kick it around.
>
> Index: Makefile
> ===
> RCS file: /cvs/src/usr.sbin/vmd/Makefile,v
> retrieving revision 1.28
> diff -u -p -r1.28 Makefile
> --- Makefile 10 Nov 2022 11:46:39 - 1.28
> +++ Makefile 11 Nov 2022 15:51:50 -
> @@ -5,7 +5,7 @@
> PROG=vmd
> SRCS=vmd.c control.c log.c priv.c proc.c config.c vmm.c
> SRCS+= vm.c loadfile_elf.c pci.c virtio.c i8259.c mc146818.c
> -SRCS+= ns8250.c i8253.c dhcp.c packet.c mmio.c
> +SRCS+= ns8250.c i8253.c dhcp.c packet.c mmio.c task.c
> SRCS+= parse.y atomicio.c vioscsi.c vioraw.c vioqcow2.c
> fw_cfg.c
> SRCS+= vm_agentx.c
>
> Index: task.c
> ===
> RCS file: task.c
> diff -N task.c
> --- /dev/null 1 Jan 1970 00:00:00 -
> +++ task.c11 Nov 2022 15:51:50 -
> @@ -0,0 +1,158 @@
> +/* $OpenBSD: task.c,v 1.2 2018/06/19 17:12:34 reyk Exp $ */
> +
> +/*
> + * Copyright (c) 2017 David Gwynne
> + *
> + * Permission to use, copy, modify, and distribute this software for any
> + * purpose with or without fee is hereby granted, provided that the above
> + * copyright notice and this permission notice appear in all copies.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> + */
> +
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +
> +#include "task.h"
> +
> +#define ISSET(_v, _m)((_v) & (_m))
> +#define SET(_v, _m) ((_v) |= (_m))
> +#define CLR(_v, _m) ((_v) &= ~(_m))
> +
> +struct taskq {
> + pthread_t thread;
> + struct task_list list;
> + pthread_mutex_t mtx;
> + pthread_cond_tcv;
> +};
> +
> +#define TASK_ONQUEUE (1 << 0)
> +
> +static void *taskq_run(void *);
> +
> +struct taskq *
> +taskq_create(const char *name)
> +{
> + struct taskq *tq;
> + int error;
> +
> + tq = malloc(sizeof(*tq));
> + if (tq == NULL)
> + return (NULL);
> +
> + TAILQ_INIT(>list);
> +
> + error = pthread_mutex_init(>mtx, NULL);
> + if (error != 0)
> + goto free;
> +
> + error = pthread_cond_init(>cv, NULL);
> + if (error != 0)
> + goto mtx;
> +
> + error = pthread_create(>thread, NULL, taskq_run, tq);
> + if (error != 0)
> + goto cv;
> +
> + pthread_set_name_np(tq->thread, name);
> +
> + return (tq);
> +
> +cv:
> + pthread_cond_destroy(>cv);
> +mtx:
> + pthread_mutex_destroy(>mtx); /* can this really fail? */
> +free:
> + free(tq);
> +
> + errno = error;
> + return (NULL);
> +}
> +
> +static void *
> +taskq_run(void *tqarg)
> +{
> + struct taskq *tq = tqarg;
> + struct task *t;
> +
> + void (*t_func)(void *);
> + void *t_arg;
> +
> + for (;;) {
> + pthread_mutex_lock(>mtx);
> + while ((t = TAILQ_FIRST(>list)) == NULL)
> + pthread_cond_wait(>cv, >mtx);
> +
> + TAILQ_REMOVE(>list, t, t_entry);
> + CLR(t->t_flags, TASK_ONQUEUE);
> +
> + t_func = t->t_func;
> + t_arg = t->t_arg;
> +
> + pthread_mutex_unlock(>mtx);
> +
> + (*t_func)(t_arg);
> + }
> +
> + return (NULL);
> +}
> +
> +void
> +task_set(struct task *t, void (*fn)(void *), void *arg)
> +{
> + t->t_func = fn;
> + t->t_arg = arg;
> + t->t_flags = 0;
> +}
> +
> +int
> +task_add(struct taskq *tq, struct task *t)
> +{
> + int rv = 1;
> +
> + if (ISSET(t->t_flags, TASK_ONQUEUE))
> + return (0);
> +
> + pthread_mutex_lock(>mtx);
> + if (ISSET(t->t_flags, TASK_ONQUEUE))
> + rv = 0;
> + else {
> + SET(t->t_flags, TASK_ONQUEUE);
> + TAILQ_INSERT_TAIL(>list, t, t_entry);
> + pthread_cond_signal(>cv);
> + }
> + pthread_mutex_unlock(>mtx);
> +
> + return (rv);
> +}
> +
> +int
> +task_del(struct taskq *tq, struct task *t)
> +{
> + int rv = 1;
> +
> + if (!ISSET(t->t_flags, TASK_ONQUEUE))
> + return (0);
> +
> +
Re: move vmd vioblk handling to another thread
Hi David,
Updated the machine to latest snap and applied the patch.
The VMs I have on the machine I am testing with aren't booting properly.
OpenBSD/amd64 BOOT 3.53
boot>
NOTE: random seed is being reused.
booting hd0a:/bsd: 15549720+3695624+345456+0+1171456
[1142327+128+1218000+922932]=0x16f0dc0
entry point at 0x81001000
[ using 3284416 bytes of bsd ELF symbol table ]
Copyright (c) 1982, 1986, 1989, 1991, 1993
The Regents of the University of California. All rights
reserved.
Copyright (c) 1995-2022 OpenBSD. All rights reserved.
https://www.OpenBSD.org
OpenBSD 7.1 (GENERIC) #3: Sun May 15 10:25:28 MDT 2022
r...@syspatch-71-amd64.openbsd.org:/usr/src/sys/arch/amd64/compile/GENERIC
real mem = 1056952320 (1007MB)
avail mem = 1007792128 (961MB)
random: good seed from bootblocks
mpath0 at root
scsibus0 at mpath0: 256 targets
mainbus0 at root
bios0 at mainbus0: SMBIOS rev. 2.4 @ 0xf36e0 (10 entries)
bios0: vendor SeaBIOS version "1.14.0p0-OpenBSD-vmm" date 01/01/2011
bios0: OpenBSD VMM
acpi at bios0 not configured
cpu0 at mainbus0: (uniprocessor)
cpu0: Intel(R) Xeon(R) CPU E5-2640 0 @ 2.50GHz, 2500.01 MHz, 06-2d-07
cpu0:
FPU,VME,DE,PSE,TSC,MSR,PAE,CX8,SEP,PGE,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,SSE3,PCLMUL,SSSE3,CX16,SSE4.1,SSE4.2,POPCNT,AES,XSAVE,AVX,HV,NXE,PAGE1GB,LONG,LAHF,ITSC,MD_CLEAR,MELTDOWN
cpu0: 256KB 64b/line 8-way L2 cache
cpu0: smt 0, core 0, package 0
cpu0: using VERW MDS workaround
pvbus0 at mainbus0: OpenBSD
pvclock0 at pvbus0
pci0 at mainbus0 bus 0
pchb0 at pci0 dev 0 function 0 "OpenBSD VMM Host" rev 0x00
virtio0 at pci0 dev 1 function 0 "Qumranet Virtio RNG" rev 0x00
viornd0 at virtio0
virtio0: irq 3
virtio1 at pci0 dev 2 function 0 "Qumranet Virtio Network" rev 0x00
vio0 at virtio1: address fe:e1:bb:6a:00:04
virtio1: irq 5
virtio2 at pci0 dev 3 function 0 "Qumranet Virtio Storage" rev 0x00
vioblk0 at virtio2
scsibus1 at vioblk0: 1 targets
sd0 at scsibus1 targ 0 lun 0:
sd0: 51200MB, 512 bytes/sector, 104857600 sectors
virtio2: irq 6
virtio3 at pci0 dev 4 function 0 "OpenBSD VMM Control" rev 0x00
vmmci0 at virtio3
virtio3: irq 7
isa0 at mainbus0
isadma0 at isa0
com0 at isa0 port 0x3f8/8 irq 4: ns8250, no fifo
com0: console
vscsi0 at root
scsibus2 at vscsi0: 256 targets
softraid0 at root
scsibus3 at softraid0: 256 targets
After this nothing happens.
Mischa
On 2022-11-11 16:52, David Gwynne wrote:
this updates a diff i had from a few years ago to move the vioblk
handling in vmd into a separate thread.
basically disk io in your virtual machine should not block the vcpu
from
running now.
just throwing this out so people can give it a go and kick it around.
Index: Makefile
===
RCS file: /cvs/src/usr.sbin/vmd/Makefile,v
retrieving revision 1.28
diff -u -p -r1.28 Makefile
--- Makefile10 Nov 2022 11:46:39 - 1.28
+++ Makefile11 Nov 2022 15:51:50 -
@@ -5,7 +5,7 @@
PROG= vmd
SRCS= vmd.c control.c log.c priv.c proc.c config.c vmm.c
SRCS+= vm.c loadfile_elf.c pci.c virtio.c i8259.c mc146818.c
-SRCS+= ns8250.c i8253.c dhcp.c packet.c mmio.c
+SRCS+= ns8250.c i8253.c dhcp.c packet.c mmio.c task.c
SRCS+= parse.y atomicio.c vioscsi.c vioraw.c vioqcow2.c fw_cfg.c
SRCS+= vm_agentx.c
Index: task.c
===
RCS file: task.c
diff -N task.c
--- /dev/null 1 Jan 1970 00:00:00 -
+++ task.c 11 Nov 2022 15:51:50 -
@@ -0,0 +1,158 @@
+/* $OpenBSD: task.c,v 1.2 2018/06/19 17:12:34 reyk Exp $ */
+
+/*
+ * Copyright (c) 2017 David Gwynne
+ *
+ * Permission to use, copy, modify, and distribute this software for
any
+ * purpose with or without fee is hereby granted, provided that the
above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE
FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "task.h"
+
+#define ISSET(_v, _m) ((_v) & (_m))
+#define SET(_v, _m)((_v) |= (_m))
+#define CLR(_v, _m)((_v) &= ~(_m))
+
+struct taskq {
+ pthread_t thread;
+ struct task_list list;
+ pthread_mutex_t mtx;
+ pthread_cond_tcv;
+};
+
+#define TASK_ONQUEUE (1 << 0)
+
+static void *taskq_run(void *);
+
+struct taskq *
+taskq_create(const char *name)
+{
+ struct taskq *tq;
+ int error;
+
+
move vmd vioblk handling to another thread
this updates a diff i had from a few years ago to move the vioblk
handling in vmd into a separate thread.
basically disk io in your virtual machine should not block the vcpu from
running now.
just throwing this out so people can give it a go and kick it around.
Index: Makefile
===
RCS file: /cvs/src/usr.sbin/vmd/Makefile,v
retrieving revision 1.28
diff -u -p -r1.28 Makefile
--- Makefile10 Nov 2022 11:46:39 - 1.28
+++ Makefile11 Nov 2022 15:51:50 -
@@ -5,7 +5,7 @@
PROG= vmd
SRCS= vmd.c control.c log.c priv.c proc.c config.c vmm.c
SRCS+= vm.c loadfile_elf.c pci.c virtio.c i8259.c mc146818.c
-SRCS+= ns8250.c i8253.c dhcp.c packet.c mmio.c
+SRCS+= ns8250.c i8253.c dhcp.c packet.c mmio.c task.c
SRCS+= parse.y atomicio.c vioscsi.c vioraw.c vioqcow2.c fw_cfg.c
SRCS+= vm_agentx.c
Index: task.c
===
RCS file: task.c
diff -N task.c
--- /dev/null 1 Jan 1970 00:00:00 -
+++ task.c 11 Nov 2022 15:51:50 -
@@ -0,0 +1,158 @@
+/* $OpenBSD: task.c,v 1.2 2018/06/19 17:12:34 reyk Exp $ */
+
+/*
+ * Copyright (c) 2017 David Gwynne
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "task.h"
+
+#define ISSET(_v, _m) ((_v) & (_m))
+#define SET(_v, _m)((_v) |= (_m))
+#define CLR(_v, _m)((_v) &= ~(_m))
+
+struct taskq {
+ pthread_t thread;
+ struct task_list list;
+ pthread_mutex_t mtx;
+ pthread_cond_tcv;
+};
+
+#define TASK_ONQUEUE (1 << 0)
+
+static void *taskq_run(void *);
+
+struct taskq *
+taskq_create(const char *name)
+{
+ struct taskq *tq;
+ int error;
+
+ tq = malloc(sizeof(*tq));
+ if (tq == NULL)
+ return (NULL);
+
+ TAILQ_INIT(>list);
+
+ error = pthread_mutex_init(>mtx, NULL);
+ if (error != 0)
+ goto free;
+
+ error = pthread_cond_init(>cv, NULL);
+ if (error != 0)
+ goto mtx;
+
+ error = pthread_create(>thread, NULL, taskq_run, tq);
+ if (error != 0)
+ goto cv;
+
+ pthread_set_name_np(tq->thread, name);
+
+ return (tq);
+
+cv:
+ pthread_cond_destroy(>cv);
+mtx:
+ pthread_mutex_destroy(>mtx); /* can this really fail? */
+free:
+ free(tq);
+
+ errno = error;
+ return (NULL);
+}
+
+static void *
+taskq_run(void *tqarg)
+{
+ struct taskq *tq = tqarg;
+ struct task *t;
+
+ void (*t_func)(void *);
+ void *t_arg;
+
+ for (;;) {
+ pthread_mutex_lock(>mtx);
+ while ((t = TAILQ_FIRST(>list)) == NULL)
+ pthread_cond_wait(>cv, >mtx);
+
+ TAILQ_REMOVE(>list, t, t_entry);
+ CLR(t->t_flags, TASK_ONQUEUE);
+
+ t_func = t->t_func;
+ t_arg = t->t_arg;
+
+ pthread_mutex_unlock(>mtx);
+
+ (*t_func)(t_arg);
+ }
+
+ return (NULL);
+}
+
+void
+task_set(struct task *t, void (*fn)(void *), void *arg)
+{
+ t->t_func = fn;
+ t->t_arg = arg;
+ t->t_flags = 0;
+}
+
+int
+task_add(struct taskq *tq, struct task *t)
+{
+ int rv = 1;
+
+ if (ISSET(t->t_flags, TASK_ONQUEUE))
+ return (0);
+
+ pthread_mutex_lock(>mtx);
+ if (ISSET(t->t_flags, TASK_ONQUEUE))
+ rv = 0;
+ else {
+ SET(t->t_flags, TASK_ONQUEUE);
+ TAILQ_INSERT_TAIL(>list, t, t_entry);
+ pthread_cond_signal(>cv);
+ }
+ pthread_mutex_unlock(>mtx);
+
+ return (rv);
+}
+
+int
+task_del(struct taskq *tq, struct task *t)
+{
+ int rv = 1;
+
+ if (!ISSET(t->t_flags, TASK_ONQUEUE))
+ return (0);
+
+ pthread_mutex_lock(>mtx);
+ if (!ISSET(t->t_flags, TASK_ONQUEUE))
+ rv = 0;
+ else {
+ TAILQ_REMOVE(>list, t, t_entry);
+ CLR(t->t_flags, TASK_ONQUEUE);
+ }
+ pthread_mutex_unlock(>mtx);
+
+ return (rv);
+}
Index: task.h
