On Thu, May 12, 2011 at 12:54 PM, Stefan Hajnoczi <stefa...@linux.vnet.ibm.com> wrote: > From: Kevin Wolf <kw...@redhat.com> > > Asynchronous code is becoming very complex. At the same time > synchronous code is growing because it is convenient to write. > Sometimes duplicate code paths are even added, one synchronous and the > other asynchronous. This patch introduces coroutines which allow code > that looks synchronous but is asynchronous under the covers. > > A coroutine has its own stack and is therefore able to preserve state > across blocking operations, which traditionally require callback > functions and manual marshalling of parameters. > > Creating and starting a coroutine is easy: > > coroutine = qemu_coroutine_create(my_coroutine); > qemu_coroutine_enter(coroutine, my_data); > > The coroutine then executes until it returns or yields: > > void coroutine_fn my_coroutine(void *opaque) { > MyData *my_data = opaque; > > /* do some work */ > > qemu_coroutine_yield(); > > /* do some more work */ > } > > Yielding switches control back to the caller of qemu_coroutine_enter(). > This is typically used to switch back to the main thread's event loop > after issuing an asynchronous I/O request. The request callback will > then invoke qemu_coroutine_enter() once more to switch back to the > coroutine. > > Note that coroutines never execute concurrently and should only be used > from threads which hold the global mutex. This restriction makes > programming with coroutines easier than with threads. Race conditions > cannot occur since only one coroutine may be active at any time. Other > coroutines can only run across yield. > > This coroutines implementation is based on the gtk-vnc implementation > written by Anthony Liguori <anth...@codemonkey.ws> but it has been > significantly rewritten by Kevin Wolf <kw...@redhat.com> to use > setjmp()/longjmp() instead of the more expensive swapcontext(). > > Signed-off-by: Kevin Wolf <kw...@redhat.com> > Signed-off-by: Stefan Hajnoczi <stefa...@linux.vnet.ibm.com> > --- > Makefile.objs | 7 +++ > coroutine-ucontext.c | 73 +++++++++++++++++++++++++++++ > coroutine-win32.c | 57 +++++++++++++++++++++++ > qemu-coroutine-int.h | 53 +++++++++++++++++++++ > qemu-coroutine.c | 125 > ++++++++++++++++++++++++++++++++++++++++++++++++++ > qemu-coroutine.h | 82 +++++++++++++++++++++++++++++++++ > trace-events | 5 ++ > 7 files changed, 402 insertions(+), 0 deletions(-) > create mode 100644 coroutine-ucontext.c > create mode 100644 coroutine-win32.c > create mode 100644 qemu-coroutine-int.h > create mode 100644 qemu-coroutine.c > create mode 100644 qemu-coroutine.h > > diff --git a/Makefile.objs b/Makefile.objs > index 9d8851e..cba6c2b 100644 > --- a/Makefile.objs > +++ b/Makefile.objs > @@ -11,6 +11,12 @@ oslib-obj-$(CONFIG_WIN32) += oslib-win32.o > oslib-obj-$(CONFIG_POSIX) += oslib-posix.o > > ####################################################################### > +# coroutines > +coroutine-obj-y = qemu-coroutine.o > +coroutine-obj-$(CONFIG_POSIX) += coroutine-ucontext.o > +coroutine-obj-$(CONFIG_WIN32) += coroutine-win32.o > + > +####################################################################### > # block-obj-y is code used by both qemu system emulation and qemu-img > > block-obj-y = cutils.o cache-utils.o qemu-malloc.o qemu-option.o module.o > async.o > @@ -67,6 +73,7 @@ common-obj-y += readline.o console.o cursor.o qemu-error.o > common-obj-y += $(oslib-obj-y) > common-obj-$(CONFIG_WIN32) += os-win32.o > common-obj-$(CONFIG_POSIX) += os-posix.o > +common-obj-y += $(coroutine-obj-y) > > common-obj-y += tcg-runtime.o host-utils.o > common-obj-y += irq.o ioport.o input.o > diff --git a/coroutine-ucontext.c b/coroutine-ucontext.c > new file mode 100644 > index 0000000..3b14ebf > --- /dev/null > +++ b/coroutine-ucontext.c > @@ -0,0 +1,73 @@ > +/* > + * ucontext coroutine initialization code > + * > + * Copyright (C) 2006 Anthony Liguori <anth...@codemonkey.ws> > + * Copyright (C) 2011 Kevin Wolf <kw...@redhat.com> > + * > + * This library is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.0 of the License, or (at your option) any later version. > + * > + * This library is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with this library; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 > USA
Please use the web link version. > + */ > + > +/* XXX Is there a nicer way to disable glibc's stack check for longjmp? */ What is the problem? > +#ifdef _FORTIFY_SOURCE > +#undef _FORTIFY_SOURCE > +#endif > +#include <setjmp.h> > +#include <stdint.h> > +#include <ucontext.h> > +#include "qemu-coroutine-int.h" > + > +static Coroutine *new_coroutine; > + > +static void continuation_trampoline(void) > +{ > + Coroutine *co = new_coroutine; > + > + /* Initialize longjmp environment and switch back to > + * qemu_coroutine_init_env() in the old ucontext. */ > + if (!setjmp(co->env)) { > + return; > + } > + > + while (true) { > + co->entry(co->data); > + if (!setjmp(co->env)) { > + longjmp(co->caller->env, COROUTINE_TERMINATE); > + } > + } > +} > + > +int qemu_coroutine_init_env(Coroutine *co, size_t stack_size) > +{ > + ucontext_t old_uc, uc; > + > + /* Create a new ucontext for switching to the coroutine stack and setting > + * up a longjmp environment. */ > + if (getcontext(&uc) == -1) { > + return -errno; > + } > + > + uc.uc_link = &old_uc; > + uc.uc_stack.ss_sp = co->stack; > + uc.uc_stack.ss_size = stack_size; > + uc.uc_stack.ss_flags = 0; > + > + new_coroutine = co; > + makecontext(&uc, (void *)continuation_trampoline, 0); > + > + /* Initialize the longjmp environment */ > + swapcontext(&old_uc, &uc); > + > + return 0; > +} > diff --git a/coroutine-win32.c b/coroutine-win32.c > new file mode 100644 > index 0000000..99141dd > --- /dev/null > +++ b/coroutine-win32.c > @@ -0,0 +1,57 @@ > +/* > + * Win32 coroutine initialization code > + * > + * Copyright (c) 2011 Kevin Wolf <kw...@redhat.com> > + * > + * Permission is hereby granted, free of charge, to any person obtaining a > copy > + * of this software and associated documentation files (the "Software"), to > deal > + * in the Software without restriction, including without limitation the > rights > + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell > + * copies of the Software, and to permit persons to whom the Software is > + * furnished to do so, subject to the following conditions: > + * > + * The above copyright notice and this permission notice shall be included in > + * all copies or substantial portions of the Software. > + * > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR > + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, > + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL > + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER > + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING > FROM, > + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN > + * THE SOFTWARE. > + */ > + > +#include "qemu-coroutine-int.h" > + > +static void __attribute__((used)) trampoline(Coroutine *co) > +{ > + if (!setjmp(co->env)) { > + return; > + } > + > + while (true) { > + co->entry(co->data); > + if (!setjmp(co->env)) { > + longjmp(co->caller->env, COROUTINE_TERMINATE); > + } > + } > +} > + > +int qemu_coroutine_init_env(Coroutine *co, size_t stack_size) > +{ > +#ifdef __i386__ > + asm volatile( > + "mov %%esp, %%ebx;" > + "mov %0, %%esp;" > + "pushl %1;" > + "call _trampoline;" > + "mov %%ebx, %%esp;" > + : : "r" (co->stack + stack_size), "r" (co) : "ebx" > + ); > +#else > + #error This host architecture is not supported for win32 > +#endif > + > + return 0; > +} > diff --git a/qemu-coroutine-int.h b/qemu-coroutine-int.h > new file mode 100644 > index 0000000..71c6ee9 > --- /dev/null > +++ b/qemu-coroutine-int.h > @@ -0,0 +1,53 @@ > +/* > + * Coroutine internals > + * > + * Copyright (c) 2011 Kevin Wolf <kw...@redhat.com> > + * > + * Permission is hereby granted, free of charge, to any person obtaining a > copy > + * of this software and associated documentation files (the "Software"), to > deal > + * in the Software without restriction, including without limitation the > rights > + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell > + * copies of the Software, and to permit persons to whom the Software is > + * furnished to do so, subject to the following conditions: > + * > + * The above copyright notice and this permission notice shall be included in > + * all copies or substantial portions of the Software. > + * > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR > + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, > + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL > + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER > + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING > FROM, > + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN > + * THE SOFTWARE. > + */ > + > +#ifndef QEMU_COROUTINE_INT_H > +#define QEMU_COROUTINE_INT_H > + > +#include <setjmp.h> > +#include "qemu-common.h" > +#include "qemu-queue.h" > +#include "qemu-coroutine.h" > + > +enum { > + /* setjmp() return values */ > + COROUTINE_YIELD = 1, > + COROUTINE_TERMINATE = 2, > +}; > + > +struct Coroutine { > + struct Coroutine *caller; > + > + char *stack; > + > + /* Used to pass arguments/return values for coroutines */ > + void *data; > + CoroutineEntry *entry; > + > + jmp_buf env; > +}; > + > +int qemu_coroutine_init_env(Coroutine *co, size_t stack_size); > + > +#endif > diff --git a/qemu-coroutine.c b/qemu-coroutine.c > new file mode 100644 > index 0000000..80bed14 > --- /dev/null > +++ b/qemu-coroutine.c > @@ -0,0 +1,125 @@ > +/* > + * QEMU coroutines > + * > + * Copyright IBM, Corp. 2011 > + * > + * Authors: > + * Stefan Hajnoczi <stefa...@linux.vnet.ibm.com> > + * Kevin Wolf <kw...@redhat.com> > + * > + * This work is licensed under the terms of the GNU LGPL, version 2 or later. > + * See the COPYING.LIB file in the top-level directory. > + * > + */ > + > +/* XXX Is there a nicer way to disable glibc's stack check for longjmp? */ > +#ifdef _FORTIFY_SOURCE > +#undef _FORTIFY_SOURCE > +#endif > +#include <setjmp.h> > + > +#include "trace.h" > +#include "qemu-queue.h" > +#include "qemu-common.h" > +#include "qemu-coroutine.h" > +#include "qemu-coroutine-int.h" > + > +static __thread Coroutine leader; $ cat thread.c static __thread int sigusr1_wfd; $ gcc thread.c -c -pthread thread.c:1: error: thread-local storage not supported for this target $ gcc -v Reading specs from /usr/bin/../lib/gcc-lib/sparc64-unknown-openbsd4.9/4.2.1/specs Target: sparc64-unknown-openbsd4.9 Configured with: OpenBSD/sparc64 system compiler Thread model: posix gcc version 4.2.1 20070719 > +static __thread Coroutine *current; > + > +static void coroutine_terminate(Coroutine *co) > +{ > + trace_qemu_coroutine_terminate(co); > + qemu_free(co->stack); > + qemu_free(co); > +} > + > +static Coroutine *coroutine_new(void) > +{ > + const size_t stack_size = 4 << 20; > + Coroutine *co; > + > + co = qemu_mallocz(sizeof(*co)); > + co->stack = qemu_malloc(stack_size); > + qemu_coroutine_init_env(co, stack_size); > + return co; > +} > + > +Coroutine *qemu_coroutine_create(CoroutineEntry *entry) > +{ > + Coroutine *co; > + > + co = coroutine_new(); > + co->entry = entry; > + > + return co; > +} > + > +Coroutine *coroutine_fn qemu_coroutine_self(void) > +{ > + if (current == NULL) { > + current = &leader; > + } > + > + return current; > +} > + > +bool qemu_in_coroutine(void) > +{ > + return qemu_coroutine_self() != &leader; > +} > + > +static void *coroutine_swap(Coroutine *from, Coroutine *to, void *opaque) > +{ > + int ret; > + void *to_data; > + > + to->data = opaque; > + > + ret = setjmp(from->env); > + switch (ret) { > + case COROUTINE_YIELD: > + return from->data; > + case COROUTINE_TERMINATE: > + current = to->caller; > + to_data = to->data; > + coroutine_terminate(to); > + return to_data; > + default: > + /* Switch to called coroutine */ > + current = to; > + longjmp(to->env, COROUTINE_YIELD); > + return NULL; > + } > +} > + > +void qemu_coroutine_enter(Coroutine *co, void *opaque) > +{ > + Coroutine *self = qemu_coroutine_self(); > + > + trace_qemu_coroutine_enter(self, co, opaque); > + > + if (co->caller) { > + fprintf(stderr, "Co-routine re-entered recursively\n"); > + abort(); > + } > + > + co->caller = self; > + coroutine_swap(self, co, opaque); > +} > + > +void *coroutine_fn qemu_coroutine_yield(void) > +{ > + Coroutine *self = qemu_coroutine_self(); > + Coroutine *to = self->caller; > + > + trace_qemu_coroutine_yield(self, self->caller); > + > + if (!to) { > + fprintf(stderr, "Co-routine is yielding to no one\n"); > + abort(); > + } > + > + self->caller = NULL; > + return coroutine_swap(self, to, NULL); > +} > diff --git a/qemu-coroutine.h b/qemu-coroutine.h > new file mode 100644 > index 0000000..b79b4bf > --- /dev/null > +++ b/qemu-coroutine.h > @@ -0,0 +1,82 @@ > +/* > + * QEMU coroutine implementation > + * > + * Copyright IBM, Corp. 2011 > + * > + * Authors: > + * Stefan Hajnoczi <stefa...@linux.vnet.ibm.com> > + * > + * This work is licensed under the terms of the GNU LGPL, version 2 or later. > + * See the COPYING.LIB file in the top-level directory. > + * > + */ > + > +#ifndef QEMU_COROUTINE_H > +#define QEMU_COROUTINE_H > + > +#include <stdbool.h> > + > +/** > + * Mark a function that executes in coroutine context > + * > + * Functions that execute in coroutine context cannot be called directly from > + * normal functions. In the future it would be nice to enable compiler or > + * static checker support for catching such errors. This annotation might > make > + * it possible and in the meantime it serves as documentation. > + * > + * For example: > + * > + * static void coroutine_fn foo(void) { > + * .... > + * } > + */ > +#define coroutine_fn > + > +typedef struct Coroutine Coroutine; > + > +/** > + * Coroutine entry point > + * > + * When the coroutine is entered for the first time, opaque is passed in as > an > + * argument. > + * > + * When this function returns, the coroutine is destroyed automatically and > + * execution continues in the caller who last entered the coroutine. > + */ > +typedef void coroutine_fn CoroutineEntry(void *opaque); > + > +/** > + * Create a new coroutine > + * > + * Use qemu_coroutine_enter() to actually transfer control to the coroutine. > + */ > +Coroutine *qemu_coroutine_create(CoroutineEntry *entry); > + > +/** > + * Transfer control to a coroutine > + * > + * The opaque argument is made available to the coroutine either as the entry > + * function argument if this is the first time a new coroutine is entered, or > + * as the return value from qemu_coroutine_yield(). > + */ > +void qemu_coroutine_enter(Coroutine *coroutine, void *opaque); > + > +/** > + * Transfer control back to a coroutine's caller > + * > + * The return value is the argument passed back in from the next > + * qemu_coroutine_enter(). > + */ > +void *coroutine_fn qemu_coroutine_yield(void); > + > +/** > + * Get the currently executing coroutine > + */ > +Coroutine *coroutine_fn qemu_coroutine_self(void); > + > +/** > + * Return whether or not currently inside a coroutine > + */ > +bool qemu_in_coroutine(void); > + > +#endif /* QEMU_COROUTINE_H */ > diff --git a/trace-events b/trace-events > index 4f965e2..2d4db05 100644 > --- a/trace-events > +++ b/trace-events > @@ -361,3 +361,8 @@ disable milkymist_uart_pulse_irq_tx(void) "Pulse IRQ TX" > # hw/milkymist-vgafb.c > disable milkymist_vgafb_memory_read(uint32_t addr, uint32_t value) "addr > %08x value %08x" > disable milkymist_vgafb_memory_write(uint32_t addr, uint32_t value) "addr > %08x value %08x" > + > +# qemu-coroutine.c > +qemu_coroutine_enter(void *from, void *to, void *opaque) "from %p to %p > opaque %p" > +qemu_coroutine_yield(void *from, void *to) "from %p to %p" > +qemu_coroutine_terminate(void *co) "self %p" Not disabled?