From: Liu Yuan <[email protected]>
There are mainly two changes:
1 remove dumb ring_buffer and use strbuf to store trace items
2 use another mechanism to suspend/resume worker threads for dynamic patching.
As before, if users don't configure with --enable-trace, there is actually zero
impact on the code base. All the tracer funtions will be complied out.
With reworks, add a new field to trace output: thread name (worker_name +
worker_idx).
For e.x, we can only filter the trace output and get the gateway thread trace:
$ collie/collie debug trace start
... cluster is operating
$ collie/collie debug trace stop
$ collie/collie debug trace cat > t
$ grep 'gway 27' t | > t1
$ cat t1
gway 27 | | do_process_work() {
gway 27 | | gateway_write_obj() {
gway 27 | | gateway_forward_request() {
gway 27 | | gateway_to_peer_opcode() {
gway 27 | 0.793| }
gway 27 | | get_sd_op() {
gway 27 | 0.598| }
gway 27 | | get_req_copy_number() {
gway 27 | 0.547| }
gway 27 | | sheep_get_sockfd() {
gway 27 | | addr_to_str() {
gway 27 | 1.904| }
gway 27 | | sockfd_cache_search() {
gway 27 | 0.675| }
gway 27 | | xmalloc() {
gway 27 | 1.36 | }
gway 27 | 19.773| }
gway 27 | | send_req() {
gway 27 | 29.548| }
gway 27 | | sheep_get_sockfd() {
gway 27 | | addr_to_str() {
gway 27 | 2.499| }
gway 27 | | sockfd_cache_search() {
gway 27 | 5.33 | }
gway 27 | | xmalloc() {
gway 27 | 1.859| }
gway 27 | 35.478| }
gway 27 | | send_req() {
gway 27 | 27.173| }
gway 27 | | sheep_do_op_work() {
gway 27 | | peer_write_obj() {
gway 27 | | do_write_obj() {
gway 27 | | strbuf_addf() {
gway 27 | 5.809| }
gway 27 | | jrnl_begin() {
gway 27 | | xzalloc() {
gway 27 | | xmalloc() {
gway 27 | 1.143| }
gway 27 | 3.141| }
gway 27 | | xpwrite() {
gway 27 | 41206.762| }
gway 27 | | xpwrite() {
gway 27 | 27106.822| }
gway 27 | | xpwrite() {
gway 27 | 16433.765| }
gway 27 | 84822.558| }
gway 27 | | default_write() {
gway 27 | | get_obj_path() {
gway 27 | 1.873| }
gway 27 | | xpwrite() {
gway 27 | 37716.486| }
gway 27 | 37739.304| }
...
Signed-off-by: Liu Yuan <[email protected]>
---
collie/debug.c | 5 +-
configure.ac | 9 +--
include/sheep.h | 2 +
include/util.h | 23 --------
lib/Makefile.am | 3 +-
lib/ring_buffer.c | 54 -----------------
lib/strbuf.c | 4 +-
lib/util.c | 2 +-
sheep/ops.c | 3 +-
sheep/trace/graph.c | 20 +++----
sheep/trace/trace.c | 163 +++++++++++++++++++++++++++++++++------------------
sheep/trace/trace.h | 18 +++---
sheep/work.c | 4 ++
sheep/work.h | 2 +
14 files changed, 142 insertions(+), 170 deletions(-)
delete mode 100644 lib/ring_buffer.c
diff --git a/collie/debug.c b/collie/debug.c
index d016e85..2be873f 100644
--- a/collie/debug.c
+++ b/collie/debug.c
@@ -72,10 +72,9 @@ static int do_trace_cat(void)
static int debug_trace(int argc, char **argv)
{
- int fd, ret, i, l;
+ int ret, i, l;
struct sd_req hdr;
struct sd_rsp *rsp = (struct sd_rsp *)&hdr;
- unsigned rlen, wlen;
char *cmd = argv[optind];
int enabled;
@@ -118,7 +117,7 @@ static int debug_parser(int ch, char *opt)
static struct subcommand debug_cmd[] = {
{"trace", "{start, stop, cat}", "aph", "trace the node",
- 0, debug_trace},
+ NULL, 0, debug_trace},
{NULL,},
};
diff --git a/configure.ac b/configure.ac
index 91126e2..d6ed0dd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -202,12 +202,9 @@ AC_ARG_WITH([initddir],
[ INITDDIR="$withval" ],
[ INITDDIR="$sysconfdir/init.d" ])
-# Fixme: tracer is disabled for 0.4.0 and should be enabled later.
-#
-#AC_ARG_ENABLE([trace],
-# [ --enable-trace : enable trace],,
-# [ enable_trace="no" ],)
-enable_trace="no"
+AC_ARG_ENABLE([trace],
+ [ --enable-trace : enable trace],,
+ [ enable_trace="no" ],)
AM_CONDITIONAL(BUILD_TRACE, test x$enable_trace = xyes)
PKG_CHECK_MODULES([fuse],[fuse], HAVE_FUSE="yes", HAVE_FUSE="no")
diff --git a/include/sheep.h b/include/sheep.h
index fbbab85..a71b026 100644
--- a/include/sheep.h
+++ b/include/sheep.h
@@ -46,8 +46,10 @@ struct vdi_copy {
#define TRACE_BUF_LEN (1024 * 1024 * 8)
#define TRACE_FNAME_LEN 36
+#define TRACE_THREAD_LEN 20
struct trace_graph_item {
+ char tname[TRACE_THREAD_LEN];
int type;
char fname[TRACE_FNAME_LEN];
int depth;
diff --git a/include/util.h b/include/util.h
index fcd59db..c482c12 100644
--- a/include/util.h
+++ b/include/util.h
@@ -77,27 +77,4 @@ extern ssize_t xpread(int fd, void *buf, size_t count, off_t
offset);
extern ssize_t xpwrite(int fd, const void *buf, size_t count, off_t offset);
extern int rmdir_r(char *dir_path);
-/* ring_buffer.c */
-struct rbuffer {
- struct list_head list;
- char *buffer; /* data buffer */
- char *buffer_end;
- size_t capacity; /* initial maximum number of items in the
buffer */
- size_t count; /* number of items in the buffer */
- size_t sz; /* size of each item in the buffer */
- char *head;
- char *tail;
-};
-
-static inline size_t rbuffer_size(struct rbuffer *rbuf)
-{
- return rbuf->count * rbuf->sz;
-}
-
-void rbuffer_push(struct rbuffer *rbuf, const void *item);
-void rbuffer_pop(struct rbuffer *rbuf, void *item);
-void rbuffer_destroy(struct rbuffer *rbuf);
-void rbuffer_create(struct rbuffer *rbuf, size_t capacity, size_t item_size);
-void rbuffer_reset(struct rbuffer *rbuf);
-
#endif
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 0e11ec8..1c962fd 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -4,5 +4,4 @@ INCLUDES = -I$(top_builddir)/include
-I$(top_srcdir)/include
noinst_LIBRARIES = libsheepdog.a
-libsheepdog_a_SOURCES = event.c logger.c net.c util.c rbtree.c ring_buffer.c \
- strbuf.c sha1.c
+libsheepdog_a_SOURCES = event.c logger.c net.c util.c rbtree.c strbuf.c sha1.c
diff --git a/lib/ring_buffer.c b/lib/ring_buffer.c
deleted file mode 100644
index a37d6c7..0000000
--- a/lib/ring_buffer.c
+++ /dev/null
@@ -1,54 +0,0 @@
-#include <pthread.h>
-#include <stdlib.h>
-
-#include "util.h"
-#include "logger.h"
-
-notrace void rbuffer_create(struct rbuffer *rbuf, size_t capacity, size_t sz)
-{
- rbuf->buffer = xmalloc(capacity * sz);
- rbuf->buffer_end = rbuf->buffer + capacity * sz;
- rbuf->capacity = capacity;
- rbuf->count = 0;
- rbuf->sz = sz;
- rbuf->head = rbuf->tail = rbuf->buffer;
-}
-
-notrace void rbuffer_destroy(struct rbuffer *rbuf)
-{
- free(rbuf->buffer);
-}
-
-notrace void rbuffer_reset(struct rbuffer *rbuf)
-{
- rbuf->count = 0;
- rbuf->head = rbuf->tail = rbuf->buffer;
-}
-
-/* Push the item to the tail of the buffer */
-notrace void rbuffer_push(struct rbuffer *rbuf, const void *item)
-{
- if (rbuf->count == rbuf->capacity) {
- dprintf("buffer full\n");
- return;
- }
- memcpy(rbuf->tail, item, rbuf->sz);
- rbuf->tail += rbuf->sz;
- if (rbuf->tail == rbuf->buffer_end)
- rbuf->tail = rbuf->buffer;
- rbuf->count++;
-}
-
-/* Push the item from the head of the buffer */
-notrace void rbuffer_pop(struct rbuffer *rbuf, void *item)
-{
- if (rbuf->count == 0) {
- dprintf("no item left\n");
- return;
- }
- memcpy(item, rbuf->head, rbuf->sz);
- rbuf->head += rbuf->sz;
- if (rbuf->head == rbuf->buffer_end)
- rbuf->head = rbuf->buffer;
- rbuf->count--;
-}
diff --git a/lib/strbuf.c b/lib/strbuf.c
index f87400e..c425b17 100644
--- a/lib/strbuf.c
+++ b/lib/strbuf.c
@@ -48,7 +48,7 @@ void strbuf_attach(struct strbuf *sb, void *buf, size_t len,
size_t alloc)
sb->buf[sb->len] = '\0';
}
-void strbuf_grow(struct strbuf *sb, size_t extra)
+notrace void strbuf_grow(struct strbuf *sb, size_t extra)
{
if (sb->len + extra + 1 <= sb->len)
panic("you want to use way too much memory");
@@ -96,7 +96,7 @@ void strbuf_remove(struct strbuf *sb, size_t pos, size_t len)
strbuf_splice(sb, pos, len, NULL, 0);
}
-void strbuf_add(struct strbuf *sb, const void *data, size_t len)
+notrace void strbuf_add(struct strbuf *sb, const void *data, size_t len)
{
strbuf_grow(sb, len);
memcpy(sb->buf + sb->len, data, len);
diff --git a/lib/util.c b/lib/util.c
index ea8f1b8..78bba8c 100644
--- a/lib/util.c
+++ b/lib/util.c
@@ -60,7 +60,7 @@ void *xzalloc(size_t size)
return ret;
}
-void *xrealloc(void *ptr, size_t size)
+notrace void *xrealloc(void *ptr, size_t size)
{
void *ret = realloc(ptr, size);
if (!ret && !size)
diff --git a/sheep/ops.c b/sheep/ops.c
index ccb1c5e..c2886a0 100644
--- a/sheep/ops.c
+++ b/sheep/ops.c
@@ -674,9 +674,8 @@ static int local_trace_ops(const struct sd_req *req, struct
sd_rsp *rsp, void *d
static int local_trace_cat_ops(const struct sd_req *req, struct sd_rsp *rsp,
void *data)
{
- rsp->data_length = trace_copy_buffer(data);
+ rsp->data_length = trace_buffer_pop(data, req->data_length);
dprintf("%u\n", rsp->data_length);
- trace_reset_buffer();
return SD_RES_SUCCESS;
}
diff --git a/sheep/trace/graph.c b/sheep/trace/graph.c
index 5da4a54..8a7b73e 100644
--- a/sheep/trace/graph.c
+++ b/sheep/trace/graph.c
@@ -13,6 +13,7 @@
#include <time.h>
#include <assert.h>
+#include <sched.h>
#include "trace.h"
#include "logger.h"
@@ -25,8 +26,6 @@ static __thread struct trace_ret_stack {
unsigned long long entry_time;
} trace_ret_stack[100]; /* FIXME: consider stack overrun */
-static __thread struct rbuffer rbuf;
-
static void push_return_trace(unsigned long ret, unsigned long long etime,
unsigned long func, int *depth)
{
@@ -55,12 +54,12 @@ static notrace uint64_t clock_get_time(void)
static notrace void default_trace_graph_entry(struct trace_graph_item *item)
{
- rbuffer_push(&rbuf, item);
+ trace_buffer_push(sched_getcpu(), item);
}
static notrace void default_trace_graph_return(struct trace_graph_item *item)
{
- rbuffer_push(&rbuf, item);
+ trace_buffer_push(sched_getcpu(), item);
}
static trace_func_graph_ent_t trace_graph_entry = default_trace_graph_entry;
@@ -71,6 +70,9 @@ notrace unsigned long trace_return_call(void)
struct trace_graph_item trace;
unsigned long ret;
+ memset(&trace, 0, sizeof(trace));
+
+ get_thread_name(trace.tname);
trace.return_time = clock_get_time();
pop_return_trace(&trace, &ret);
trace.type = TRACE_GRAPH_RETURN;
@@ -79,13 +81,6 @@ notrace unsigned long trace_return_call(void)
return ret;
}
-notrace void trace_init_buffer(struct list_head *list)
-{
- int sz = sizeof(struct trace_graph_item);
- rbuffer_create(&rbuf, TRACE_BUF_LEN / sz, sz);
- list_add(&rbuf.list, list);
-}
-
/* Hook the return address and push it in the trace_ret_stack.
*
* ip: the address of the call instruction in the code.
@@ -98,10 +93,13 @@ static notrace void graph_tracer(unsigned long ip, unsigned
long *ret_addr)
struct trace_graph_item trace;
struct caller *cr;
+ memset(&trace, 0, sizeof(trace));
+
cr = trace_lookup_ip(ip, 0);
assert(cr->namelen + 1 < TRACE_FNAME_LEN);
memcpy(trace.fname, cr->name, cr->namelen);
memset(trace.fname + cr->namelen, '\0', 1);
+ get_thread_name(trace.tname);
*ret_addr = (unsigned long)trace_return_caller;
entry_time = clock_get_time();
diff --git a/sheep/trace/trace.c b/sheep/trace/trace.c
index 56817e8..7911101 100644
--- a/sheep/trace/trace.c
+++ b/sheep/trace/trace.c
@@ -16,12 +16,15 @@
#include <unistd.h>
#include <pthread.h>
#include <signal.h>
+#include <sys/eventfd.h>
#include "trace.h"
#include "logger.h"
#include "list.h"
#include "work.h"
#include "sheepdog_proto.h"
+#include "strbuf.h"
+#include "event.h"
#define TRACE_HASH_BITS 7
#define TRACE_HASH_SIZE (1 << TRACE_HASH_BITS)
@@ -31,12 +34,15 @@ static LIST_HEAD(caller_list);
static pthread_mutex_t trace_lock = PTHREAD_MUTEX_INITIALIZER;
static trace_func_t trace_func = trace_call;
-static int trace_count;
-static int trace_buffer_inited;
-static LIST_HEAD(buffer_list);
-pthread_cond_t trace_cond = PTHREAD_COND_INITIALIZER;
-pthread_mutex_t trace_mux = PTHREAD_MUTEX_INITIALIZER;
+static int trace_efd;
+static int nr_short_thread;
+static int trace_in_patch;
+
+pthread_mutex_t suspend_lock = PTHREAD_MUTEX_INITIALIZER;
+
+static struct strbuf *buffer;
+static int nr_cpu;
union instruction {
unsigned char start[INSN_SIZE];
@@ -48,15 +54,11 @@ union instruction {
static notrace void suspend(int num)
{
- dprintf("worker thread %u going to suspend\n", (int)pthread_self());
-
- pthread_mutex_lock(&trace_mux);
- trace_count--;
- if (!trace_buffer_inited)
- trace_init_buffer(&buffer_list); /* init worker threads rbuffer
*/
- pthread_cond_wait(&trace_cond, &trace_mux);
- pthread_mutex_unlock(&trace_mux);
- dprintf("worker thread going to resume\n");
+ dprintf("going to suspend\n");
+ pthread_mutex_lock(&suspend_lock);
+ /* Now I am suspended and sleep on suspend_lock */
+ pthread_mutex_unlock(&suspend_lock);
+ dprintf("going to resume\n");
}
static inline int trace_hash(unsigned long ip)
@@ -178,29 +180,19 @@ notrace int register_trace_function(trace_func_t func)
static notrace void suspend_worker_threads(void)
{
struct worker_info *wi;
- int i;
- trace_count = total_nr_workers;
+
+ /* Hold the lock, then all other worker can sleep on it */
+ pthread_mutex_lock(&suspend_lock);
list_for_each_entry(wi, &worker_info_list, worker_info_siblings) {
- for (i = 0; i < wi->nr_threads; i++)
- if (pthread_kill(wi->worker_thread[i], SIGUSR2) != 0)
- dprintf("%m\n");
- }
-wait_for_worker_suspend:
- pthread_mutex_lock(&trace_mux);
- if (trace_count > 0) {
- pthread_mutex_unlock(&trace_mux);
- pthread_yield();
- goto wait_for_worker_suspend;
+ if (wi->worker_thread &&
+ pthread_kill(wi->worker_thread, SIGUSR2) != 0)
+ dprintf("%m\n");
}
- pthread_mutex_unlock(&trace_mux);
- trace_buffer_inited = 1;
}
static notrace void resume_worker_threads(void)
{
- pthread_mutex_lock(&trace_mux);
- pthread_cond_broadcast(&trace_cond);
- pthread_mutex_unlock(&trace_mux);
+ pthread_mutex_unlock(&suspend_lock);
}
static notrace void patch_all_sites(unsigned long addr)
@@ -227,6 +219,40 @@ static notrace void nop_all_sites(void)
pthread_mutex_unlock(&trace_lock);
}
+static notrace void enable_tracer(int fd, int events, void *data)
+{
+ eventfd_t value;
+ int ret;
+
+ ret = eventfd_read(trace_efd, &value);
+ if (ret < 0)
+ eprintf("%m");
+
+ suspend_worker_threads();
+ patch_all_sites((unsigned long)trace_caller);
+ resume_worker_threads();
+ unregister_event(trace_efd);
+ trace_in_patch = 0;
+ dprintf("tracer enabled\n");
+}
+
+static notrace void disable_tracer(int fd, int events, void *data)
+{
+ eventfd_t value;
+ int ret;
+
+ ret = eventfd_read(fd, &value);
+ if (ret < 0)
+ eprintf("%m");
+
+ suspend_worker_threads();
+ nop_all_sites();
+ resume_worker_threads();
+ unregister_event(trace_efd);
+ trace_in_patch = 0;
+ dprintf("tracer disabled\n");
+}
+
notrace int trace_enable(void)
{
if (trace_func == trace_call) {
@@ -234,19 +260,17 @@ notrace int trace_enable(void)
return SD_RES_NO_TAG;
}
- suspend_worker_threads();
- patch_all_sites((unsigned long)trace_caller);
- resume_worker_threads();
- dprintf("patch tracer done\n");
+ register_event(trace_efd, enable_tracer, NULL);
+ trace_in_patch = 1;
+
return SD_RES_SUCCESS;
}
notrace int trace_disable(void)
{
- suspend_worker_threads();
- nop_all_sites();
- resume_worker_threads();
- dprintf("patch nop done\n");
+ register_event(trace_efd, disable_tracer, NULL);
+ trace_in_patch = 1;
+
return SD_RES_SUCCESS;
}
@@ -264,33 +288,50 @@ int trace_init_signal(void)
return 0;
}
-notrace int trace_copy_buffer(void *buf)
+notrace uint32_t trace_buffer_pop(void *buf, uint32_t len)
{
- struct rbuffer *rbuf;
- int total = 0;
-
- list_for_each_entry(rbuf, &buffer_list, list) {
- int rbuf_size = rbuffer_size(rbuf);
- if (rbuf_size) {
- memcpy((char *)buf + total, rbuf->buffer, rbuf_size);
- total += rbuf_size;
- }
+ uint32_t readin, count = 0, requested = len;
+ char *buff = (char *)buf;
+ int i;
+
+ for (i = 0; i < nr_cpu; i++) {
+ readin = strbuf_stripout(&buffer[i], buff, len);
+ count += readin;
+ if (count == requested)
+ return count;
+ if (readin == 0)
+ continue;
+
+ len -= readin;
+ buff += readin;
}
- return total;
+
+ return count;
}
-notrace void trace_reset_buffer(void)
+notrace void trace_buffer_push(int cpuid, struct trace_graph_item *item)
{
- struct rbuffer *rbuf;
+ strbuf_add(&buffer[cpuid], item, sizeof(*item));
+}
- list_for_each_entry(rbuf, &buffer_list, list) {
- rbuffer_reset(rbuf);
- }
+void short_thread_begin(void)
+{
+ nr_short_thread++;
+}
+
+void short_thread_end(void)
+{
+ eventfd_t value = 1;
+
+ nr_short_thread--;
+ if (trace_in_patch && nr_short_thread == 0)
+ eventfd_write(trace_efd, value);
}
notrace int trace_init()
{
sigset_t block;
+ int i;
sigemptyset(&block);
sigaddset(&block, SIGUSR2);
@@ -304,9 +345,15 @@ notrace int trace_init()
return -1;
}
- trace_init_buffer(&buffer_list); /* init main thread ring buffer */
replace_mcount_call((unsigned long)do_trace_init);
- dprintf("main thread %u\n", (int)pthread_self());
- dprintf("trace support enabled.\n");
+
+ nr_cpu = sysconf(_SC_NPROCESSORS_ONLN);
+ buffer = xzalloc(sizeof(*buffer) * nr_cpu);
+ for (i = 0; i < nr_cpu; i++)
+ strbuf_init(&buffer[i], 0);
+
+ trace_efd = eventfd(0, EFD_NONBLOCK);
+
+ dprintf("trace support enabled. cpu count %d.\n", nr_cpu);
return 0;
}
diff --git a/sheep/trace/trace.h b/sheep/trace/trace.h
index dd95a39..19e79c5 100644
--- a/sheep/trace/trace.h
+++ b/sheep/trace/trace.h
@@ -34,7 +34,6 @@ typedef void (*trace_func_graph_ret_t)(struct
trace_graph_item *);
typedef void (*trace_func_graph_ent_t)(struct trace_graph_item *);
/* graph.c */
-extern void trace_init_buffer(struct list_head *list);
/* stabs.c */
extern int get_ipinfo(unsigned long ip, struct ipinfo *info);
@@ -50,24 +49,27 @@ extern unsigned long trace_return_call(void);
/* trace.c */
#ifdef ENABLE_TRACE
- extern pthread_cond_t trace_cond;
- extern pthread_mutex_t trace_mux;
-
extern int trace_init_signal(void);
extern int trace_init(void);
extern int register_trace_function(trace_func_t func);
extern int trace_enable(void);
extern int trace_disable(void);
extern struct caller *trace_lookup_ip(unsigned long ip, int create);
- extern int trace_copy_buffer(void *buf);
- extern void trace_reset_buffer(void);
+ extern uint32_t trace_buffer_pop(void *buf, uint32_t len);
+ extern void trace_buffer_push(int cpuid, struct trace_graph_item *item);
+ extern void short_thread_begin(void);
+ extern void short_thread_end(void);
#else
static inline int trace_init_signal(void) { return 0; }
static inline int trace_init(void) { return 0; }
static inline int trace_enable(void) { return 0; }
static inline int trace_disable(void) { return 0; }
- static inline int trace_copy_buffer(void *buf) { return 0; }
- static inline void trace_reset_buffer(void) {}
+ static inline uint32_t trace_buffer_pop(void *buf) { return 0; }
+ static inline void trace_buffer_push(int cpuid, struct
+ trace_graph_item *item) { return; }
+ extern inline void short_thread_begin(void) { return; }
+ extern inline void short_thread_end(void) { return; }
+
#endif /* ENABLE_TRACE */
#define register_tracer(new) \
diff --git a/sheep/work.c b/sheep/work.c
index 68f04ce..80096e3 100644
--- a/sheep/work.c
+++ b/sheep/work.c
@@ -32,6 +32,7 @@
#include "work.h"
#include "logger.h"
#include "event.h"
+#include "trace/trace.h"
static int efd;
int total_nr_workers;
@@ -95,6 +96,7 @@ static inline void create_short_thread(struct worker_info *wi,
err = pthread_create(&thread, NULL, run_short_thread, sw);
if (err)
panic("%s\n", strerror(err));
+ short_thread_begin();
}
void queue_work(struct work_queue *q, struct work *work)
@@ -133,6 +135,8 @@ static void bs_thread_request_done(int fd, int events, void
*data)
list_del(&work->w_list);
work->done(work);
+ if (!wi->ordered)
+ short_thread_end();
}
}
}
diff --git a/sheep/work.h b/sheep/work.h
index 3634560..bcb8383 100644
--- a/sheep/work.h
+++ b/sheep/work.h
@@ -1,6 +1,8 @@
#ifndef __WORK_H__
#define __WORK_H__
+#include <stdbool.h>
+
struct work;
struct work_queue;
--
1.7.10.2
--
sheepdog mailing list
[email protected]
http://lists.wpkg.org/mailman/listinfo/sheepdog
