Module Name: src
Committed By: pooka
Date: Tue Mar 8 15:34:38 UTC 2011
Modified Files:
src/lib/librumpclient: rumpclient.c
src/lib/librumpuser: rumpuser_sp.c sp_common.c
Log Message:
Use sendmsg() instead multiple calls to sendto(). It behaves
slightly better in the multithreaded exec case in terms of partial
frames sent. Plus, it's theoretically a little cheaper.
There's still a gray area with partial transfer from sendmsg(),
but I'm not sure I care enough about some academic scenario to
start fixing it (it basically needs an OOB exec signaller).
To generate a diff of this commit:
cvs rdiff -u -r1.38 -r1.39 src/lib/librumpclient/rumpclient.c
cvs rdiff -u -r1.44 -r1.45 src/lib/librumpuser/rumpuser_sp.c
cvs rdiff -u -r1.30 -r1.31 src/lib/librumpuser/sp_common.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/lib/librumpclient/rumpclient.c
diff -u src/lib/librumpclient/rumpclient.c:1.38 src/lib/librumpclient/rumpclient.c:1.39
--- src/lib/librumpclient/rumpclient.c:1.38 Sun Feb 27 12:58:29 2011
+++ src/lib/librumpclient/rumpclient.c Tue Mar 8 15:34:37 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: rumpclient.c,v 1.38 2011/02/27 12:58:29 pooka Exp $ */
+/* $NetBSD: rumpclient.c,v 1.39 2011/03/08 15:34:37 pooka Exp $ */
/*
* Copyright (c) 2010, 2011 Antti Kantee. All Rights Reserved.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__RCSID("$NetBSD: rumpclient.c,v 1.38 2011/02/27 12:58:29 pooka Exp $");
+__RCSID("$NetBSD: rumpclient.c,v 1.39 2011/03/08 15:34:37 pooka Exp $");
#include <sys/param.h>
#include <sys/event.h>
@@ -66,8 +66,7 @@
int (*host_fcntl)(int, int, ...);
int (*host_poll)(struct pollfd *, nfds_t, int);
ssize_t (*host_read)(int, void *, size_t);
-ssize_t (*host_sendto)(int, const void *, size_t, int,
- const struct sockaddr *, socklen_t);
+ssize_t (*host_sendmsg)(int, const struct msghdr *, int);
int (*host_setsockopt)(int, int, int, const void *, socklen_t);
int (*host_dup)(int);
@@ -96,7 +95,7 @@
static time_t retrytimo = 0;
static int
-send_with_recon(struct spclient *spc, const void *data, size_t dlen)
+send_with_recon(struct spclient *spc, struct iovec *iov, size_t iovlen)
{
struct timeval starttime, curtime;
time_t prevreconmsg;
@@ -104,7 +103,7 @@
int rv;
for (prevreconmsg = 0, reconretries = 0;;) {
- rv = dosend(spc, data, dlen);
+ rv = dosend(spc, iov, iovlen);
if (__predict_false(rv == ENOTCONN || rv == EBADF)) {
/* no persistent connections */
if (retrytimo == 0) {
@@ -287,6 +286,7 @@
{
struct rsp_hdr rhdr;
struct respwait rw;
+ struct iovec iov[2];
int rv;
rhdr.rsp_len = sizeof(rhdr) + dlen;
@@ -294,13 +294,12 @@
rhdr.rsp_type = RUMPSP_SYSCALL;
rhdr.rsp_sysnum = sysnum;
+ IOVPUT(iov[0], rhdr);
+ IOVPUT_WITHSIZE(iov[1], __UNCONST(data), dlen);
+
do {
putwait(spc, &rw, &rhdr);
- if ((rv = send_with_recon(spc, &rhdr, sizeof(rhdr))) != 0) {
- unputwait(spc, &rw);
- continue;
- }
- if ((rv = send_with_recon(spc, data, dlen)) != 0) {
+ if ((rv = send_with_recon(spc, iov, __arraycount(iov))) != 0) {
unputwait(spc, &rw);
continue;
}
@@ -319,16 +318,19 @@
int cancel, bool haslock)
{
struct handshake_fork rf;
+ const char *myprogname;
struct rsp_hdr rhdr;
struct respwait rw;
sigset_t omask;
size_t bonus;
+ struct iovec iov[2];
int rv;
if (type == HANDSHAKE_FORK) {
bonus = sizeof(rf);
} else {
- bonus = strlen(getprogname())+1;
+ myprogname = getprogname();
+ bonus = strlen(myprogname)+1;
}
/* performs server handshake */
@@ -337,19 +339,21 @@
rhdr.rsp_type = RUMPSP_HANDSHAKE;
rhdr.rsp_handshake = type;
+ IOVPUT(iov[0], rhdr);
+
pthread_sigmask(SIG_SETMASK, &fullset, &omask);
if (haslock)
putwait_locked(spc, &rw, &rhdr);
else
putwait(spc, &rw, &rhdr);
- rv = dosend(spc, &rhdr, sizeof(rhdr));
if (type == HANDSHAKE_FORK) {
memcpy(rf.rf_auth, data, sizeof(rf.rf_auth)); /* uh, why? */
rf.rf_cancel = cancel;
- rv = send_with_recon(spc, &rf, sizeof(rf));
+ IOVPUT(iov[1], rf);
} else {
- rv = dosend(spc, getprogname(), strlen(getprogname())+1);
+ IOVPUT_WITHSIZE(iov[1], __UNCONST(getprogname()), bonus);
}
+ rv = send_with_recon(spc, iov, __arraycount(iov));
if (rv || cancel) {
if (haslock)
unputwait_locked(spc, &rw);
@@ -377,6 +381,7 @@
{
struct rsp_hdr rhdr;
struct respwait rw;
+ struct iovec iov[1];
int rv;
rhdr.rsp_len = sizeof(rhdr);
@@ -384,9 +389,11 @@
rhdr.rsp_type = RUMPSP_PREFORK;
rhdr.rsp_error = 0;
+ IOVPUT(iov[0], rhdr);
+
do {
putwait(spc, &rw, &rhdr);
- rv = send_with_recon(spc, &rhdr, sizeof(rhdr));
+ rv = send_with_recon(spc, iov, __arraycount(iov));
if (rv != 0) {
unputwait(spc, &rw);
continue;
@@ -430,6 +437,7 @@
int wantstr)
{
struct rsp_hdr rhdr;
+ struct iovec iov[2];
if (wantstr)
dlen = MIN(dlen, strlen(data)+1);
@@ -440,10 +448,12 @@
rhdr.rsp_type = RUMPSP_COPYIN;
rhdr.rsp_sysnum = 0;
+ IOVPUT(iov[0], rhdr);
+ IOVPUT_WITHSIZE(iov[1], data, dlen);
+
if (resp_sendlock(spc) != 0)
return;
- (void)dosend(spc, &rhdr, sizeof(rhdr));
- (void)dosend(spc, data, dlen);
+ (void)SENDIOV(spc, iov);
sendunlock(spc);
}
@@ -451,6 +461,7 @@
send_anonmmap_resp(struct spclient *spc, uint64_t reqno, void *addr)
{
struct rsp_hdr rhdr;
+ struct iovec iov[2];
rhdr.rsp_len = sizeof(rhdr) + sizeof(addr);
rhdr.rsp_reqno = reqno;
@@ -458,10 +469,12 @@
rhdr.rsp_type = RUMPSP_ANONMMAP;
rhdr.rsp_sysnum = 0;
+ IOVPUT(iov[0], rhdr);
+ IOVPUT(iov[1], addr);
+
if (resp_sendlock(spc) != 0)
return;
- (void)dosend(spc, &rhdr, sizeof(rhdr));
- (void)dosend(spc, &addr, sizeof(addr));
+ (void)SENDIOV(spc, iov);
sendunlock(spc);
}
@@ -775,7 +788,7 @@
FINDSYM(fcntl)
FINDSYM(poll)
FINDSYM(read)
- FINDSYM(sendto)
+ FINDSYM(sendmsg)
FINDSYM(setsockopt)
FINDSYM(dup)
FINDSYM(kqueue)
Index: src/lib/librumpuser/rumpuser_sp.c
diff -u src/lib/librumpuser/rumpuser_sp.c:1.44 src/lib/librumpuser/rumpuser_sp.c:1.45
--- src/lib/librumpuser/rumpuser_sp.c:1.44 Tue Mar 8 12:39:28 2011
+++ src/lib/librumpuser/rumpuser_sp.c Tue Mar 8 15:34:37 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: rumpuser_sp.c,v 1.44 2011/03/08 12:39:28 pooka Exp $ */
+/* $NetBSD: rumpuser_sp.c,v 1.45 2011/03/08 15:34:37 pooka Exp $ */
/*
* Copyright (c) 2010, 2011 Antti Kantee. All Rights Reserved.
@@ -35,7 +35,7 @@
*/
#include <sys/cdefs.h>
-__RCSID("$NetBSD: rumpuser_sp.c,v 1.44 2011/03/08 12:39:28 pooka Exp $");
+__RCSID("$NetBSD: rumpuser_sp.c,v 1.45 2011/03/08 15:34:37 pooka Exp $");
#include <sys/types.h>
#include <sys/atomic.h>
@@ -238,10 +238,16 @@
return nw;
}
+/*
+ * XXX: we send responses with "blocking" I/O. This is not
+ * ok for the main thread. XXXFIXME
+ */
+
static void
send_error_resp(struct spclient *spc, uint64_t reqno, int error)
{
struct rsp_hdr rhdr;
+ struct iovec iov[1];
rhdr.rsp_len = sizeof(rhdr);
rhdr.rsp_reqno = reqno;
@@ -249,8 +255,10 @@
rhdr.rsp_type = 0;
rhdr.rsp_error = error;
+ IOVPUT(iov[0], rhdr);
+
sendlock(spc);
- (void)dosend(spc, &rhdr, sizeof(rhdr));
+ (void)SENDIOV(spc, iov);
sendunlock(spc);
}
@@ -258,6 +266,7 @@
send_handshake_resp(struct spclient *spc, uint64_t reqno, int error)
{
struct rsp_hdr rhdr;
+ struct iovec iov[2];
int rv;
rhdr.rsp_len = sizeof(rhdr) + sizeof(error);
@@ -266,9 +275,11 @@
rhdr.rsp_type = RUMPSP_HANDSHAKE;
rhdr.rsp_error = 0;
+ IOVPUT(iov[0], rhdr);
+ IOVPUT(iov[1], error);
+
sendlock(spc);
- rv = dosend(spc, &rhdr, sizeof(rhdr));
- rv = dosend(spc, &error, sizeof(error));
+ rv = SENDIOV(spc, iov);
sendunlock(spc);
return rv;
@@ -280,6 +291,7 @@
{
struct rsp_hdr rhdr;
struct rsp_sysresp sysresp;
+ struct iovec iov[2];
int rv;
rhdr.rsp_len = sizeof(rhdr) + sizeof(sysresp);
@@ -291,9 +303,11 @@
sysresp.rsys_error = error;
memcpy(sysresp.rsys_retval, retval, sizeof(sysresp.rsys_retval));
+ IOVPUT(iov[0], rhdr);
+ IOVPUT(iov[1], sysresp);
+
sendlock(spc);
- rv = dosend(spc, &rhdr, sizeof(rhdr));
- rv = dosend(spc, &sysresp, sizeof(sysresp));
+ rv = SENDIOV(spc, iov);
sendunlock(spc);
return rv;
@@ -303,6 +317,7 @@
send_prefork_resp(struct spclient *spc, uint64_t reqno, uint32_t *auth)
{
struct rsp_hdr rhdr;
+ struct iovec iov[2];
int rv;
rhdr.rsp_len = sizeof(rhdr) + AUTHLEN*sizeof(*auth);
@@ -311,9 +326,11 @@
rhdr.rsp_type = RUMPSP_PREFORK;
rhdr.rsp_sysnum = 0;
+ IOVPUT(iov[0], rhdr);
+ IOVPUT_WITHSIZE(iov[1], auth, AUTHLEN*sizeof(*auth));
+
sendlock(spc);
- rv = dosend(spc, &rhdr, sizeof(rhdr));
- rv = dosend(spc, auth, AUTHLEN*sizeof(*auth));
+ rv = SENDIOV(spc, iov);
sendunlock(spc);
return rv;
@@ -326,6 +343,7 @@
struct rsp_hdr rhdr;
struct rsp_copydata copydata;
struct respwait rw;
+ struct iovec iov[2];
int rv;
DPRINTF(("copyin_req: %zu bytes from %p\n", *dlen, remaddr));
@@ -341,9 +359,11 @@
copydata.rcp_addr = __UNCONST(remaddr);
copydata.rcp_len = *dlen;
+ IOVPUT(iov[0], rhdr);
+ IOVPUT(iov[1], copydata);
+
putwait(spc, &rw, &rhdr);
- rv = dosend(spc, &rhdr, sizeof(rhdr));
- rv = dosend(spc, ©data, sizeof(copydata));
+ rv = SENDIOV(spc, iov);
if (rv) {
unputwait(spc, &rw);
return rv;
@@ -367,6 +387,7 @@
{
struct rsp_hdr rhdr;
struct rsp_copydata copydata;
+ struct iovec iov[3];
int rv;
DPRINTF(("copyout_req (async): %zu bytes to %p\n", dlen, remaddr));
@@ -380,10 +401,12 @@
copydata.rcp_addr = __UNCONST(remaddr);
copydata.rcp_len = dlen;
+ IOVPUT(iov[0], rhdr);
+ IOVPUT(iov[1], copydata);
+ IOVPUT_WITHSIZE(iov[2], __UNCONST(data), dlen);
+
sendlock(spc);
- rv = dosend(spc, &rhdr, sizeof(rhdr));
- rv = dosend(spc, ©data, sizeof(copydata));
- rv = dosend(spc, data, dlen);
+ rv = SENDIOV(spc, iov);
sendunlock(spc);
return rv;
@@ -394,6 +417,7 @@
{
struct rsp_hdr rhdr;
struct respwait rw;
+ struct iovec iov[2];
int rv;
DPRINTF(("anonmmap_req: %zu bytes\n", howmuch));
@@ -403,9 +427,11 @@
rhdr.rsp_type = RUMPSP_ANONMMAP;
rhdr.rsp_sysnum = 0;
+ IOVPUT(iov[0], rhdr);
+ IOVPUT(iov[1], howmuch);
+
putwait(spc, &rw, &rhdr);
- rv = dosend(spc, &rhdr, sizeof(rhdr));
- rv = dosend(spc, &howmuch, sizeof(howmuch));
+ rv = SENDIOV(spc, iov);
if (rv) {
unputwait(spc, &rw);
return rv;
@@ -424,6 +450,7 @@
send_raise_req(struct spclient *spc, int signo)
{
struct rsp_hdr rhdr;
+ struct iovec iov[1];
int rv;
rhdr.rsp_len = sizeof(rhdr);
@@ -431,8 +458,10 @@
rhdr.rsp_type = RUMPSP_RAISE;
rhdr.rsp_signo = signo;
+ IOVPUT(iov[0], rhdr);
+
sendlock(spc);
- rv = dosend(spc, &rhdr, sizeof(rhdr));
+ rv = SENDIOV(spc, iov);
sendunlock(spc);
return rv;
Index: src/lib/librumpuser/sp_common.c
diff -u src/lib/librumpuser/sp_common.c:1.30 src/lib/librumpuser/sp_common.c:1.31
--- src/lib/librumpuser/sp_common.c:1.30 Tue Mar 8 12:39:28 2011
+++ src/lib/librumpuser/sp_common.c Tue Mar 8 15:34:37 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: sp_common.c,v 1.30 2011/03/08 12:39:28 pooka Exp $ */
+/* $NetBSD: sp_common.c,v 1.31 2011/03/08 15:34:37 pooka Exp $ */
/*
* Copyright (c) 2010, 2011 Antti Kantee. All Rights Reserved.
@@ -74,10 +74,14 @@
#ifndef HOSTOPS
#define host_poll poll
#define host_read read
-#define host_sendto sendto
+#define host_sendmsg sendmsg
#define host_setsockopt setsockopt
#endif
+#define IOVPUT(_io_, _b_) _io_.iov_base = &_b_; _io_.iov_len = sizeof(_b_);
+#define IOVPUT_WITHSIZE(_io_, _b_, _l_) _io_.iov_base = _b_; _io_.iov_len = _l_;
+#define SENDIOV(_spc_, _iov_) dosend(_spc_, _iov_, __arraycount(_iov_))
+
/*
* Bah, I hate writing on-off-wire conversions in C
*/
@@ -254,18 +258,20 @@
}
static int
-dosend(struct spclient *spc, const void *data, size_t dlen)
+dosend(struct spclient *spc, struct iovec *iov, size_t iovlen)
{
+ struct msghdr msg;
struct pollfd pfd;
- const uint8_t *sdata = data;
- ssize_t n;
- size_t sent;
+ ssize_t n = 0;
int fd = spc->spc_fd;
pfd.fd = fd;
pfd.events = POLLOUT;
- for (sent = 0, n = 0; sent < dlen; ) {
+ memset(&msg, 0, sizeof(msg));
+
+ for (;;) {
+ /* not first round? poll */
if (n) {
if (host_poll(&pfd, 1, INFTIM) == -1) {
if (errno == EINTR)
@@ -274,8 +280,9 @@
}
}
- n = host_sendto(fd, sdata + sent, dlen - sent,
- MSG_NOSIGNAL, NULL, 0);
+ msg.msg_iov = iov;
+ msg.msg_iovlen = iovlen;
+ n = host_sendmsg(fd, &msg, MSG_NOSIGNAL);
if (n == -1) {
if (errno == EPIPE)
return ENOTCONN;
@@ -286,7 +293,21 @@
if (n == 0) {
return ENOTCONN;
}
- sent += n;
+
+ /* ok, need to adjust iovec for potential next round */
+ while (n >= (ssize_t)iov[0].iov_len && iovlen) {
+ n -= iov[0].iov_len;
+ iov++;
+ iovlen--;
+ }
+
+ if (iovlen == 0) {
+ _DIAGASSERT(n == 0);
+ break;
+ } else {
+ iov[0].iov_base = (uint8_t *)iov[0].iov_base + n;
+ iov[0].iov_len -= n;
+ }
}
return 0;
