On Mon, Jun 17, 2019 at 09:43:00PM +0200, Claudio Jeker wrote:
> I noticed that by default the send and recv socket buffers for
> socketpair(2) is rather low (4k IIRC). The result is a fairly inefficent
> write/read behaviour on the imsg sockets. Increasing SO_SNDBUF and
> SO_RCVBUF seems to help increase the data sent and received per syscall.
>
> Another option would be to make the default socketbuffer watermarks for
> socketpair(2) a bit less limited. Then all imsg users would benefit at the
> same time.
Here is an updated version of this diff. It does not blow up the socket
buffer to the maximum size possible but instead uses 4 times the
READ_BUF_SIZE (or 256kB). This seems to be enough to make both the
sender and receiver run as efficent as possible.
Additionally tune the session messgage low and high value. msgbuf_write()
can send up to 1000 messages in one go. So a low water mark of 50 is to
low to be efficent. In my testing this makes bgpd a bit quicker at
sending updates out.
--
:wq Claudio
Index: bgpd.c
===================================================================
RCS file: /cvs/src/usr.sbin/bgpd/bgpd.c,v
retrieving revision 1.219
diff -u -p -r1.219 bgpd.c
--- bgpd.c 29 May 2019 08:48:00 -0000 1.219
+++ bgpd.c 10 Jul 2019 19:36:51 -0000
@@ -46,6 +46,7 @@ int send_filterset(struct imsgbuf *, st
int reconfigure(char *, struct bgpd_config *);
int dispatch_imsg(struct imsgbuf *, int, struct bgpd_config *);
int control_setup(struct bgpd_config *);
+static void getsockpair(int [2]);
int imsg_send_sockets(struct imsgbuf *, struct imsgbuf *);
int cflags;
@@ -203,12 +204,8 @@ main(int argc, char *argv[])
log_info("startup");
- if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC | SOCK_NONBLOCK,
- PF_UNSPEC, pipe_m2s) == -1)
- fatal("socketpair");
- if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC | SOCK_NONBLOCK,
- PF_UNSPEC, pipe_m2r) == -1)
- fatal("socketpair");
+ getsockpair(pipe_m2s);
+ getsockpair(pipe_m2r);
/* fork children */
rde_pid = start_child(PROC_RDE, saved_argv0, pipe_m2r[1], debug,
@@ -1073,18 +1070,51 @@ handle_pollfd(struct pollfd *pfd, struct
return (0);
}
+static void
+getsockpair(int pipe[2])
+{
+ int bsize, i;
+
+ if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC | SOCK_NONBLOCK,
+ PF_UNSPEC, pipe) == -1)
+ fatal("socketpair");
+
+ for (i = 0; i < 2; i++) {
+ for (bsize = MAX_SOCK_BUF; bsize >= 16 * 1024; bsize /= 2) {
+ if (setsockopt(pipe[i], SOL_SOCKET, SO_RCVBUF,
+ &bsize, sizeof(bsize)) == -1) {
+ if (errno != ENOBUFS)
+ fatal("setsockopt(SO_RCVBUF, %d)",
+ bsize);
+ log_warn("setsockopt(SO_RCVBUF, %d)", bsize);
+ continue;
+ }
+ break;
+ }
+ }
+ for (i = 0; i < 2; i++) {
+ for (bsize = MAX_SOCK_BUF; bsize >= 16 * 1024; bsize /= 2) {
+ if (setsockopt(pipe[i], SOL_SOCKET, SO_SNDBUF,
+ &bsize, sizeof(bsize)) == -1) {
+ if (errno != ENOBUFS)
+ fatal("setsockopt(SO_SNDBUF, %d)",
+ bsize);
+ log_warn("setsockopt(SO_SNDBUF, %d)", bsize);
+ continue;
+ }
+ break;
+ }
+ }
+}
+
int
imsg_send_sockets(struct imsgbuf *se, struct imsgbuf *rde)
{
int pipe_s2r[2];
int pipe_s2r_ctl[2];
- if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC | SOCK_NONBLOCK,
- PF_UNSPEC, pipe_s2r) == -1)
- return (-1);
- if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC | SOCK_NONBLOCK,
- PF_UNSPEC, pipe_s2r_ctl) == -1)
- return (-1);
+ getsockpair(pipe_s2r);
+ getsockpair(pipe_s2r_ctl);
if (imsg_compose(se, IMSG_SOCKET_CONN, 0, 0, pipe_s2r[0],
NULL, 0) == -1)
Index: bgpd.h
===================================================================
RCS file: /cvs/src/usr.sbin/bgpd/bgpd.h,v
retrieving revision 1.388
diff -u -p -r1.388 bgpd.h
--- bgpd.h 22 Jun 2019 05:36:40 -0000 1.388
+++ bgpd.h 10 Jul 2019 19:36:02 -0000
@@ -48,6 +48,7 @@
#define MAX_PKTSIZE 4096
#define MIN_HOLDTIME 3
#define READ_BUF_SIZE 65535
+#define MAX_SOCK_BUF (4 * READ_BUF_SIZE)
#define RT_BUF_SIZE 16384
#define MAX_RTSOCK_BUF (2 * 1024 * 1024)
#define MAX_COMM_MATCH 3
@@ -110,8 +111,8 @@
* IMSG_XON message will be sent and the RDE will produce more messages again.
*/
#define RDE_RUNNER_ROUNDS 100
-#define SESS_MSG_HIGH_MARK 300
-#define SESS_MSG_LOW_MARK 50
+#define SESS_MSG_HIGH_MARK 2000
+#define SESS_MSG_LOW_MARK 500
#define CTL_MSG_HIGH_MARK 500
#define CTL_MSG_LOW_MARK 100
