Hi,
Continuing my journey on detecting dead workers I reduced the design to the
minimal, and eliminated the messy file descriptors.
I only have:
- a router socket, with some number of peers
- a monitor socket attached to the router socket

When the monitor detects a disconnect on the router socket:
- do setsockopt(ZMQ_ROUTER_MANDATORY, 1);
- send heartbeat message to every known peer
- if EHOSTUNREACH returned: remove the peer
- do setsockopt(ZMQ_ROUTER_MANDATORY, 0);

What happens: _my app regularly hangs_ in zmq_msg_send(). Roughly 20% of
the invocations. The call never returns, I have to kill the application.

What am I doing wrong??? According to the RFC's router sockets should never
block.
I attached a full stacktrace with info locals and args for each relevant
frame (sorry for the machine readable format).

Env:
libzmq 4.2.1 stable, debug build
Ubuntu 16.04 64bit (the same happens with ubuntu packaged lib)

Regards,
  Gyorgy
<741bt
>&"bt\n"
>~"#0  0x00007f827d342e8d in poll () at ../sysdeps/unix/syscall-template.S:84\n"
>~"#1  0x0000000000524c06 in zmq::signaler_t::wait (this=0x2392138, 
>timeout_=-1) at src/signaler.cpp:228\n"
>~"#2  0x00000000005144d1 in zmq::mailbox_t::recv (this=0x23920d0, 
>cmd_=0x7fff10f6bf80, timeout_=-1) at src/mailbox.cpp:81\n"
>~"#3  0x000000000052a443 in zmq::socket_base_t::process_commands 
>(this=0x2393510, timeout_=-1, throttle_=false) at src/socket_base.cpp:1328\n"
>~"#4  0x0000000000529c1c in zmq::socket_base_t::send (this=0x2393510, 
>msg_=0x7fff10f6c190, flags_=2) at src/socket_base.cpp:1142\n"
>~"#5  0x0000000000500bf1 in s_sendmsg (s_=0x2393510, msg_=0x7fff10f6c190, 
>flags_=2) at src/zmq.cpp:375\n"
>~"#6  0x0000000000501aeb in zmq_msg_send (msg_=0x7fff10f6c190, s_=0x2393510, 
>flags_=2) at src/zmq.cpp:642\n"
>~"#7  0x00000000004c7ede in zmq::socket_t::send (this=0x7fff10f6cba0, 
>flags_=2, msg_=...) at 
>/home/twinsen/Git/gehc-av-broker/import/zmq-20160511/include/zmq.hpp:612\n"
>~"#8  zmq::multipart_t::send (this=0x7fff10f6c2d0, socket=...) at 
>/home/twinsen/Git/gehc-av-broker/import/zmq-20160511/include/zmq_addon.hpp:124\n"
>~"#9  0x00000000004c155c in reqrep::Service::handleWorkerDisconnect 
>(this=0x7fff10f6cb70, event=<optimized out>, fd=<optimized out>) at 
>/home/twinsen/Git/gehc-av-broker/src/main/src/reqrepService.cpp:132\n"
>~"#10 0x00000000004d54de in std::function<void (int, int)>::operator()(int, 
>int) const (__args#1=46, __args#0=512, this=0x7fff10f6c610) at 
>/usr/include/c++/5/functional:2267\n"
>~"#11 broker::Monitor::monitorEvent(int, std::__cxx11::basic_string<char, 
>std::char_traits<char>, std::allocator<char> > const&, std::function<void 
>(int, int)>) (this=0x7fff10f6cb70, socketId=<optimized out>, name=..., 
>callback=...) at 
>/home/twinsen/Git/gehc-av-broker/src/main/src/monitor.cpp:134\n"
>~"#12 0x00000000004d613b in broker::Monitor::<lambda()>::operator() 
>(__closure=0x23c8d30) at 
>/home/twinsen/Git/gehc-av-broker/src/main/src/monitor.cpp:69\n"
>~"#13 std::_Function_handler<void(), 
>broker::Monitor::watchSocket(zmq::socket_t&, const string&, 
>std::function<void(int, int)>, int)::<lambda()> >::_M_invoke(const 
>std::_Any_data &) (__functor=...) at /usr/include/c++/5/functional:1871\n"
>~"#14 0x00000000004d2d84 in std::function<void ()>::operator()() const 
>(this=<optimized out>) at /usr/include/c++/5/functional:2267\n"
>~"#15 broker::Reactor::processPollSet (this=this@entry=0x7fff10f6c8b0) at 
>/home/twinsen/Git/gehc-av-broker/src/main/src/reactor.cpp:210\n"
>~"#16 0x00000000004d2f92 in broker::Reactor::start 
>(this=this@entry=0x7fff10f6c8b0, testMode=testMode@entry=false) at 
>/home/twinsen/Git/gehc-av-broker/src/main/src/reactor.cpp:163\n"
>~"#17 0x00000000004780ae in broker::Broker::start 
>(this=this@entry=0x7fff10f6d78f, reqrepParams=..., pubsubParams=..., 
>httpParams=...) at 
>/home/twinsen/Git/gehc-av-broker/src/main/src/broker.cpp:71\n"
>~"#18 0x0000000000472c86 in main (argc=1, argv=0x7fff10f6d960) at 
>/home/twinsen/Git/gehc-av-broker/src/main/src/main.cpp:30\n"
>741^done


<744frame 1
>&"frame 1\n"
>~"#1  0x0000000000524c06 in zmq::signaler_t::wait (this=0x2392138, 
>timeout_=-1) at src/signaler.cpp:228\n"
>~"228\t    int rc = poll (&pfd, 1, timeout_);\n"
>744^done


<745info locals
>&"info locals\n"
>~"pfd = {fd = 14"
>~", events = 1"
>~", revents = 0"
>~"}"
>~"\n"
>~"rc = 0"
>~"\n"
>745^done


<746info args
>&"info args\n"
>~"this = 0x2392138"
>~"\n"
>~"timeout_ = -1"
>~"\n"
>746^done


<747frame 2
>&"frame 2\n"
>~"#2  0x00000000005144d1 in zmq::mailbox_t::recv (this=0x23920d0, 
>cmd_=0x7fff10f6bf80, timeout_=-1) at src/mailbox.cpp:81\n"
>~"81\t    int rc = signaler.wait (timeout_);\n"
>747^done


<748info locals
>&"info locals\n"
>~"rc = 32767"
>~"\n"
>~"ok = 16"
>~"\n"
>748^done


<749info args
>&"info args\n"
>~"this = 0x23920d0"
>~"\n"
>~"cmd_ = 0x7fff10f6bf80"
>~"\n"
>~"timeout_ = -1"
>~"\n"
>749^done


<750frame 3
>&"frame 3\n"
>~"#3  0x000000000052a443 in zmq::socket_base_t::process_commands 
>(this=0x2393510, timeout_=-1, throttle_=false) at src/socket_base.cpp:1328\n"
>~"1328\t        rc = mailbox->recv (&cmd, timeout_);\n"
>750^done


<751info locals
>&"info locals\n"
>~"rc = 32767"
>~"\n"
>~"cmd = {destination = 0x7fff10f6bfe0"
>~", type = 284606496"
>~", args = {stop = {<No data fields>}"
>~", plug = {<No data fields>}"
>~", own = {object = 0x7fff10f6c060"
>~"}"
>~", attach = {engine = 0x7fff10f6c060"
>~"}"
>~", bind = {pipe = 0x7fff10f6c060"
>~"}"
>~", activate_read = {<No data fields>}"
>~", activate_write = {msgs_read = 140733477994592"
>~"}"
>~", hiccup = {pipe = 0x7fff10f6c060"
>~"}"
>~", pipe_term = {<No data fields>}"
>~", pipe_term_ack = {<No data fields>}"
>~", term_req = {object = 0x7fff10f6c060"
>~"}"
>~", term = {linger = 284606560"
>~"}"
>~", term_ack = {<No data fields>}"
>~", reap = {socket = 0x7fff10f6c060"
>~"}"
>~", reaped = {<No data fields>}"
>~", done = {<No data fields>}"
>~"}"
>~"}"
>~"\n"
>751^done


<752info args
>&"info args\n"
>~"this = 0x2393510"
>~"\n"
>~"timeout_ = -1"
>~"\n"
>~"throttle_ = false"
>~"\n"
>752^done


<753frame 4
>&"frame 4\n"
>~"#4  0x0000000000529c1c in zmq::socket_base_t::send (this=0x2393510, 
>msg_=0x7fff10f6c190, flags_=2) at src/socket_base.cpp:1142\n"
>~"1142\t        if (unlikely (process_commands (timeout, false) != 0)) {\n"
>753^done


<754info locals
>&"info locals\n"
>~"sync_lock = {mutex = 0x0"
>~"}"
>~"\n"
>~"rc = -1"
>~"\n"
>~"timeout = -1"
>~"\n"
>~"end = 0"
>~"\n"
>754^done


<755info args
>&"info args\n"
>~"this = 0x2393510"
>~"\n"
>~"msg_ = 0x7fff10f6c190"
>~"\n"
>~"flags_ = 2"
>~"\n"
>755^done


<756frame 5
>&"frame 5\n"
>~"#5  0x0000000000500bf1 in s_sendmsg (s_=0x2393510, msg_=0x7fff10f6c190, 
>flags_=2) at src/zmq.cpp:375\n"
>~"375\t    int rc = s_->send ((zmq::msg_t *) msg_, flags_);\n"
>756^done


<757info locals
>&"info locals\n"
>~"sz = 5"
>~"\n"
>~"rc = 32767"
>~"\n"
>~"max_msgsz = 37303568"
>~"\n"
>757^done


<758info args
>&"info args\n"
>~"s_ = 0x2393510"
>~"\n"
>~"msg_ = 0x7fff10f6c190"
>~"\n"
>~"flags_ = 2"
>~"\n"
>758^done


<759frame 6
>&"frame 6\n"
>~"#6  0x0000000000501aeb in zmq_msg_send (msg_=0x7fff10f6c190, s_=0x2393510, 
>flags_=2) at src/zmq.cpp:642\n"
>~"642\t    int result = s_sendmsg (s, msg_, flags_);\n"
>759^done


<760info locals
>&"info locals\n"
>~"s = 0x2393510"
>~"\n"
>~"result = 32767"
>~"\n"
>760^done


<761info args
>&"info args\n"
>~"msg_ = 0x7fff10f6c190"
>~"\n"
>~"s_ = 0x2393510"
>~"\n"
>~"flags_ = 2"
>~"\n"
>761^done


<762frame 7
>&"frame 7\n"
>~"#7  0x00000000004c7ede in zmq::socket_t::send (this=0x7fff10f6cba0, 
>flags_=2, msg_=...) at 
>/home/twinsen/Git/gehc-av-broker/import/zmq-20160511/include/zmq.hpp:612\n"
>~"612\t            int nbytes = zmq_msg_send (&(msg_.msg), ptr, flags_);\n"
>762^done


<763info locals
>&"info locals\n"
>~"nbytes = <optimized out>\n"
>763^done


<764info args
>&"info args\n"
>~"this = 0x7fff10f6cba0"
>~"\n"
>~"flags_ = 2"
>~"\n"
>~"msg_ = @0x7fff10f6c190: {msg = {_ = 
>\"\\000\\000\\000\\000\\000\\000\\000\\000\\000\\344<\\230o\\000\\000\\000\\020\\277<\\002\\000\\000\\000\\000x~\\232}\\202\\177\\000\\000\\b\\277<\\002\\000\\000\\000\\000\\242\\005e\\001\\000\\000\\000\\000\\330\\302\\366\\020\\377\\177\\000\\000\\b\\277<\\002\\000\\000\\000\""
>~", p = 0x0"
>~"}"
>~"}"
>~"\n"
>764^done


<765frame 8
>&"frame 8\n"
>~"#8  zmq::multipart_t::send (this=0x7fff10f6c2d0, socket=...) at 
>/home/twinsen/Git/gehc-av-broker/import/zmq-20160511/include/zmq_addon.hpp:124\n"
>~"124\t            if (!socket.send(message, more ? ZMQ_SNDMORE : 0))\n"
>765^done


<766info locals
>&"info locals\n"
>~"message = {msg = {_ = 
>\"\\000\\000\\000\\000\\000\\000\\000\\000\\000\\344<\\230o\\000\\000\\000\\020\\277<\\002\\000\\000\\000\\000x~\\232}\\202\\177\\000\\000\\b\\277<\\002\\000\\000\\000\\000\\242\\005e\\001\\000\\000\\000\\000\\330\\302\\366\\020\\377\\177\\000\\000\\b\\277<\\002\\000\\000\\000\""
>~", p = 0x0"
>~"}"
>~"}"
>~"\n"
>~"more = true"
>~"\n"
>766^done


<767info args
>&"info args\n"
>~"this = 0x7fff10f6c2d0"
>~"\n"
>~"socket = @0x7fff10f6cba0: {ptr = 0x2393510"
>~", ctxptr = 0x238daa0"
>~"}"
>~"\n"
>767^done
_______________________________________________
zeromq-dev mailing list
zeromq-dev@lists.zeromq.org
https://lists.zeromq.org/mailman/listinfo/zeromq-dev

Reply via email to