bdgranger commented on issue #7581:
URL: https://github.com/apache/trafficserver/issues/7581#issuecomment-800536478
An example of the Http1ClientSession::release crash:
```c++
(gdb) bt
#0 0x0000000000718022 in Mutex_trylock (t=0x2b7d8f341010, m=<optimized out>)
at /usr/src/debug/trafficserver-8.1.2/iocore/eventsystem/I_Lock.h:287
#1 MutexTryLock (t=0x2b7d8f341010, am=..., this=0x2b7d95ac7a60)
at /usr/src/debug/trafficserver-8.1.2/iocore/eventsystem/I_Lock.h:555
#2 _close_vc (total_idle_count=<synthetic pointer>,
total_idle_time=<synthetic pointer>, closed=<synthetic pointer>,
handle_event=<synthetic pointer>, now=1615727914286925795, vc=<optimized
out>, this=0x2b7d8f344cf0) at UnixNet.cc:648
#3 manage_keep_alive_queue (this=0x2b7d8f344cf0) at UnixNet.cc:626
#4 NetHandler::add_to_keep_alive_queue (this=0x2b7d8f344cf0, vc=<optimized
out>) at UnixNet.cc:698
#5 0x00000000004e63c5 in Http1ClientSession::release (this=0x2b7daa9663c0,
trans=0x2b7daa9666f0) at Http1ClientSession.cc:485
#6 0x00000000006c3c5d in ProxyClientTransaction::release
(this=0x2b7daa9666f0, r=<optimized out>)
at ProxyClientTransaction.cc:71
#7 0x00000000004f6540 in HttpSM::tunnel_handler_ua (this=0x2b866faebd80,
event=103, c=0x2b866faeccc0) at HttpSM.cc:3274
#8 0x000000000055bd61 in HttpTunnel::consumer_handler
(this=this@entry=0x2b866faecc80, event=103, c=0x2b866faeccc0)
at HttpTunnel.cc:1389
#9 0x000000000055c214 in HttpTunnel::main_handler (this=0x2b866faecc80,
event=<optimized out>, data=<optimized out>)
at HttpTunnel.cc:1678
#10 0x000000000048c8f2 in handleEvent (data=0x2b81106a4528, event=<optimized
out>, this=<optimized out>)
at
/usr/src/debug/trafficserver-8.1.2/iocore/eventsystem/I_Continuation.h:160
#11 write_signal_and_update (vc=0x2b81106a4320, event=<optimized out>) at
UnixNetVConnection.cc:114
#12 write_signal_done (event=<optimized out>, nh=0x2b7d8f344cf0,
vc=0x2b81106a4320) at UnixNetVConnection.cc:156
#13 0x000000000072ca34 in write_to_net_io (nh=nh@entry=0x2b7d8f344cf0,
vc=0x2b81106a4320, thread=<optimized out>)
at UnixNetVConnection.cc:523
#14 0x000000000072ccd8 in write_to_net (nh=nh@entry=0x2b7d8f344cf0,
vc=vc@entry=0x2b81106a4320, thread=<optimized out>)
at UnixNetVConnection.cc:368
#15 0x00000000007154cf in NetHandler::process_ready_list
(this=this@entry=0x2b7d8f344cf0) at UnixNet.cc:416
#16 0x000000000071578d in NetHandler::waitForActivity (this=0x2b7d8f344cf0,
timeout=<optimized out>) at UnixNet.cc:532
#17 0x0000000000761d09 in EThread::execute_regular (this=0x2b7d8f341010) at
UnixEThread.cc:278
#18 0x000000000076030a in spawn_thread_internal (a=0x17b3b40) at Thread.cc:85
#19 0x00002b7d8b976e65 in start_thread () from /lib64/libpthread.so.0
#20 0x00002b7d8c6ac88d in clone () from /lib64/libc.so.6
```
bt full:
```c++
(gdb) bt full
#0 0x0000000000718022 in Mutex_trylock (t=0x2b7d8f341010, m=<optimized out>)
at /usr/src/debug/trafficserver-8.1.2/iocore/eventsystem/I_Lock.h:287
No locals.
#1 MutexTryLock (t=0x2b7d8f341010, am=..., this=0x2b7d95ac7a60)
at /usr/src/debug/trafficserver-8.1.2/iocore/eventsystem/I_Lock.h:555
No locals.
#2 _close_vc (total_idle_count=<synthetic pointer>,
total_idle_time=<synthetic pointer>, closed=<synthetic pointer>,
handle_event=<synthetic pointer>, now=1615727914286925795, vc=<optimized
out>, this=0x2b7d8f344cf0) at UnixNet.cc:648
lock = {m = {m_ptr = 0x0}, lock_acquired = false}
diff = <optimized out>
#3 manage_keep_alive_queue (this=0x2b7d8f344cf0) at UnixNet.cc:626
vc = 0x2b80f4393400
total_connections_in = <optimized out>
vc_next = 0x0
closed = 0
total_idle_time = 0
handle_event = 0
total_idle_count = 0
#4 NetHandler::add_to_keep_alive_queue (this=0x2b7d8f344cf0, vc=<optimized
out>) at UnixNet.cc:698
__FUNCTION__ = "add_to_keep_alive_queue"
#5 0x00000000004e63c5 in Http1ClientSession::release (this=0x2b7daa9663c0,
trans=0x2b7daa9666f0) at Http1ClientSession.cc:485
more_to_read = false
#6 0x00000000006c3c5d in ProxyClientTransaction::release
(this=0x2b7daa9666f0, r=<optimized out>)
at ProxyClientTransaction.cc:71
__FUNCTION__ = "release"
#7 0x00000000004f6540 in HttpSM::tunnel_handler_ua (this=0x2b866faebd80,
event=103, c=0x2b866faeccc0) at HttpSM.cc:3274
close_connection = false
p = <optimized out>
selfc = <optimized out>
__FUNCTION__ = "tunnel_handler_ua"
original_source = <optimized out>
#8 0x000000000055bd61 in HttpTunnel::consumer_handler
(this=this@entry=0x2b866faecc80, event=103, c=0x2b866faeccc0)
at HttpTunnel.cc:1389
sm_callback = false
jump_point = <optimized out>
p = 0x2b866faece80
__FUNCTION__ = "consumer_handler"
#9 0x000000000055c214 in HttpTunnel::main_handler (this=0x2b866faecc80,
event=<optimized out>, data=<optimized out>)
at HttpTunnel.cc:1678
p = <optimized out>
c = <optimized out>
sm_callback = false
#10 0x000000000048c8f2 in handleEvent (data=0x2b81106a4528, event=<optimized
out>, this=<optimized out>)
at
/usr/src/debug/trafficserver-8.1.2/iocore/eventsystem/I_Continuation.h:160
No locals.
#11 write_signal_and_update (vc=0x2b81106a4320, event=<optimized out>) at
UnixNetVConnection.cc:114
No locals.
#12 write_signal_done (event=<optimized out>, nh=0x2b7d8f344cf0,
vc=0x2b81106a4320) at UnixNetVConnection.cc:156
No locals.
#13 0x000000000072ca34 in write_to_net_io (nh=nh@entry=0x2b7d8f344cf0,
vc=0x2b81106a4320, thread=<optimized out>)
at UnixNetVConnection.cc:523
wbe_event = <optimized out>
e = <optimized out>
s = 0x2b81106a4520
lock = {m = {m_ptr = 0x2b7de2fcf410}, lock_acquired = true}
r_ip = "79.27.137.59\000\000\000"
buf = @0x2b81106a4548: {mbuf = 0x2b8668177630, entry =
0x2b8668177670}
towrite = 105066
signalled = <optimized out>
needs = -2147483644
total_written = 105066
r = 105066
__FUNCTION__ = "write_to_net_io"
#14 0x000000000072ccd8 in write_to_net (nh=nh@entry=0x2b7d8f344cf0,
vc=vc@entry=0x2b81106a4320, thread=<optimized out>)
at UnixNetVConnection.cc:368
No locals.
#15 0x00000000007154cf in NetHandler::process_ready_list
(this=this@entry=0x2b7d8f344cf0) at UnixNet.cc:416
vc = <optimized out>
#16 0x000000000071578d in NetHandler::waitForActivity (this=0x2b7d8f344cf0,
timeout=<optimized out>) at UnixNet.cc:532
epd = <optimized out>
lock = {m = {m_ptr = 0x2b7d90002d60}, locked_p = true}
pd = 0x2b7d95f12010
vc = <optimized out>
__FUNCTION__ = "waitForActivity"
#17 0x0000000000761d09 in EThread::execute_regular (this=0x2b7d8f341010) at
UnixEThread.cc:278
done_one = <optimized out>
sleep_time = <optimized out>
e = <optimized out>
NegativeQueue = {<DLL<Event, Event::Link_link>> = {head = 0x0}, tail
= 0x0}
next_time = <optimized out>
delta = <optimized out>
prev_metric = <optimized out>
nq_count = 2
ev_count = 2
METRIC_INIT = {_loop_time = {_start = 0, _min = 9223372036854775807,
_max = 0}, _events = {_min = 2147483647, _max = 0,
_total = 0}, _count = 0, _wait = 0}
#18 0x000000000076030a in spawn_thread_internal (a=0x17b3b40) at Thread.cc:85
p = 0x17b3b40
#19 0x00002b7d8b976e65 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#20 0x00002b7d8c6ac88d in clone () from /lib64/libc.so.6
No symbol table info available.
```
HttpSM history (from frame 7):
```c++
(gdb) frame 7
#7 0x00000000004f6540 in HttpSM::tunnel_handler_ua (this=0x2b866faebd80,
event=103, c=0x2b866faeccc0) at HttpSM.cc:3274
3274 ua_txn->release(ua_buffer_reader);
(gdb) print history
$1 = {history = {{location = {file = 0x77d5de "HttpSM.cc",
func = 0x781500 <HttpSM::state_read_client_request_header(int,
void*)::__FUNCTION__> "state_read_client_request_header",
line = 586}, event = 100, reentrancy = 2}, {location = {file =
0x77d5de "HttpSM.cc",
func = 0x781390 <HttpSM::state_api_callback(int,
void*)::__FUNCTION__> "state_api_callback", line = 1290},
event = 60000, reentrancy = 3}, {location = {file = 0x77d5de
"HttpSM.cc",
func = 0x781370 <HttpSM::state_api_callout(int,
void*)::__FUNCTION__> "state_api_callout", line = 1330}, event = 60000,
reentrancy = 3}, {location = {file = 0x77d5de "HttpSM.cc",
func = 0x781390 <HttpSM::state_api_callback(int,
void*)::__FUNCTION__> "state_api_callback", line = 1290},
event = 60000, reentrancy = 4}, {location = {file = 0x77d5de
"HttpSM.cc",
func = 0x781370 <HttpSM::state_api_callout(int,
void*)::__FUNCTION__> "state_api_callout", line = 1330}, event = 60000,
reentrancy = 4}, {location = {file = 0x77d5de "HttpSM.cc",
func = 0x781390 <HttpSM::state_api_callback(int,
void*)::__FUNCTION__> "state_api_callback", line = 1290},
event = 60000, reentrancy = 5}, {location = {file = 0x77d5de
"HttpSM.cc",
func = 0x781370 <HttpSM::state_api_callout(int,
void*)::__FUNCTION__> "state_api_callout", line = 1330}, event = 60000,
reentrancy = 5}, {location = {file = 0x77d5de "HttpSM.cc",
func = 0x781390 <HttpSM::state_api_callback(int,
void*)::__FUNCTION__> "state_api_callback", line = 1290},
event = 60000, reentrancy = 6}, {location = {file = 0x77d5de
"HttpSM.cc",
func = 0x781370 <HttpSM::state_api_callout(int,
void*)::__FUNCTION__> "state_api_callout", line = 1330}, event = 60000,
reentrancy = 6}, {location = {file = 0x77d5de "HttpSM.cc",
func = 0x780918 <HttpSM::set_next_state()::__FUNCTION__>
"set_next_state", line = 7308}, event = 34463, reentrancy = 6},
{location = {file = 0x77d5de "HttpSM.cc", func = 0x780918
<HttpSM::set_next_state()::__FUNCTION__> "set_next_state",
line = 7348}, event = 34463, reentrancy = 6}, {location = {file =
0x78cc60 "HttpCacheSM.cc",
func = 0x78ceb0 <HttpCacheSM::state_cache_open_read(int,
void*)::__FUNCTION__> "state_cache_open_read", line = 116},
event = 1102, reentrancy = -31073}, {location = {file = 0x77d5de
"HttpSM.cc",
func = 0x7811a0 <HttpSM::state_cache_open_read(int,
void*)::__FUNCTION__> "state_cache_open_read", line = 2474},
event = 1102, reentrancy = 7}, {location = {file = 0x77d5de
"HttpSM.cc",
func = 0x781390 <HttpSM::state_api_callback(int,
void*)::__FUNCTION__> "state_api_callback", line = 1290},
event = 60000, reentrancy = 8}, {location = {file = 0x77d5de
"HttpSM.cc",
func = 0x781370 <HttpSM::state_api_callout(int,
void*)::__FUNCTION__> "state_api_callout", line = 1330}, event = 60000,
reentrancy = 8}, {location = {file = 0x77d5de "HttpSM.cc",
func = 0x781390 <HttpSM::state_api_callback(int,
void*)::__FUNCTION__> "state_api_callback", line = 1290},
event = 60000, reentrancy = 9}, {location = {file = 0x77d5de
"HttpSM.cc",
func = 0x781370 <HttpSM::state_api_callout(int,
void*)::__FUNCTION__> "state_api_callout", line = 1330}, event = 60000,
reentrancy = 9}, {location = {file = 0x77d5de "HttpSM.cc",
func = 0x780b90 <HttpSM::setup_cache_read_transfer()::__FUNCTION__>
"setup_cache_read_transfer", line = 6103},
event = 34463, reentrancy = 9}, {location = {file = 0x77d5de
"HttpSM.cc",
func = 0x781020 <HttpSM::tunnel_handler_cache_read(int,
HttpTunnelProducer*)::__FUNCTION__> "tunnel_handler_cache_read",
line = 3325}, event = 102, reentrancy = 6}, {location = {file =
0x77d5de "HttpSM.cc",
func = 0x780db0 <HttpSM::do_cache_lookup_and_read()::__FUNCTION__>
"do_cache_lookup_and_read", line = 4532}, event = 0,
reentrancy = 6}, {location = {file = 0x77d5de "HttpSM.cc",
func = 0x781060 <HttpSM::tunnel_handler_ua(int,
HttpTunnelConsumer*)::__FUNCTION__> "tunnel_handler_ua", line = 3156},
event = 103, reentrancy = 0}, {location = {file = 0x0, func = 0x0,
line = 0}, event = 0,
reentrancy = 0} <repeats 44 times>}, history_pos = 21}
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
