sudheerv opened a new issue #6946:
URL: https://github.com/apache/trafficserver/issues/6946
We are still seeing global variable corruption in prod and digging deeper it
looks like it may be somehow related to UserArg table corruption.
The crash pointed to a corrupted `http_global_hooks` global and upon
examining other globals around it, turns out all of those are corrupted as well
all the way up to `UserArgTable`. I suspect there may have been an array
boundary underflow as I see garbage entries (see below).
We are still using the old TxnArg API in all our plugins, but, since the
CPPAPI (part of ATS Core) has been modified to use the new UserArg API, we've a
scenario where we are partly using the new API and old API. Talking to @zwoop ,
this is _designed_ to still work in 9.0. so, will add some boundary check
asserts into UserArgTable writes to determine which of our plugins is
corrupting them.
```
Program terminated with signal 11, Segmentation fault.
#0 head (this=0x2541302541302589) at traffic_server/InkAPI.cc:1339
1339 traffic_server/InkAPI.cc: No such file or directory.
(gdb) bt
#0 head (this=0x2541302541302589) at traffic_server/InkAPI.cc:1339
#1 init (id=TS_HTTP_READ_RESPONSE_HDR_HOOK, feature_hooks=<optimized out>,
this=0x2ae24e773ae8) at traffic_server/InkAPI.cc:1427
#2 HttpHookState::init (this=0x2ae24e773ae0,
id=TS_HTTP_READ_RESPONSE_HDR_HOOK, global=<optimized out>, ssn=0x2ae20d377600,
txn=0x2ae24e773b50) at traffic_server/InkAPI.cc:1370
#3 0x000000000054f3cd in HttpSM::do_api_callout_internal
(this=0x2ae24e771920) at HttpSM.cc:5332
#4 0x000000000055d605 in HttpSM::do_api_callout
(this=this@entry=0x2ae24e771920) at HttpSM.cc:365
#5 0x000000000054fcab in HttpSM::state_read_server_response_header
(this=0x2ae24e771920, event=100, data=0x2ae2ae30c2c0) at HttpSM.cc:2006
#6 0x0000000000551f68 in HttpSM::main_handler (this=0x2ae24e771920,
event=100, data=0x2ae2ae30c2c0) at HttpSM.cc:2710
#7 0x0000000000776a03 in handleEvent (data=0x2ae2ae30c2c0, event=100,
this=0x2ae24e771920) at
/home/svinukon/Traffic/ATS/ats9/ats-core_trunk/ats9/src/iocore/eventsystem/I_Continuation.h:190
#8 read_signal_and_update (event=100, vc=0x2ae2ae30c0e0) at
UnixNetVConnection.cc:83
#9 0x000000000077d1ee in read_from_net (nh=0x2ae18a009dd0,
vc=0x2ae2ae30c0e0, thread=0x2ae18a006000) at UnixNetVConnection.cc:314
#10 0x0000000000762ee8 in NetHandler::process_ready_list
(this=this@entry=0x2ae18a009dd0) at UnixNet.cc:412
#11 0x00000000007631dd in NetHandler::waitForActivity (this=0x2ae18a009dd0,
timeout=<optimized out>) at UnixNet.cc:547
#12 0x00000000007c6eba in EThread::execute_regular
(this=this@entry=0x2ae18a006000) at UnixEThread.cc:266
#13 0x00000000007c7182 in EThread::execute (this=0x2ae18a006000) at
UnixEThread.cc:327
#14 0x00000000007c5529 in spawn_thread_internal (a=0x2ae188569e40) at
Thread.cc:92
#15 0x00002ae18652add5 in start_thread () from /lib64/libpthread.so.0
#16 0x00002ae1872dbead in clone () from /lib64/libc.so.6
(gdb) p hostdb_max_iobuf_index
$1 = 808592688
(gdb) p sizeof9hostdb_max_iobuf_index)
p sizeof(No symbol "sizeof9hostdb_max_iobuf_index" in current context.
(gdb) p sizeof(hostdb_max_iobuf_index)
$2 = 4
(gdb) p (int)0x2541302541302589
$3 = 1093674377
(gdb) p http_global_hooks
$4 = (HttpAPIHooks *) 0x2541302541302541
(gdb) p ssl_hooks
$5 = (SslAPIHooks *) 0x3025413025413025
(gdb) p lifecycle_hoooks
No symbol "lifecycle_hoooks" in current context.
(gdb) p lifecycle_hooks
$6 = (LifecycleAPIHooks *) 0x4130254130254130
(gdb) p TS_HTTP_LEN_PUSH
$7 = 807747888
(gdb) p TS_HTTP_LEN_CONNECT
$8 = 807747888
(gdb) p TS_HTTP_METHOD_CONNECT
$9 = 0x2541302541302541 <Address 0x2541302541302541 out of bounds>
(gdb) p TS_HTTP_VALUE_BYTES
$10 = 0x3025413025413025 <Address 0x3025413025413025 out of bounds>
(gdb) p TS_MIME_LEN_ACCEPT
$11 = 1093674305
(gdb) p MIME_LEN_ACCEPT
$12 = 6
(gdb) p TS_MIME_FIELD_ACCEPT
$13 = 0x4130254130254130 <Address 0x4130254130254130 out of bounds>
(gdb) p TS_URL_SCHEME_FILE
$14 = 0x4130254130254130 <Address 0x4130254130254130 out of bounds>
(gdb) p TS_USER_ARGS_COUNT
$15 = TS_USER_ARGS_COUNT
(gdb) p TS_USER_ARGS_GLB
$16 = TS_USER_ARGS_GLB
(gdb) p UserArgTable
$17 = {{{type = 1818585446, name = {static npos = 18446744073709551615,
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> =
{<No data fields>}, <No data fields>}, _M_p = 0x676e6e6576303425 <Address
0x676e6e6576303425 out of bounds>}, _M_string_length = 7885630523655417714, {
_M_local_buf = "%0A%0A%23contrac", _M_allocated_capacity =
3613365951073955877}}, description = {static npos = 18446744073709551615,
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> =
{<No data fields>}, <No data fields>},
_M_p = 0x6833322530322574 <Address 0x6833322530322574 out of
bounds>}, _M_string_length = 8241978093345726821, {_M_local_buf =
"e%20%23%20dynami", _M_allocated_capacity = 2680541338519348581}}}, {type =
841315171, name = {static npos = 18446744073709551615,
_M_dataplus = {<std::allocator<char>> =
{<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p
= 0x6c657665646d7263 <Address 0x6c657665646d7263 out of bounds>},
_M_string_length = 2756537384318627951, {_M_local_buf = "includeFollowed=",
_M_allocated_capacity = 5072571010795138665}},
description = {static npos = 18446744073709551615, _M_dataplus =
{<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data
fields>}, <No data fields>}, _M_p = 0x6572702665757274 <Address
0x6572702665757274 out of bounds>}, _M_string_length = 6365935209299863910,
{_M_local_buf = "XXXXXXXX&q=hasht",
_M_allocated_capacity = 6365935209750747224}}}, {type = 7563105,
name = {static npos = 18446744073709551615, _M_dataplus =
{<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data
fields>}, <No data fields>}, _M_p = 0xab0868 <UserArgTable+168> ""},
_M_string_length = 0, {
_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity =
0}}, description = {static npos = 18446744073709551615, _M_dataplus =
{<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data
fields>}, <No data fields>}, _M_p = 0xab0888 <UserArgTable+200> ""},
_M_string_length = 0, {
_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity =
0}}}, {type = TS_USER_ARGS_TXN, name = {static npos = 18446744073709551615,
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> =
{<No data fields>}, <No data fields>}, _M_p = 0xab08b0 <UserArgTable+240> ""},
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>,
_M_allocated_capacity = 0}}, description = {static npos = 18446744073709551615,
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> =
{<No data fields>}, <No data fields>}, _M_p = 0xab08d0 <UserArgTable+272> ""},
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>,
_M_allocated_capacity = 0}}}, {type = TS_USER_ARGS_TXN, name = {static npos =
18446744073709551615, _M_dataplus = {<std::allocator<char>> =
{<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
_M_p = 0xab08f8 <UserArgTable+312> ""}, _M_string_length = 0,
{_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}},
description = {static npos = 18446744073709551615, _M_dataplus =
{<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data
fields>}, <No data fields>},
_M_p = 0xab0918 <UserArgTable+344> ""}, _M_string_length = 0,
{_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}}, {type
= TS_USER_ARGS_TXN, name = {static npos = 18446744073709551615,
_M_dataplus = {<std::allocator<char>> =
{<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p
= 0xab0940 <UserArgTable+384> ""}, _M_string_length = 0, {_M_local_buf = '\000'
<repeats 15 times>, _M_allocated_capacity = 0}}, description = {static npos =
18446744073709551615,
_M_dataplus = {<std::allocator<char>> =
{<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p
= 0xab0960 <UserArgTable+416> ""}, _M_string_length = 0, {_M_local_buf = '\000'
<repeats 15 times>, _M_allocated_capacity = 0}}}, {type = TS_USER_ARGS_TXN,
name = {
static npos = 18446744073709551615, _M_dataplus =
{<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data
fields>}, <No data fields>}, _M_p = 0xab0988 <UserArgTable+456> ""},
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>,
_M_allocated_capacity = 0}}, description = {
static npos = 18446744073709551615, _M_dataplus =
{<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data
fields>}, <No data fields>}, _M_p = 0xab09a8 <UserArgTable+488> ""},
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>,
_M_allocated_capacity = 0}}}, {type = TS_USER_ARGS_TXN,
name = {static npos = 18446744073709551615, _M_dataplus =
{<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data
fields>}, <No data fields>}, _M_p = 0xab09d0 <UserArgTable+528> ""},
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>,
_M_allocated_capacity = 0}}, description = {
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
