Hello all,
We have a RHEL 6.4 server with httpd 2.2.32, acting as a front-end for
UID user authentication. Once authenticated, users are passed to the
back-end Windows/Mysql database server on a private network, via
ProxyPass/ProxyPassReverse.
PROBLEM: Pages are occasionally (40-50 times/day) being randomly
truncated, which is always accompanied by a segmentation fault in the httpd
error log:
[Thu Oct 13 14:44:08 2016] [notice] child pid 22716 exit signal
Segmentation fault (11)
A page refresh will often succeed, but occasionally several refreshes
are needed to not get the segfault. One interesting point - Chrome clients
do not have this issue, nor do IE or FF browsers if they are run in
"Private mode". Non-private mode FF and IE browsers can reproduce the issue
regularly.
I forced a core dump of the segfault, and the output of gdb is included
below, but I'm not fluent enough to be able to make sense of it (or use it
effectively), so any advice/help is greatly appreciated.
Thanks, best regards,
-Chuck
Server version: Apache/2.2.31 (Unix)
Server built: Aug 26 2016 13:53:27
Server's Module Magic Number: 20051115:40
Server loaded: APR 1.5.2, APR-Util 1.5.4
Compiled using: APR 1.5.2, APR-Util 1.5.4
Architecture: 64-bit
Server MPM: Prefork
threaded: no
forked: yes (variable process count)
Server compiled with....
-D APACHE_MPM_DIR="server/mpm/prefork"
-D APR_HAS_SENDFILE
-D APR_HAS_MMAP
-D APR_HAVE_IPV6 (IPv4-mapped addresses enabled)
-D APR_USE_SYSVSEM_SERIALIZE
-D APR_USE_PTHREAD_SERIALIZE
-D SINGLE_LISTEN_UNSERIALIZED_ACCEPT
-D APR_HAS_OTHER_CHILD
-D AP_HAVE_RELIABLE_PIPED_LOGS
-D DYNAMIC_MODULE_LIMIT=128
-D HTTPD_ROOT="/apacheSSLDEV2"
-D SUEXEC_BIN="/apacheSSLDEV2/bin/suexec"
-D DEFAULT_PIDLOG="logs/httpd.pid"
-D DEFAULT_SCOREBOARD="logs/apache_runtime_status"
-D DEFAULT_LOCKFILE="logs/accept.lock"
-D DEFAULT_ERRORLOG="logs/error_log"
-D AP_TYPES_CONFIG_FILE="conf/mime.types"
-D SERVER_CONFIG_FILE="conf/httpd.conf"
[root@vlsched ~]# gdb /apacheSSLDEV2/bin/httpd ./apacheSSLDEV2-core
GNU gdb (GDB) Red Hat Enterprise Linux (7.2-60.el6)
Copyright (C) 2010 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html
>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /apacheSSLDEV2/bin/httpd...done.
[New Thread 9796]
Missing separate debuginfo for /apacheSSLDEV2/lib/libaprutil-1.so.0
Try: yum --disablerepo='*' --enablerepo='*-debug*' install
/usr/lib/debug/.build-id/62/b961ed72279546bd42cee2d0c8d7ed0dec4298
Missing separate debuginfo for /apacheSSLDEV2/lib/libexpat.so.0
Try: yum --disablerepo='*' --enablerepo='*-debug*' install
/usr/lib/debug/.build-id/24/bb06c86683c9950e3e54d45ba40c1e786ecab9
Missing separate debuginfo for /apacheSSLDEV2/lib/libapr-1.so.0
Try: yum --disablerepo='*' --enablerepo='*-debug*' install
/usr/lib/debug/.build-id/0f/dae46acd31f1b37a5fe8484aee0792f3a45b4a
Missing separate debuginfo for /apacheSSLDEV2/modules/libopentoken.so
Try: yum --disablerepo='*' --enablerepo='*-debug*' install
/usr/lib/debug/.build-id/37/05fa4039609e067223023de62e7c134b87249a
Missing separate debuginfo for /apacheSSLDEV2/modules/mod_pf.so
Try: yum --disablerepo='*' --enablerepo='*-debug*' install
/usr/lib/debug/.build-id/22/b4b932110efff022a7fb96c3f595941a98e44c
Missing separate debuginfo for /apacheSSLDEV2/modules/mod_headers.so
Try: yum --disablerepo='*' --enablerepo='*-debug*' install
/usr/lib/debug/.build-id/35/b1c2680d26276521c2ed92eab8960a56ac1d87
Missing separate debuginfo for /apacheSSLDEV2/modules/mod_proxy.so
Try: yum --disablerepo='*' --enablerepo='*-debug*' install
/usr/lib/debug/.build-id/19/195f789e0f7d6e07cf5c44e252739bfe593f53
Missing separate debuginfo for /apacheSSLDEV2/modules/mod_proxy_connect.so
Try: yum --disablerepo='*' --enablerepo='*-debug*' install
/usr/lib/debug/.build-id/15/8eff1555e28ab232f30e4f93ac01ca60036b68
Missing separate debuginfo for /apacheSSLDEV2/modules/mod_proxy_ftp.so
Try: yum --disablerepo='*' --enablerepo='*-debug*' install
/usr/lib/debug/.build-id/b4/cfd85c4493265f65af9f6bd1f0074d25bf0459
Missing separate debuginfo for /apacheSSLDEV2/modules/mod_proxy_http.so
Try: yum --disablerepo='*' --enablerepo='*-debug*' install
/usr/lib/debug/.build-id/a1/5e9bb81f6c0efee08919907a35281df586e948
Missing separate debuginfo for
Try: yum --disablerepo='*' --enablerepo='*-debug*' install
/usr/lib/debug/.build-id/b5/d86fbcf0ccb03331e6c7c73897b96845e0a4eb
Reading symbols from /usr/lib64/libssl.so.10...(no debugging symbols
found)...done.
Loaded symbols for /usr/lib64/libssl.so.10
Reading symbols from /usr/lib64/libcrypto.so.10...(no debugging symbols
found)...done.
Loaded symbols for /usr/lib64/libcrypto.so.10
Reading symbols from /lib64/libm.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libm.so.6
Reading symbols from /apacheSSLDEV2/lib/libaprutil-1.so.0...done.
Loaded symbols for /apacheSSLDEV2/lib/libaprutil-1.so.0
Reading symbols from /apacheSSLDEV2/lib/libexpat.so.0...done.
Loaded symbols for /apacheSSLDEV2/lib/libexpat.so.0
Reading symbols from /apacheSSLDEV2/lib/libapr-1.so.0...done.
Loaded symbols for /apacheSSLDEV2/lib/libapr-1.so.0
Reading symbols from /lib64/librt.so.1...(no debugging symbols
found)...done.
Loaded symbols for /lib64/librt.so.1
Reading symbols from /lib64/libcrypt.so.1...(no debugging symbols
found)...done.
Loaded symbols for /lib64/libcrypt.so.1
Reading symbols from /lib64/libpthread.so.0...(no debugging symbols
found)...done.
[Thread debugging using libthread_db enabled]
Loaded symbols for /lib64/libpthread.so.0
Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libc.so.6
Reading symbols from /lib64/libgssapi_krb5.so.2...(no debugging symbols
found)...done.
Loaded symbols for /lib64/libgssapi_krb5.so.2
Reading symbols from /lib64/libkrb5.so.3...(no debugging symbols
found)...done.
Loaded symbols for /lib64/libkrb5.so.3
Reading symbols from /lib64/libcom_err.so.2...(no debugging symbols
found)...done.
Loaded symbols for /lib64/libcom_err.so.2
Reading symbols from /lib64/libk5crypto.so.3...(no debugging symbols
found)...done.
Loaded symbols for /lib64/libk5crypto.so.3
Reading symbols from /lib64/libdl.so.2...(no debugging symbols
found)...done.
Loaded symbols for /lib64/libdl.so.2
Reading symbols from /lib64/libz.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libz.so.1
Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols
found)...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /lib64/libfreebl3.so...(no debugging symbols
found)...done.
Loaded symbols for /lib64/libfreebl3.so
Reading symbols from /lib64/libkrb5support.so.0...(no debugging symbols
found)...done.
Loaded symbols for /lib64/libkrb5support.so.0
Reading symbols from /lib64/libkeyutils.so.1...(no debugging symbols
found)...done.
Loaded symbols for /lib64/libkeyutils.so.1
Reading symbols from /lib64/libresolv.so.2...(no debugging symbols
found)...done.
Loaded symbols for /lib64/libresolv.so.2
Reading symbols from /lib64/libselinux.so.1...(no debugging symbols
found)...done.
Loaded symbols for /lib64/libselinux.so.1
Reading symbols from /lib64/libnss_files.so.2...(no debugging symbols
found)...done.
Loaded symbols for /lib64/libnss_files.so.2
Reading symbols from /apacheSSLDEV2/modules/libopentoken.so...(no debugging
symbols found)...done.
Loaded symbols for /apacheSSLDEV2/modules/libopentoken.so
Reading symbols from /apacheSSLDEV2/modules/mod_pf.so...(no debugging
symbols found)...done.
Loaded symbols for /apacheSSLDEV2/modules/mod_pf.so
Reading symbols from /apacheSSLDEV2/modules/mod_headers.so...done.
Loaded symbols for /apacheSSLDEV2/modules/mod_headers.so
Reading symbols from /apacheSSLDEV2/modules/mod_proxy.so...done.
Loaded symbols for /apacheSSLDEV2/modules/mod_proxy.so
Reading symbols from /apacheSSLDEV2/modules/mod_proxy_connect.so...done.
Loaded symbols for /apacheSSLDEV2/modules/mod_proxy_connect.so
Reading symbols from /apacheSSLDEV2/modules/mod_proxy_ftp.so...done.
Loaded symbols for /apacheSSLDEV2/modules/mod_proxy_ftp.so
Reading symbols from /apacheSSLDEV2/modules/mod_proxy_http.so...done.
Loaded symbols for /apacheSSLDEV2/modules/mod_proxy_http.so
Core was generated by `/apacheSSLDEV2/bin/httpd -k start'.
Program terminated with signal 11, Segmentation fault.
#0 apr_table_clear (t=0x101131d5503060f) at tables/apr_tables.c:467
467 t->a.nelts = 0;
Missing separate debuginfos, use: debuginfo-install
glibc-2.12-1.107.el6_4.2.x86_64 keyutils-libs-1.4-4.el6.x86_64
krb5-libs-1.10.3-10.el6.x86_64 libcom_err-1.41.12-14.el6.x86_64
libselinux-2.0.94-5.3.el6.x86_64 nss-softokn-freebl-3.12.9-11.el6.x86_64
openssl-1.0.1e-48.el6_8.1.x86_64 zlib-1.2.3-29.el6.x86_64
(gdb) bt full
#0 apr_table_clear (t=0x101131d5503060f) at tables/apr_tables.c:467
No locals.
#1 0x0000000000472116 in read_chunked_trailers (f=<value optimized out>,
b=<value optimized out>, mode=AP_MODE_READBYTES,
block=APR_BLOCK_READ, readbytes=8192) at http_filters.c:286
rv = <value optimized out>
e = <value optimized out>
saved_headers_in = 0x1164440
saved_status = 200
r = 0x1175cb0
#2 ap_http_filter (f=<value optimized out>, b=<value optimized out>,
mode=AP_MODE_READBYTES, block=APR_BLOCK_READ, readbytes=8192)
at http_filters.c:551
conf = 0x10c0c08
e = <value optimized out>
ctx = 0x1164748
rv = <value optimized out>
http_error = 413
bb = <value optimized out>
again = 1
#3 0x00007f99d70a983d in ap_proxy_http_process_response (p=0x115e878,
r=<value optimized out>, backend=0x11423b0, origin=0x10cc4a0,
conf=0x10db2b8, server_portstr=0x7fff1f2f0000 "") at
mod_proxy_http.c:1770
readbytes = 3716
rv = <value optimized out>
mode = <value optimized out>
finish = 0
rc = <value optimized out>
c = 0x114e1b0
buffer = "\000\nte\000 Wed, 12 Oct 2016 13:35:02
GMT\000\n859-1\000\n\000\000\000\000\200IF", '\000' <repeats 14
times>"\343,
.\037\377\177\000\000p'\360×\177\000\000\000\000\000\000\000\000\000\000]\255\346t",
'\000' <repeats 12 times>"\240,
(... User data deleted here....)
buf = <value optimized out>
keepchar = <value optimized out>
rp = 0x1175cb0
e = <value optimized out>
bb = 0x1175c70
tmp_bb = 0x1163e18
pass_bb = 0x1175c90
len = 15
backasswards = <value optimized out>
interim_response = 0
pread_len = <value optimized out>
save_table = <value optimized out>
backend_broke = 0
hop_by_hop_hdrs = {0x7f99d70ac405 "Keep-Alive", 0x7f99d70ac772
"Proxy-Authenticate", 0x7f99d70ac650 "TE",
0x7f99d70ac653 "Trailer", 0x7f99d70ac65b "Upgrade", 0x0}
i = <value optimized out>
te = 0x1164368 "chunked"
original_status = 200
proxy_status = 200
original_status_line = 0x0
proxy_status_line = <value optimized out>
#4 0x00007f99d70aac5f in proxy_http_handler (r=0x115e8f0, worker=<value
optimized out>, conf=0x7fff1f2f0088,
url=0x1173cf0 "/jsp/userListForm.jsp?mode=manage", proxyname=0x1155f88
"8\277\024\001", proxyport=50336) at mod_proxy_http.c:2036
status = <value optimized out>
server_portstr = "\000\350\025\001\000\000\000\000\370\263\r\001",
'\000' <repeats 12 times>"\370, \263\r\001\000\000\000"
scheme = <value optimized out>
proxy_function = 0x7f99d70ac577 "HTTP"
u = <value optimized out>
backend = 0x11423b0
is_ssl = 18300184
c = 0x114e1b0
p = 0x115e878
uri = <value optimized out>
#5 0x00007f99d76bce62 in proxy_run_scheme_handler (r=0x115e8f0,
worker=0x10dbf10, conf=0x10db2b8,
url=0x117340e "http://(... URL deleted here ...)", proxyhost=0x0,
proxyport=0) at mod_proxy.c:2415
pHook = <value optimized out>
n = <value optimized out>
rv = -1
#6 0x00007f99d76c1457 in proxy_handler (r=0x115e8f0) at mod_proxy.c:1031
url = 0x117340e "http://(... URL deleted here ...)"
uri = 0x117340e "http://(... URL deleted here ...)"
scheme = 0x1173c00 "http"
p = 0x7fff1f2f01b8 "\340\353\025\001"
p2 = <value optimized out>
sconf = <value optimized out>
conf = 0x10db2b8
proxies = 0x10db3f8
ents = 0x10db418
i = <value optimized out>
access_status = 0
direct_connect = <value optimized out>
str = <value optimized out>
maxfwd = <value optimized out>
balancer = 0x0
worker = 0x10dbf10
attempts = 0
max_attempts = 0
list = <value optimized out>
#7 0x0000000000441680 in ap_run_handler (r=0x115e8f0) at config.c:158
pHook = <value optimized out>
n = <value optimized out>
rv = -1
#8 0x0000000000444cae in ap_invoke_handler (r=0x115e8f0) at config.c:376
handler = <value optimized out>
p = <value optimized out>
result = 0
old_handler = 0x7f99d76c827d "proxy-server"
ignore = <value optimized out>
#9 0x0000000000470650 in ap_process_request (r=0x115e8f0) at
http_request.c:294
access_status = <value optimized out>
#10 0x000000000046d5b0 in ap_process_http_connection (c=0x114e1b0) at
http_core.c:190
r = 0x115e8f0
csd = 0x0
#11 0x0000000000448c10 in ap_run_process_connection (c=0x114e1b0) at
connection.c:43
pHook = <value optimized out>
n = <value optimized out>
rv = -1
#12 0x0000000000484188 in child_main (child_num_arg=<value optimized out>)
at prefork.c:667
current_conn = 0x114e1b0
csd = 0x114dfc0
ptrans = 0x114df48
allocator = 0x1133850
status = <value optimized out>
i = <value optimized out>
lr = <value optimized out>
pollset = 0x114c0d8
sbh = 0x114c0d0
bucket_alloc = 0x1155f88
last_poll_idx = 0
#13 0x0000000000484484 in make_child (s=0x10ba930, slot=6) at prefork.c:768
pid = 0
#14 0x00000000004850d7 in perform_idle_server_maintenance (_pconf=<value
optimized out>, plog=<value optimized out>,
s=<value optimized out>) at prefork.c:903
i = <value optimized out>
idle_count = <value optimized out>
ws = <value optimized out>
free_length = <value optimized out>
free_slots = {6, 7, 17985288, 0, 17541424, 0, 17985288, 0,
17985336, 0, 4438247, 0, 17985288, 0, 17985248, 0, 17985248, 0,
4578767, 0, 17985248, 0, 14, 0, 17958728, 0, 8, 0, 17580504, 0,
17555880, 0}
last_non_dead = <value optimized out>
total_non_dead = <value optimized out>
#15 ap_mpm_run (_pconf=<value optimized out>, plog=<value optimized out>,
s=<value optimized out>) at prefork.c:1107
status = 17506872
pid = {pid = -1, in = 0x7f99d7f1e13b, out = 0x8, err = 0x10b4158}
child_slot = <value optimized out>
exitwhy = 0
processed_status = <value optimized out>
index = <value optimized out>
remaining_children_to_start = 0
rv = <value optimized out>
#16 0x000000000042da64 in main (argc=3, argv=0x7fff1f2f0748) at main.c:753
c = 0 '\000'
configtestonly = <value optimized out>
confname = 0x48e5ce "conf/httpd.conf"
def_server_root = 0x48e5bf "/apacheSSLDEV2"
temp_error_log = 0x0
error = <value optimized out>
process = 0x10ba930
server_conf = 0x10ba930
pglobal = 0x10b2148
pconf = 0x10b4158
plog = 0x10c41d8
ptemp = 0x10be1a8
pcommands = 0x10b6168
opt = 0x10b6258
rv = <value optimized out>
mod = <value optimized out>
optarg = 0x7fff1f2f06c0
"v\225b\213|\203\313Ov\225\262^\241\345C\260"
signal_server = <value optimized out>
(gdb)
