Hello,
I am encountering frequent occurrences of httpd crash in mod_perl/libperl
in HP-UX environment with Apache 2.4.18 (mpm_worker) + Mod_perl 2.0.10 +
perl 5.28 configuration
Below is my sample (httpd.conf)
LoadModule mpm_worker_module modules/mod_mpm_worker.so
PerlModule TEST::coreit
<Location /coreit>
SetHandler perl-script
PerlResponseHandler TEST::coreit
</Location>
Below is the content of the coreit.pm
# cat coreit.pm
package TEST::coreit;
use strict;
use warnings;
use Apache2::RequestRec ();
use Apache2::RequestIO ();
use CGI ':standard';
use CGI::Carp qw(fatalsToBrowser);
use Apache2::Const -compile => qw(OK);
use APR::Const -compile => qw(ECONNABORTED);
# Declare the subroutines
sub handler {
my $r = shift;
my $contentType = "text/html";
my $response = "";
my $respLength = 0;
$response = "Apache Core dump test";
$respLength = length ( $response );
$r->content_type($contentType);
$r->puts($response);
$r->rflush();
return Apache2::Const::OK;
}
1;
Below is the test driver script that found this core dump
# cat coreit.ApacheDriver.ksh
#!/usr/bin/ksh
typeset -i loopIndex=0
typeset -i upperBound=2
date
while (( loopIndex < upperBound ))
do
print $loopIndex
$PWD/Httpd.pl &
(( loopIndex = loopIndex + 1 ))
done
wait
date
# cat Httpd.pl
#!/usr/bin/perl
use strict;
use warnings;
use LWP::Simple;
use LWP::UserAgent;
use Time::HiRes;
my $browser = LWP::UserAgent->new(ssl_opts => { verify_hostname => 0 });
for (my $count=0;$count < 2000; $count=$count+1)
{
my $count=0;
while ($count<126)
{
my $url = "
https://<your_server_name>:49222/coreit?dp=cn=crl$count,o=xyz";
my $response = $browser->get($url);
print ("Apache retun code is " . $response->code . "with
count=" . $count. "\n");
Time::HiRes::sleep(0.1);
$count=$count+1;
}
}
So, when I run this script like below in backgorund mode for an hour , I am
getting nearly 40+ core dump files
ksh coreit.ApacheDriver.ksh 1>/var/tmp/outf 2>&1 &
Below are the few back traces that we are seeing:
Trace 1:
========
#0 0x1ffffffffe9b2940:1 in memmove+0x941 () from /usr/lib/hpux64/libc.so.1
(gdb) bt
#0 0x1ffffffffe9b2940:1 in memmove+0x941 () from /usr/lib/hpux64/libc.so.1
#1 0x1ffffffffeac6780:0 in real_realloc+0xdd0 ()
from /usr/lib/hpux64/libc.so.1
#2 0x1ffffffffeac5960:0 in _realloc+0xa0 () from /usr/lib/hpux64/libc.so.1
#3 0x1ffffffffead2330:0 in realloc+0x280 () from /usr/lib/hpux64/libc.so.1
#4 0x1ffffffffe2d3770:0 in Perl_safesysrealloc () at util.c:271
#5 0x1ffffffffe32b6a0:0 in S_hv_auxinit () at hv.c:2237
#6 0x1ffffffffe32b860:0 in Perl_hv_riter_p () at hv.c:2311
#7 0x1ffffffffe0e5eb0:0 in copyENV () at modperl_perl_global.c:264
#8 0x1ffffffffe0e6110:0 in modperl_perl_global_gvhv_save ()
at modperl_perl_global.c:322
#9 0x1ffffffffe0e6460:0 in modperl_perl_global_save ()
at modperl_perl_global.c:436
#10 0x1ffffffffe0e6730:0 in modperl_perl_global_request_save ()
at modperl_perl_global.c:485
#11 0x1ffffffffe09cc10:0 in modperl_response_handler_cgi () at
mod_perl.c:1073
#12 0x40000000000faf10:0 in ap_run_handler ()
#13 0x40000000000fcee0:0 in ap_invoke_handler () at config.c:448
#14 0x4000000000147790:0 in ap_process_async_request () at
http_request.c:410
#15 0x4000000000147ae0:0 in ap_process_request () at http_request.c:442
#16 0x400000000013d2a0:0 in ap_process_http_sync_connection ()
at http_core.c:209
#17 0x400000000013d360:0 in ap_process_http_connection () at http_core.c:251
---Type <return> to continue, or q <return> to quit---
#18 0x4000000000120fc0:0 in ap_process_connection () at connection.c:228
#19 0x1ffffffffdc76ba0:0 in process_socket () at worker.c:677
#20 0x1ffffffffdc785d0:0 in worker_thread () at worker.c:1068
#21 0x1fffffffffdc1180:0 in dummy_worker () at threadproc/unix/thread.c:160
#22 0x1fffffffff7b80e0:0 in __pthread_bound_body+0x1c0 ()
from /usr/lib/hpux64/libpthread.so.1
Trace 2:
========
#0 0x1ffffffffe3f4f70:0 in Perl_leave_scope () at scope.c:1148
#1 0x1ffffffffe420b80:0 in Perl_delete_eval_scope () at pp_ctl.c:4570
#2 0x1ffffffffe1b1dd0:0 in Perl_call_sv () at perl.c:3056
#3 0x1ffffffffe0aee30:0 in modperl_callback () at modperl_callback.c:100
#4 0x1ffffffffe0afc40:0 in modperl_callback_run_handlers ()
at modperl_callback.c:236
#5 0x1ffffffffe0b0d30:0 in modperl_callback_per_dir ()
at modperl_callback.c:342
#6 0x1ffffffffe09d570:0 in modperl_response_handler_run () at
mod_perl.c:1005
#7 0x1ffffffffe09cd70:0 in modperl_response_handler_cgi () at
mod_perl.c:1095
#8 0x40000000000faf10:0 in ap_run_handler ()
#9 0x40000000000fcee0:0 in ap_invoke_handler () at config.c:448
#10 0x4000000000147790:0 in ap_process_async_request () at
http_request.c:410
#11 0x4000000000147ae0:0 in ap_process_request () at http_request.c:442
#12 0x400000000013d2a0:0 in ap_process_http_sync_connection ()
at http_core.c:209
#13 0x400000000013d360:0 in ap_process_http_connection () at http_core.c:251
#14 0x4000000000120fc0:0 in ap_process_connection () at connection.c:228
#15 0x1ffffffffdc76ba0:0 in process_socket () at worker.c:677
#16 0x1ffffffffdc785d0:0 in worker_thread () at worker.c:1068
#17 0x1fffffffffdc1180:0 in dummy_worker () at threadproc/unix/thread.c:160
#18 0x1fffffffff7b80e0:0 in __pthread_bound_body+0x1c0 ()
from /usr/lib/hpux64/libpthread.so.1
Trace 3:
========
#0 0x1ffffffffe0e6aa0:1 in modperl_perl_global_request_restore ()
at modperl_perl_global.c:479
#1 0x1ffffffffe09cdb0:0 in modperl_response_handler_cgi () at
mod_perl.c:1099
#2 0x40000000000faf10:0 in ap_run_handler ()
#3 0x40000000000fcee0:0 in ap_invoke_handler () at config.c:448
#4 0x4000000000147790:0 in ap_process_async_request () at
http_request.c:410
#5 0x4000000000147ae0:0 in ap_process_request () at http_request.c:442
#6 0x400000000013d2a0:0 in ap_process_http_sync_connection ()
at http_core.c:209
#7 0x400000000013d360:0 in ap_process_http_connection () at http_core.c:251
#8 0x4000000000120fc0:0 in ap_process_connection () at connection.c:228
#9 0x1ffffffffdc76ba0:0 in process_socket () at worker.c:677
#10 0x1ffffffffdc785d0:0 in worker_thread () at worker.c:1068
#11 0x1fffffffffdc1180:0 in dummy_worker () at threadproc/unix/thread.c:160
#12 0x1fffffffff7b80e0:0 in __pthread_bound_body+0x1c0 ()
Trace 4:
========
#0 0x1ffffffffe36a7b0:0 in Perl_sv_upgrade () at sv.c:1413
#1 0x1ffffffffe381540:0 in Perl_sv_sethek () at sv.c:5012
#2 0x1ffffffffe40c560:0 in Perl_pp_caller () at pp_ctl.c:1921
#3 0x1ffffffffe33b8c0:0 in Perl_runops_standard () at run.c:42
#4 0x1ffffffffe1b1f30:0 in Perl_call_sv () at perl.c:3071
#5 0x1ffffffffe2d9210:0 in S_invoke_exception_hook () at util.c:1563
#6 0x1ffffffffe2da260:0 in Perl_vwarn () at util.c:1860
#7 0x1ffffffffe2da960:0 in Perl_vwarner () at util.c:1974
#8 0x1ffffffffe2daba0:0 in Perl_ck_warner_d () at util.c:1927
#9 0x1ffffffffe324b00:0 in Perl_hv_iternext_flags () at hv.c:2771
#10 0x1ffffffffe0e6000:0 in copyENV () at modperl_perl_global.c:272
#11 0x1ffffffffe0e6110:0 in modperl_perl_global_gvhv_save ()
at modperl_perl_global.c:322
#12 0x1ffffffffe0e6460:0 in modperl_perl_global_save ()
at modperl_perl_global.c:436
#13 0x1ffffffffe0e6730:0 in modperl_perl_global_request_save ()
at modperl_perl_global.c:485
#14 0x1ffffffffe09cc10:0 in modperl_response_handler_cgi () at
mod_perl.c:1073
#15 0x40000000000faf10:0 in ap_run_handler ()
#16 0x40000000000fcee0:0 in ap_invoke_handler () at config.c:448
#17 0x4000000000147790:0 in ap_process_async_request () at
http_request.c:410
#18 0x4000000000147ae0:0 in ap_process_request () at http_request.c:442
#19 0x400000000013d2a0:0 in ap_process_http_sync_connection ()
As you see above some back traces are different for the cores that we got.
So we feel that there is some race between threads here.
Please help us to solve this above.
Regards,
Andrew
