Hi Olivier,
Glad you're able to replicate it because I can't get it to happen
consistently!
I'd be happy if you could share the details of how it could be
replicated if that's not too complex or hard to explain via email.
Anyway, attached to this email, you'll find the haproxy configuration
that has gotten the issue.
Also, a lua script we do use and that is referenced in the configuration.
This is running on CentOS 7.5.1804 with kernel 3.10.0-862.6.3.el7.x86_64
It is running through systemd, unit file attached to this email as well.
Let me know if you need more infos, and I'll be glad to provide them or
if you have a pcap I can replay to generate this issue in our lab.
Best Regards,
--
Julien Semaan
jsem...@inverse.ca :: +1 (866) 353-6153 *155 ::www.inverse.ca
Inverse inc. :: Leaders behind SOGo (www.sogo.nu) and PacketFence
(www.packetfence.org)
On 2018-07-24 12:28 PM, Olivier Houchard wrote:
Hi Julian,
On Mon, Jul 23, 2018 at 09:07:32AM -0400, Julien Semaan wrote:
Hi all,
We're currently using haproxy in our project PacketFence
(https://packetfence.org) and are currently experiencing an issue with
haproxy segfaulting when TCP splicing is enabled.
We're currently running version 1.8.9 and are occasionally getting segfaults
on this specific line in stream.c (line 2131):
(objt_cs(si_b->end) && __objt_cs(si_b->end)->conn->xprt &&
__objt_cs(si_b->end)->conn->xprt->snd_pipe) &&
I wasn't too bright when I found it through gdb and forgot to copy the
backtrace, so I'm hoping that the issue can be found with this limited
information.
After commenting out the code for TCP splicing with the patch attached to
the email, then the issue stopped happening.
Best Regards,
I can seem to reproduce this.
Care to share your configuration and your setup ?
Thanks !
Olivier
-- Update host on the fly
core.register_action("change_host", { 'http-req'}, function(txn)
if txn.sf:req_fhdr("Host") == nil then
txn.set_var(txn,"req.host","wireless.zammitcorp.ca") -- Update host on the fly
end
end)
-- Select backend based on Host header
local string = require("string");
core.register_action("select", { "http-req" }, function(txn)
if ( string.match(txn.sf:path(), '/profile.xml')) then
-- nothing, we let it go through for the XML profiles
elseif ( not ( string.match(txn.sf:req_fhdr("Host"):lower(), '^192%.168%.2%.10[0-9:]*$') or string.match(txn.sf:req_fhdr("Host"):lower(), '^192%.168%.4%.10[0-9:]*$') or string.match(txn.sf:req_fhdr("Host"):lower(), '^10%.61%.126%.10[0-9:]*$') or string.match(txn.sf:req_fhdr("Host"):lower(), '^wireless%.zammitcorp%.ca[0-9:]*$') or false ) ) then
txn:set_var("req.action","proxy")
else
if (txn.sf:path() == '/') then
txn:set_var("req.action","proxy")
elseif ( string.match(txn.sf:path(), '^/common') or string.match(txn.sf:path(), '^/content') or string.match(txn.sf:path(), '^/favicon.ico$') ) then
txn:set_var("req.action","static")
end
end
end)
global
external-check
user haproxy
group haproxy
daemon
pidfile /usr/local/pf/var/run/haproxy-portal.pid
log /dev/log local0
stats socket /tmp/proxystats level admin process 1
maxconn 4000
#Followup of https://github.com/inverse-inc/packetfence/pull/893
#haproxy 1.6.11 | intermediate profile | OpenSSL 1.0.1e | SRC:
https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=haproxy-1.6.11&openssl=1.0.1e&hsts=yes&profile=intermediate
#Oldest compatible clients: Firefox 1, Chrome 1, IE 7, Opera 5, Safari
1, Windows XP IE8, Android 2.3, Java 7
tune.ssl.default-dh-param 2048
ssl-default-bind-ciphers
ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
ssl-default-bind-options no-sslv3 no-tls-tickets
ssl-default-server-ciphers
ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
ssl-default-server-options no-sslv3 no-tls-tickets
#OLD SSL CONFIGURATION. IF RC4 is required or if you must support
clients older then the precendent list, comment all the block between this
comment and the precedent and uncomment the following line
#ssl-default-bind-ciphers
ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
lua-load /usr/local/pf/var/conf/passthrough.lua
listen stats
bind 10.61.126.10:1025
mode http
timeout connect 10s
timeout client 1m
timeout server 1m
stats enable
stats uri /stats
stats realm HAProxy\ Statistics
stats auth admin:packetfence
defaults
log global
mode http
option httplog
option dontlognull
timeout connect 5000
timeout client 50000
timeout server 50000
errorfile 403
/usr/local/pf/html/captive-portal/templates/rate-limiting.http
backend proxy
option httpclose
option http_proxy
option forwardfor
# Need to have a proxy listening on localhost port 8888
acl paramsquery query -m found
http-request set-uri http://127.0.0.1:8888%[path]?%[query] if paramsquery
http-request set-uri http://127.0.0.1:8888%[path] unless paramsquery
backend static
option httpclose
option http_proxy
option forwardfor
http-request set-uri http://127.0.0.1:8889%[path]?%[query]
frontend portal-http-192.168.2.10
bind 192.168.2.10:80
stick-table type ip size 1m expire 10s store gpc0,http_req_rate(10s)
tcp-request connection track-sc1 src
http-request lua.change_host
acl host_exist var(req.host) -m found
http-request set-header Host %[var(req.host)] if host_exist
http-request lua.select
acl action var(req.action) -m found
acl unflag_abuser src_clr_gpc0 --
http-request allow if action unflag_abuser
http-request deny if { src_get_gpc0 gt 0 }
reqadd X-Forwarded-Proto:\ http
use_backend %[var(req.action)]
default_backend 192.168.2.10-backend
frontend portal-https-192.168.2.10
bind 192.168.2.10:443 ssl no-sslv3 crt /usr/local/pf/conf/ssl/server.pem
stick-table type ip size 1m expire 10s store gpc0,http_req_rate(10s)
tcp-request connection track-sc1 src
http-request lua.change_host
acl host_exist var(req.host) -m found
http-request set-header Host %[var(req.host)] if host_exist
http-request lua.select
acl action var(req.action) -m found
acl unflag_abuser src_clr_gpc0 --
http-request allow if action unflag_abuser
http-request deny if { src_get_gpc0 gt 0 }
reqadd X-Forwarded-Proto:\ https
use_backend %[var(req.action)]
default_backend 192.168.2.10-backend
backend 192.168.2.10-backend
balance source
option httpclose
option forwardfor
acl status_501 status 501
acl abuse src_http_req_rate(portal-http-192.168.2.10) ge 48
acl flag_abuser src_inc_gpc0(portal-http-192.168.2.10) --
acl abuse src_http_req_rate(portal-https-192.168.2.10) ge 48
acl flag_abuser src_inc_gpc0(portal-https-192.168.2.10) --
http-response deny if abuse status_501 flag_abuser
server 127.0.0.1 127.0.0.1:80 check
frontend portal-http-192.168.4.10
bind 192.168.4.10:80
stick-table type ip size 1m expire 10s store gpc0,http_req_rate(10s)
tcp-request connection track-sc1 src
http-request lua.change_host
acl host_exist var(req.host) -m found
http-request set-header Host %[var(req.host)] if host_exist
http-request lua.select
acl action var(req.action) -m found
acl unflag_abuser src_clr_gpc0 --
http-request allow if action unflag_abuser
http-request deny if { src_get_gpc0 gt 0 }
reqadd X-Forwarded-Proto:\ http
use_backend %[var(req.action)]
default_backend 192.168.4.10-backend
frontend portal-https-192.168.4.10
bind 192.168.4.10:443 ssl no-sslv3 crt /usr/local/pf/conf/ssl/server.pem
stick-table type ip size 1m expire 10s store gpc0,http_req_rate(10s)
tcp-request connection track-sc1 src
http-request lua.change_host
acl host_exist var(req.host) -m found
http-request set-header Host %[var(req.host)] if host_exist
http-request lua.select
acl action var(req.action) -m found
acl unflag_abuser src_clr_gpc0 --
http-request allow if action unflag_abuser
http-request deny if { src_get_gpc0 gt 0 }
reqadd X-Forwarded-Proto:\ https
use_backend %[var(req.action)]
default_backend 192.168.4.10-backend
backend 192.168.4.10-backend
balance source
option httpclose
option forwardfor
acl status_501 status 501
acl abuse src_http_req_rate(portal-http-192.168.4.10) ge 48
acl flag_abuser src_inc_gpc0(portal-http-192.168.4.10) --
acl abuse src_http_req_rate(portal-https-192.168.4.10) ge 48
acl flag_abuser src_inc_gpc0(portal-https-192.168.4.10) --
http-response deny if abuse status_501 flag_abuser
server 127.0.0.1 127.0.0.1:80 check
frontend portal-http-10.61.126.10
bind 10.61.126.10:80
stick-table type ip size 1m expire 10s store gpc0,http_req_rate(10s)
tcp-request connection track-sc1 src
http-request lua.change_host
acl host_exist var(req.host) -m found
http-request set-header Host %[var(req.host)] if host_exist
http-request lua.select
acl action var(req.action) -m found
acl unflag_abuser src_clr_gpc0 --
http-request allow if action unflag_abuser
http-request deny if { src_get_gpc0 gt 0 }
reqadd X-Forwarded-Proto:\ http
use_backend %[var(req.action)]
default_backend 10.61.126.10-backend
frontend portal-https-10.61.126.10
bind 10.61.126.10:443 ssl no-sslv3 crt /usr/local/pf/conf/ssl/server.pem
stick-table type ip size 1m expire 10s store gpc0,http_req_rate(10s)
tcp-request connection track-sc1 src
http-request lua.change_host
acl host_exist var(req.host) -m found
http-request set-header Host %[var(req.host)] if host_exist
http-request lua.select
acl action var(req.action) -m found
acl unflag_abuser src_clr_gpc0 --
http-request allow if action unflag_abuser
http-request deny if { src_get_gpc0 gt 0 }
reqadd X-Forwarded-Proto:\ https
use_backend %[var(req.action)]
default_backend 10.61.126.10-backend
backend 10.61.126.10-backend
balance source
option httpclose
option forwardfor
acl status_501 status 501
acl abuse src_http_req_rate(portal-http-10.61.126.10) ge 48
acl flag_abuser src_inc_gpc0(portal-http-10.61.126.10) --
acl abuse src_http_req_rate(portal-https-10.61.126.10) ge 48
acl flag_abuser src_inc_gpc0(portal-https-10.61.126.10) --
http-response deny if abuse status_501 flag_abuser
server 127.0.0.1 127.0.0.1:80 check
[Unit]
Description=PacketFence HAProxy Load Balancer for the captive portal
Before=packetfence-httpd.portal.service packetfence-httpd.admin.service
Wants=packetfence-config.service
[Service]
Type=notify
StartLimitBurst=3
StartLimitInterval=10
ExecStartPre=/bin/perl -I/usr/local/pf/lib
'-Mpf::services::manager::haproxy_portal' -e
'pf::services::manager::haproxy_portal->new()->generateConfig()'
ExecStart=/usr/sbin/haproxy -Ws -f /usr/local/pf/var/conf/haproxy-portal.conf
-p /usr/local/pf/var/run/haproxy-portal.pid
ExecReload=/bin/kill -USR2 $MAINPID
Restart=on-failure
Slice=packetfence.slice
[Install]
WantedBy=packetfence.target
