Hi,
we've got a problem with our logging infrastructure.
Right now in our network we've three datacenters. In every datacenter there
are over 400 VMs (Linux) which logs to rsyslog relay servers. There are two
such servers (rsyslog relays).
These two servers has virtual IP with haproxy on it. Haproxy balances
connection to rsyslog relays and then messages are forwarded to one central
rsyslog server (common for all datacenters).
Problem which we have is that messages are lost. We use RELP but it doesn't
help.
What i can see in haproxy that there are planty of such logs:
Jul 16 07:23:55 syslog-satellite2 haproxy[8495]: 10.122.200.20:18827
[16/Jul/2016:07:23:50.049] relp_20514 relp_20514/syslog-satellite2 1/0/5005
103 CD 171/171/171/78/0 0/0 [10.122.7.22 54085]
And for all servers which IPs got CD haproxy error code no messages are
written on central log server.
Only one thing helps - restarting rsyslog on client side. But it helps for
few hours and situation repeats, logs doesn't show on central server,
haproxy show errors.
And one extra thing, restarting client side doesn't send overdue logs, they
are lost.
CLIENT RSYSLOG CONFIG
$ cat /etc/rsyslog.conf | egrep -v '^#|^$'
$PreserveFQDN on
$ModLoad imuxsock # provides support for local system logging
$ModLoad imklog # provides kernel logging support (previously done by
rklogd)
$MaxMessageSize 64k
$FileOwner root
$FileGroup root
$FileCreateMode 0600
$DirOwner root
$DirGroup root
$DirCreateMode 0750
$PrivDropToUser root
$PrivDropToGroup root
$WorkDirectory /var/lib/rsyslog
$Umask 0000
$IncludeConfig /etc/rsyslog.d/*.conf
*.emerg :omusrmsg:*
$ sudo cat /etc/rsyslog.d/* | egrep -v '^#|^$'
$WorkDirectory /var/lib/rsyslog # where to place spool files
$ActionQueueFileName queue # unique name prefix for spool files
$ActionQueueMaxDiskSpace 1g # spool space limit (use as much as
possible)
$ActionQueueSaveOnShutdown on # save messages to disk on shutdown
$ActionQueueType LinkedList # run asynchronously
$ActionResumeRetryCount -1 # infinety retries if host is down
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
$EscapeControlCharactersOnReceive off
$RepeatedMsgReduction off
$ActionResumeInterval 5
$ModLoad omrelp
*.* :omrelp:syslog-satellite.dc:20514;RSYSLOG_ForwardFormat
$outchannel log_rotation,/var/log/log_rotation.log,
52428800,/usr/local/sbin/log_rotation_script.sh
*.* :omfile:$log_rotation
$ModLoad imfile
$InputFileName /usr/share/jboss-as/standalone/log/server.log
$InputFileTag jboss-as
$InputFileStateFile state-server.log
$InputFileSeverity notice
$InputFileFacility local0
$InputFilePollInterval 10
$InputFilePersistStateInterval 0
$InputRunFileMonitor
RELAY HOSTS (syslog-satellite.dc):
* haproxy:
/etc/haproxy/haproxy.cfg
# Managed by Puppet
global
nbproc 1
maxconn 2000000
ulimit-n 4000034
log /dev/log local0
log /dev/log local1 notice
#chroot /var/lib/haproxy
stats socket /var/run/haproxy.sock mode 666 level admin
user haproxy
group haproxy
daemon
#debug
#quiet
# Default SSL material locations
ca-base /etc/ssl/certs
crt-base /etc/ssl/private
# Default ciphers to use on SSL-enabled listening sockets.
# For more information, see ciphers(1SSL).
ssl-default-bind-ciphers
kEECDH+aRSA+AES:kRSA+AES:+AES256:RC4-SHA:!kEDH:!LOW:!EXP:!MD5:!aNULL:!eNULL
tune.ssl.default-dh-param 4096
defaults
log global
option httplog
option redispatch
option dontlognull
option log-separate-errors
retries 3
# maxconn 2000
timeout connect 60s
timeout client 300s
timeout server 300s
listen stats 0.0.0.0:1936
mode http
stats enable
stats uri /
stats realm HAProxy
stats auth noc:mykEwyoug3
listen relp_20514 :20514
mode tcp
balance roundrobin
option tcplog
maxconn 2000000
log-format %ci:%cp\ [%t]\ %ft\ %b/%s\ %Tw/%Tc/%Tt\ %B\ %ts\
%ac/%fc/%bc/%sc/%rc\ %sq/%bq\ [%bi\ %bp]
server syslog-satellite1.dc 10.122.7.7:20515 weight 5 check inter
2000 maxconn 100000
server syslog-satellite2.dc 10.122.7.22:20515 weight 5 check inter
2000 maxconn 100000
sysctl.conf:
fs.file-max = 10000000
fs.nr_open = 10000000
net.ipv4.tcp_mem = 786432 1697152 1945728
net.ipv4.tcp_rmem = 4096 4096 16777216
net.ipv4.tcp_wmem = 4096 4096 16777216
net.ipv4.ip_local_port_range = 1000 65535
net.ipv4.tcp_keepalive_time = 600
rsyslog
$ cat /etc/rsyslog.conf | egrep -v '^#|^$'
$PreserveFQDN on
$ModLoad imuxsock # provides support for local system logging
$ModLoad imklog # provides kernel logging support (previously done by
rklogd)
$MaxMessageSize 64k
$FileOwner root
$FileGroup root
$FileCreateMode 0600
$DirOwner root
$DirGroup root
$DirCreateMode 0750
$PrivDropToUser root
$PrivDropToGroup root
$WorkDirectory /var/lib/rsyslog
$Umask 0000
$IncludeConfig /etc/rsyslog.d/*.conf
*.emerg :omusrmsg:*
$ sudo cat /etc/rsyslog.d/* | egrep -v '^#|^$'
module(load="imtcp" MaxSessions="10000")
input(type="imtcp" port="514")
module(load="imrelp")
input(type="imrelp"
port="20515")
KeepAlive="on"
KeepAlive.Probes="3"
KeepAlive.Interval="30"
KeepAlive.Time="900")
module(load="omrelp")
action(type="omrelp"
target="central.server.log"
port="20514"
action.reportSuspension="on"
action.reportSuspensionContinuation="on")
$ModLoad imudp
$UDPServerRun 514
$FileCreateMode 0644
$DirCreateMode 0755
$EscapeControlCharactersOnReceive off
$ActionQueueFileName srvrfwd # set file name, also enables disk mode
$ActionResumeRetryCount -1 # infinite retries on insert failure
$ActionQueueSaveOnShutdown on # save in-memory data if rsyslog shuts down
main_queue(
queue.workerThreads="4"
queue.dequeueBatchSize="1000"
queue.highWatermark="3000000" # max no. of events to hold in memory
queue.lowWatermark="200000" # use memory queue again, when it's back
to this level
queue.spoolDirectory="/var/lib/rsyslog" # where to write on disk
queue.fileName="stats_ruleset"
queue.maxDiskSpace="5g" # it will stop at this much disk space
queue.size="5000000" # or this many messages
queue.saveOnShutdown="on" # save memory queue contents to disk when
rsyslog is exiting
)
module(load="impstats"
interval="600"
severity="7"
log.syslog="off"
resetCounters="on"
# need to turn log stream logging off
log.file="/var/log/rsyslog-stats.log")
CENTRAL SERVER LOG
cat /etc/rsyslog.conf | egrep -v '^#|^$'
$PreserveFQDN on
$ModLoad imuxsock # provides support for local system logging
$ModLoad imklog # provides kernel logging support (previously done by
rklogd)
$MaxMessageSize 64k
$FileOwner root
$FileGroup root
$FileCreateMode 0600
$DirOwner root
$DirGroup root
$DirCreateMode 0750
$PrivDropToUser root
$PrivDropToGroup root
$WorkDirectory /var/lib/rsyslog
$Umask 0000
$IncludeConfig /etc/rsyslog.d/*.conf
*.emerg :omusrmsg:*
sudo cat /etc/rsyslog.d/* | egrep -v '^#|^$'
$WorkDirectory /var/lib/rsyslog # where to place spool files
$ActionQueueFileName queue # unique name prefix for spool files
$ActionQueueMaxDiskSpace 1g # spool space limit (use as much as
possible)
$ActionQueueSaveOnShutdown on # save messages to disk on shutdown
$ActionQueueType LinkedList # run asynchronously
$ActionResumeRetryCount -1 # infinety retries if host is down
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
module(load="imtcp" MaxSessions="1000")
input(type="imtcp" port="514")
module(load="imrelp")
input(type="imrelp" port="20514")
$ModLoad imudp
$UDPServerRun 514
$FileCreateMode 0644
$DirCreateMode 0755
$EscapeControlCharactersOnReceive off
$template
REMOTE,"/export/%HOSTNAME%/%$year%/%$month%/%$day%/%syslogfacility-text%.log"
*.* -?REMOTE
Can anybody help with this? Mabye point what we misunderstand in
configuration of rsyslog architecture?
regards,
p.
_______________________________________________
rsyslog mailing list
http://lists.adiscon.net/mailman/listinfo/rsyslog
http://www.rsyslog.com/professional-services/
What's up with rsyslog? Follow https://twitter.com/rgerhards
NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad of
sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you DON'T LIKE
THAT.
