From: Peter Portante <[email protected]>
Reply-To: rsyslog-users <[email protected]>
To: rsyslog-users <[email protected]>
Subject: Re: [rsyslog] Adding properties to a received log entry
On Thu, Oct 29, 2015 at 4:22 PM, Peter Portante <
[email protected]>
wrote:
On Thu, Oct 29, 2015 at 3:47 PM, David Lang <[email protected]> wrote:
On Thu, 29 Oct 2015, Peter Portante wrote:
On Thu, Oct 29, 2015 at 9:52 AM, Dave Caplinger <
[email protected]> wrote:
There are two approaches:
1) Modify the syslog line to insert the property by using a template
2) Use JSON to preserve the original unmodified log line and add the
property as an extra key/value pair
Either way, I'd suggest you first validate if you have or can get a
modern
(v8) version of Rsyslog; your config snippet suggests you may be using
version 5 or lower.
Yes, so RHEL 7 ships v7 right now, but they did not change the
rsyslog.conf
file that ships with RHEL. :(
I have the ability to run v8, so I'd like to do #2. Here is a JSON
template that I use, would I reference the "$!roles" variable as below?
set $!roles = "openstack-0,controller,db"
template(name="mytemplate"
type="list") {
constant(value="{")
constant(value="\"@timestamp\":\"")
property(name="timereported" dateFormat="rfc3339")
constant(value="\",\"@version\":\"2015.09.24-0")
constant(value="\",\"message\":\"")
property(name="msg"
format="json")
constant(value="\",\"hostname\":\"")
property(name="hostname")
constant(value="\",\"level\":\"")
property(name="syslogseverity-text")
constant(value="\",\"pid\":\"")
property(name="procid")
constant(value="\",\"roles\":\"")
property(name="!roles")
constant(value="\",\"rsyslog\": {")
constant(value="\"facility\":\"")
property(name="syslogfacility-text")
constant(value="\",\"programname\":\"")
property(name="programname")
constant(value="\",\"fromhost\":\"")
property(name="fromhost")
constant(value="\",\"fromhost-ip\":\"")
property(name="fromhost-ip")
constant(value="\",\"timegenerated\":\"")
property(name="timegenerated" dateFormat="rfc3339")
constant(value="\",\"protocol-version\":\"")
property(name="protocol-version")
constant(value="\",\"structured-data\":\"")
property(name="structured-data")
constant(value="\",\"app-name\":\"")
property(name="app-name")
constant(value="\",\"msgid\":\"")
property(name="msgid")
constant(value="\",\"inputname\":\"")
property(name="inputname")
constant(value="\"} }")
}
So since I have a number of systems that are running v5 and v7 of
rsyslog,
is there a way in those versions to modify the
RSYSLOG_SyslogProtocol23Format template to add a constant value?
you are working too hard on this.
Yes, you are right, I can see what need to do here.
create a template similar to:
$template structuredmsg,"<%pri%>%timereported% %hostname% %syslogtag%
@cee:%$!%\n"
This works with v7* so this is wonderful.
then log things with the format RSYSLOG_DebugFormat and make sure that
everything you want to send to the far side shows up in either hostname,
programname, facility/severity, or as a $! variable.
if you use mmjsonparse, it will parse the message if it's json, and it
will make $!msg = $msg if it's not json.
So I have v8.8.0 right now, and the mmjsonparse module appears to go into
an infinite loop on some message. So I'll get the latest setup on my box
and work with that.
Well, I moved to v8.14.0, and when I add the
"action(type=\"mmjsonparse\")"
before I do anything else, no logging occurs, and the messages begin to
pile up internally (if I following the debug output correctly).
I am probably missing something here:
module(
load="imuxsock"
SysSock.Name="/run/systemd/journal/syslog"
SysSock.Use="off")
module(load="imjournal" StateFile="/var/lib/rsyslog/imjournal.state")
module(load="imklog")
module(load="imudp")
input(type="imudp" port="514")
module(load="imptcp")
input(type="imptcp" port="514")
module(load="omelasticsearch")
module(load="mmjsonparse")
module(load="mmsnmptrapd")
module(load="mmutf8fix")
global(
# Where to place auxiliary files
workDirectory="/var/lib/rsyslog"
# perf-dept: we want fully qualified domain names for common logging
preserveFQDN="on")
main_queue(
# Beaf up the internal message queue
queue.size="131072"
# 90% of QueueSize
queue.discardmark="117964"
# If we reach the discard mark, we'll throw out notice, info, and debug
messages
queue.discardseverity="5")
template(name="logstash-index-pattern" type="list") {
constant(value="logstash-")
property(name="timereported" dateFormat="rfc3339" position.from="1"
position.to="4")
constant(value=".")
property(name="timereported" dateFormat="rfc3339" position.from="6"
position.to="7")
constant(value=".")
property(name="timereported" dateFormat="rfc3339" position.from="9"
position.to="10")
}
template(name="com-redhat-rsyslog-hier"
type="list") {
constant(value="{")
constant(value="\"@timestamp\":\"")
property(name="timereported" dateFormat="rfc3339")
constant(value="\",\"@version\":\"2015.09.24-0")
constant(value="\",\"message\":\"") property(name="msg"
format="json")
constant(value="\",\"hostname\":\"")
property(name="hostname")
constant(value="\",\"level\":\"")
property(name="syslogseverity-text")
constant(value="\",\"pid\":\"")
property(name="procid")
constant(value="\",\"rsyslog\": {")
constant(value="\"facility\":\"")
property(name="syslogfacility-text")
constant(value="\",\"programname\":\"")
property(name="programname")
constant(value="\",\"fromhost\":\"")
property(name="fromhost")
constant(value="\",\"fromhost-ip\":\"")
property(name="fromhost-ip")
constant(value="\",\"timegenerated\":\"")
property(name="timegenerated" dateFormat="rfc3339")
constant(value="\",\"protocol-version\":\"")
property(name="protocol-version")
constant(value="\",\"structured-data\":\"")
property(name="structured-data")
constant(value="\",\"app-name\":\"")
property(name="app-name")
constant(value="\",\"msgid\":\"")
property(name="msgid")
constant(value="\",\"inputname\":\"")
property(name="inputname")
constant(value="\",\"cee\":\"")
property(name="$!all-json")
constant(value="\"} }")
}
$ActionFileDefaultTemplate RSYSLOG_FileFormat
$IncludeConfig /etc/rsyslog.d/*.conf
action(type="mmutf8fix" mode="utf-8")
*.* :mmsnmptrapd:
action(type="mmjsonparse")
action(
type="omelasticsearch"
server="172.18.40.3"
serverport="9200"
template="com-redhat-rsyslog-hier"
searchIndex="logstash-index-pattern"
dynSearchIndex="on"
searchType="rsyslog"
bulkmode="on"
queue.type="linkedlist"
queue.size="5000"
queue.dequeuebatchsize="300"
action.resumeretrycount="-1")
Thanks for any help you can offer.
-peter
Thanks for setting me straight, David!
-peter
you then send via the structuredmsg I show above, and on the receiving
system, use mmjsonparse and it will recreate the $! variables that you
sent
(validate via RSYSLOG_DebugFormat
David Lang
_______________________________________________
rsyslog mailing list
http://lists.adiscon.net/mailman/listinfo/rsyslog
http://www.rsyslog.com/professional-services/
What's up with rsyslog? Follow https://twitter.com/rgerhards
NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad
of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you
DON'T LIKE THAT.
_______________________________________________
rsyslog mailing list
http://lists.adiscon.net/mailman/listinfo/rsyslog
http://www.rsyslog.com/professional-services/
What's up with rsyslog? Follow https://twitter.com/rgerhards
NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad
of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you
DON'T LIKE THAT.
_______________________________________________
