Hi,
I got much further now! It actually is now shipping all the logs to
ElasticSearch and I am able to see the logs in ElasticSearch, I use now the
following: (it is indeed json file not logstash).
module(load="imfile")module(load="mmjsonparse")module(load="omelasticsearch")
template(name="logstash-index" type="list") { constant(value="logstash-")
property(name="timereported" dateFormat="rfc3339" position.from="1"
position.to="4") constant(value=".") property(name="timereported"
dateFormat="rfc3339" position.from="6" position.to="7") constant(value=".")
property(name="timereported" dateFormat="rfc3339" position.from="9"
position.to="10")}
input(type="imfile" File="/var/log/nginx/access.json" Tag="nginxulyaoth"
PersistStateInterval="10000" StateFile="nginxulyaoth"
Severity="info" MaxSubmitAtOnce="20000" Facility="user"
Ruleset="nginxrule")
template(name="uly-nginx" type="list") { constant(value="{")
constant(value="\"@timestamp\":\"") property(name="timereported"
dateFormat="rfc3339") constant(value="\",\"host\":\"")
property(name="hostname") constant(value="\",\"severity\":\"")
property(name="syslogseverity-text") constant(value="\",\"facility\":\"")
property(name="syslogfacility-text") constant(value="\",\"tag\":\"")
property(name="syslogtag" format="json") constant(value="\",\"message\":\"")
property(name="msg" format="json") constant(value="\"}")}
ruleset(name="nginxrule") { action(type="mmjsonparse" name="jsonparse")
action(type="omelasticsearch" server="loghost.ulyaoth.net" serverport="9200"
template="uly-nginx" searchIndex="logstash-index" dynSearchIndex="on"
errorFile="/var/log/rsyslog/ES-error.log")}
This is what the debug does say, so it looks all correct.
8566.967355859:7fa3b2bbe700: omelasticsearch: beginTransaction
8566.967361436:7fa3b2bbe700: Action 0x7fa3bde6cd90 transitioned to state:
itx8566.967366495:7fa3b2bbe700: entering actionCalldoAction(), state:
itx8566.968573170:7fa3b2bbe700: omelasticsearch: pData replyLen =
'103'8566.968588657:7fa3b2bbe700: omelasticsearch: pData reply:
'{"_index":"logstash-index","_type":"events","_id":"OvpqDR7WT4uPosXVV2n74Q","_version":1,"created":true}'8566.968636314:7fa3b2bbe700:
omelasticsearch: error record: '{ "request": {
"url": "http://10.8.153.71:9200/logstash-index/events?";,
"postdata":
"{\"@timestamp\":\"2014-10-08T10:35:55.943964+02:00\",\"host\":\"loghost\",\"severity\":\"info\",\"facility\":\"user\",\"tag\":\"nginxulyaoth\",\"message\":\"{
\\\"@timestamp\\\": \\\"2014-10-08T10:35:55+02:00\\\", \\\"message\\\":
\\\"127.0.0.1 - admin [08/Oct/2014:10:38:47 +0200] \\\\\\\"GET /__status
HTTP/1.1\\\\\\\" 200 1443 \\\\\\\"-\\\\\\\" \\\\\\\"Go 1.1 package
http\\\\\\\"\\\", \\\"tags\\\": [\\\"nginx_acce
ss\\\"], \\\"realip\\\": \\\"\\\"127.0.0.1\\\", \\\"proxyip\\\": \\\"-\\\",
\\\"remote_user\\\": \\\"admin\\\", \\\"contenttype\\\":
\\\"application/json\\\", \\\"bytes\\\": 1443, \\\"duration\\\": \\\"0.012\\\",
\\\"status\\\": \\\"200\\\", \\\"request\\\": \\\"GET /__status HTTP/1.1\\\",
\\\"method\\\": \\\"GET\\\", \\\"referrer\\\": \\\"-\\\", \\\"useragent\\\":
\\\"Go 1.1 package http\\\" }\"}" }, "reply": {
"_index": "logstash-index", "_type": "events",
"_id": "OvpqDR7WT4uPosXVV2n74Q", "_version":
1, "created": true }}'8566.968711815:7fa3b2bbe700:
omelasticsearch: result doAction: 0 (bulkmode 0)8566.968718959:7fa3b2bbe700:
Action 0x7fa3bde6cd90 transitioned to state: rdy
It now seems to place the full json string as one line "message" in
kibana:message:
{ "@timestamp": "2014-10-08T10:35:55+02:00", "message": "127.0.0.1 - -
[08/Oct/2014:10:35:55 +0200] \"GET /test HTTP/1.1\" 200 84 \"-\"
\"curl/7.30.0\"", "tags": ["nginx_access"], "realip": ""127.0.0.1", "proxyip":
"-", "remote_user": "-", "contenttype": "application/json; charset=utf-8",
"bytes": 84, "duration": "0.006", "status": "200", "request": "GET /test
HTTP/1.1", "method": "GET", "referrer": "-", "useragent": "curl/7.30.0" }
_______________________________________________
rsyslog mailing list
http://lists.adiscon.net/mailman/listinfo/rsyslog
http://www.rsyslog.com/professional-services/
What's up with rsyslog? Follow https://twitter.com/rgerhards
NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad of
sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you DON'T LIKE
THAT.
