I am parsing a logfile using logstash. But somehow logstash is not parsing
whole log file
attaching the error dump.
I have also attached the my logstash config file. Please help
root@ryudt-023:/etc/logstash/
>
> conf.d# /opt/logstash/bin/logstash agent -f akamai-log.conf
> Using milestone 2 input plugin 'file'. This plugin should be stable, but
> if you see strange behavior, please let us know! For more information on
> plugin milestones, see
> http://logstash.net/docs/1.4.2-modified/plugin-milestones {:level=>:warn}
> Using milestone 2 filter plugin 'urldecode'. This plugin should be stable,
> but if you see strange behavior, please let us know! For more information
> on plugin milestones, see
> http://logstash.net/docs/1.4.2-modified/plugin-milestones {:level=>:warn}
> Using milestone 2 filter plugin 'json'. This plugin should be stable, but
> if you see strange behavior, please let us know! For more information on
> plugin milestones, see
> http://logstash.net/docs/1.4.2-modified/plugin-milestones {:level=>:warn}
> Trouble parsing json {:source=>"message",
> :raw=>"index.php\",\"reqQuery\":\"path=%2F..%2Fboot.ini............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................%00.&route=product%2Fcategory\",\"respCT\":\"text/html\",\"respLen\":\"286\",\"bytes\":\"286\",\"UA\":\"mozilla-earth\",\"fwdHost\":\"
> origin-demo2-akamaized.scoe-sil.net\"},\"reqHdr\":{\"accEnc\":\"gzip,
> deflate\",\"cookie\":\"PHPSESSID=no94vbt0q4hc33ncv9oeog16b3\"},\"respHdr\":{\"date\":\"Tue,
>
> 08 Jul 2014 22:14:44 GMT\",\"expires\":\"Tue, 08 Jul 2014 22:14:44
> GMT\",\"server\":\"AkamaiGHost\",\"setCookie\":\"\"},\"netPerf\":{\"downloadTime\":\"5\",\"lastMileRTT\":\"95\",\"cacheStatus\":\"0\",\"firstByte\":\"1\",\"lastByte\":\"1\",\"asnum\":\"12222\",\"edgeIP\":\"8.18.42.173\"},\"geo\":{\"country\":\"US\",\"region\":\"CA\",\"city\":\"SANFRANCISCO\"},\"waf\":{\"ver\":\"2.0\",\"policy\":\"qik1_12418\",\"ruleVer\":\"2.2.6\",\"mode\":\"nrm\",\"rsr\":\"0\",\"dor\":\"1\",\"oft\":\"0\",\"riskGroups\":\"\",\"riskTuples\":\"\",\"riskScores\":\"\",\"pAction\":\"\",\"pRate\":\"\",\"warnRules\":\"3000002\",\"warnData\":\"Ym9vdC5pbmk=\",\"warnSlrs\":\"ARGS:path\",\"denyRules\":\"950005\",\"denyData\":\"Ym9vdC5pbmk=\"}}",
>
> :exception=>#<JSON::ParserError: unexpected token at
> 'index.php","reqQuery":"path=%2F..%2Fboot.ini............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................%00.&route=product%2Fcategory","respCT":"text/html","respLen":"286","bytes":"286","UA":"mozilla-earth","fwdHost":"
> origin-demo2-akamaized.scoe-sil.net"},"reqHdr":{"accEnc":"gzip,
> deflate","cookie":"PHPSESSID=no94vbt0q4hc33ncv9oeog16b3"},"respHdr":{"date":"Tue,
>
> 08 Jul 2014 22:14:44 GMT","expires":"Tue, 08 Jul 2014 22:14:44
> GMT","server":"AkamaiGHost","setCookie":""},"netPerf":{"downloadTime":"5","lastMileRTT":"95","cacheStatus":"0","firstByte":"1","lastByte":"1","asnum":"12222","edgeIP":"8.18.42.173"},"geo":{"country":"US","region":"CA","city":"SANFRANCISCO"},"waf":{"ver":"2.0","policy":"qik1_12418","ruleVer":"2.2.6","mode":"nrm","rsr":"0","dor":"1","oft":"0","riskGroups":"","riskTuples":"","riskScores":"","pAction":"","pRate":"","warnRules":"3000002","warnData":"Ym9vdC5pbmk=","warnSlrs":"ARGS:path","denyRules":"950005","denyData":"Ym9vdC5pbmk="}}'>,
>
> :level=>:warn}
> Exception in filterworker {"exception"=>#<TypeError: can't convert Fixnum
> into String>, "backtrace"=>["org/jruby/RubyString.java:3898:in `[]='",
> "/opt/logstash/lib/logstash/util/accessors.rb:40:in `set'",
> "/opt/logstash/lib/logstash/event.rb:138:in `[]='",
> "/opt/logstash/lib/logstash/filters/mutate.rb:272:in `convert'",
> "org/jruby/RubyHash.java:1339:in `each'",
> "/opt/logstash/lib/logstash/filters/mutate.rb:255:in `convert'",
> "/opt/logstash/lib/logstash/filters/mutate.rb:209:in `filter'",
> "(eval):75:in `initialize'", "org/jruby/RubyProc.java:271:in `call'",
> "/opt/logstash/lib/logstash/pipeline.rb:262:in `filter'",
> "/opt/logstash/lib/logstash/pipeline.rb:203:in `filterworker'",
> "/opt/logstash/lib/logstash/pipeline.rb:143:in `start_filters'"],
> :level=>:error}
> log4j, [2014-09-12T12:22:06.304] WARN:
> org.elasticsearch.discovery.zen.ping.unicast:
> [logstash-ryudt-023-5023-4010] failed to send ping to
> [[#zen_unicast_5#][ryudt-023][inet[localhost/127.0.0.1:9304]]]
> org.elasticsearch.transport.ReceiveTimeoutTransportException:
> [][inet[localhost/127.0.0.1:9304]][discovery/zen/unicast] request_id [2]
> timed out after [3751ms]
> at
> org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:356)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> log4j, [2014-09-12T12:22:06.304] WARN:
> org.elasticsearch.discovery.zen.ping.unicast:
> [logstash-ryudt-023-5023-4010] failed to send ping to
> [[#zen_unicast_6#][ryudt-023][inet[localhost/127.0.0.1:9305]]]
> org.elasticsearch.transport.ReceiveTimeoutTransportException:
> [][inet[localhost/127.0.0.1:9305]][discovery/zen/unicast] request_id [5]
> timed out after [3750ms]
> at
> org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:356)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
--
You received this message because you are subscribed to the Google Groups
"elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To view this discussion on the web visit
https://groups.google.com/d/msgid/elasticsearch/647c756b-4913-44e3-85d2-c2583175932e%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
input {
file {
path => "/home/atul/Music/akamai.log"
start_position => beginning
}
}
filter {
urldecode {
field => "message"
}
json {
source => "message"
}
mutate {
convert => [
"[message][bytes]", "integer",
"[message][reqPort]", "integer",
"[message][respLen]", "integer",
"[netPerf][asnum]", "integer",
"[netPerf][cacheStatus]", "integer",
"[netPerf][downloadTime]", "integer",
"[netPerf][firstByte]", "integer",
"[netPerf][lastByte]", "integer",
"[netPerf][lastMileRTT]", "integer",
"[netPerf][netOriginLatency]", "integer"
]
}
geoip {
source => "[message][cliIP]"
target => "geoip"
database => "/home/atul/Downloads/GeoLiteCity.dat"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float" ]
}
date {
target => "respHdrDate"
match => [ "[respHdr][date]", "EEE, dd MMM yyyy HH:mm:ss z" ]
}
date {
target => "respHdrExpires"
match => [ "[respHdr][expires]", "EEE, dd MMM yyyy HH:mm:ss z" ]
}
useragent {
source => "[message][UI]"
target => "UserAgent"
}
}
output {
elasticsearch {
host => localhost
}
stdout { codec => rubydebug }
}
