OS: Windows 7 SP1 32 bit LogStash: 1.4.2 Java: jdk1.7.0_51 I have 2 files, app1.log and app2.log (attached). My config file (attached) uses the following input:
input {
file {
path => [
"C:/elk/app*.log"
]
start_position => "beginning"
sincedb_path => "C:/elk/.sincedb"
}
}
When processing these two files separately everything works as expected
(see app1-output.txt and app2-output.txt).
If I attempt to process them together or if I do not delete .sincedb after
processing app1.log I get an issue processing app2.log where it appears
that LogStash is 'dropping' the first few lines of the file (see below).
The problem can also be reproduced if referring to the files explicitly...
path => [ "C:/elk/app1.log", "C:/elk/app2.log" ]
I'm checking that I'm not doing something stupid before raising as a bug.
Many thanks, Chris
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:00.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app1.log",
"message" => "Starting...\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:01.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app1.log",
"message" => "Started\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:02.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app1.log",
"message" => "Listening...\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-09T00:00:00.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app1.log",
"message" => "Command received [shutdown]\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-09T00:00:01.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app1.log",
"message" => "Shutting down...\r"
}
*{*
* "message" => "\r",*
* "@version" => "1",*
* "@timestamp" => "2014-09-09T14:45:32.364Z",*
* "host" => "LIVLT0111295",*
* "path" => "C:/elk/app2.log",*
* "tags" => [*
* [0] "_grokparsefailure"*
* ]*
*}*
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:04.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app2.log",
"message" => "Registering 1 listeners(s)\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:05.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app2.log",
"message" => "Registered listeners #1\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:06.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app2.log",
"message" => "Registration complete successfully\r"
}
--
You received this message because you are subscribed to the Google Groups
"elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To view this discussion on the web visit
https://groups.google.com/d/msgid/elasticsearch/637e110e-f3b8-4a5b-8e73-c42363ab6cc1%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:00.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app1.log",
"message" => "Starting...\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:01.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app1.log",
"message" => "Started\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:02.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app1.log",
"message" => "Listening...\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-09T00:00:00.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app1.log",
"message" => "Command received [shutdown]\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-09T00:00:01.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app1.log",
"message" => "Shutting down...\r"
}
app2.log
Description: Binary data
app1.log
Description: Binary data
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:00.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app2.log",
"message" => "Starting initialization\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:01.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app2.log",
"message" => "Initialization completed successfully\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:02.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app2.log",
"message" => "Scanning for listeners\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:03.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app2.log",
"message" => "Identified 1 listener(s)\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:04.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app2.log",
"message" => "Registering 1 listeners(s)\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:05.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app2.log",
"message" => "Registered listeners #1\r"
}
{
"@version" => "1",
"@timestamp" => "2014-09-08T23:00:06.000Z",
"host" => "LIVLT0111295",
"path" => "C:/elk/app2.log",
"message" => "Registration complete successfully\r"
}input {
file {
path => [
"C:/elk/app*.log"
]
start_position => "beginning"
sincedb_path => "C:/elk/.sincedb"
}
}
filter {
grok {
match => [ "message", "(?<datetime>\d\d\d\d-\d\d-\d\d
\d\d:\d\d:\d\d\.\d\d\d\+\d\d\d\d)\s+(?<text>.*)" ]
}
if "_grokparsefailure" not in [tags] {
date {
match => [ "datetime", "YYYY-MM-dd HH:mm:ss.SSSZ" ]
}
mutate {
remove_field => [ "message", "datetime" ]
}
mutate {
rename => [ "text", "message" ]
}
}
}
output {
stdout {
codec => rubydebug
}
}
