Hi, maybe some people are interested, i wrote some Grok patterns and logstash filters to inject Bareos logs in logstash for Kibana use. I'm currently using logstash 1.5 so ruby filter is using old syntax. It's probably not optimum and needs review. It's based on the "Examine Bareos Logs" OSB conf by Daniel Neuberger ( https://www.youtube.com/watch?v=hNBnrYSJL1U) and the old bacula grok patterns (https://github.com/elastic/logstash/blob/v1.4.2/patterns/grok-patterns).
Please if you have any suggestions , fell free to respond ! Best, -- You received this message because you are subscribed to the Google Groups "bareos-users" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. To post to this group, send email to [email protected]. For more options, visit https://groups.google.com/d/optout.
bareos_grok
Description: Binary data
filter {
if [type] == "bareos-log" and [message] == "Retrying ..." { drop {} }
if [type] == "bareos-log" and [message] !~ /.*Build OS.*/ { # and "multiline"
not in [tags] {
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BAREOS_LOGLINE}"]
add_tag => ["bareos_grok"]
tag_on_failure => ["_failed_grok_for_bareos"]
break_on_match => true
}
date {
match => ["bareos_timestamp","dd-MMM HH:mm"]
target => "@timestamp"
}
}
if [type] == "bareos-log" and [message] =~ /.*Build OS.*/ { # and "multiline"
in [tags] {
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BAREOS_TIMESTAMP:bareos_timestamp}
%{BAREOS_HOST} JobId %{INT}:.*"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BUILDOS}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPJOBID}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPPREVJOBID}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPJOB}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPPREVJOB}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPLEVEL}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPCLIENT}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPFILESET}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPPOOL}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPREADPOOL}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPWRITEPOOL}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPNEXTPOOL}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPCATALOG}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPSTORAGE}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPREADSTORAGE}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPWRITESTORAGE}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPSCHEDTIME}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPSTARTTIME}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPENDTIME}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPELAPSEDTIME}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPPRIORITY}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPFDFILES}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPSDFILES}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPFDBYTES}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPSDBYTES}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPRATE}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPCOMPRESSION}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPVSS}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPENCRYPTION}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPACCURATE}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPVOLNAME}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPVOLSESID}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPVOLSESTIME}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPLASTVOLBYTES}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPNONFATALFDERROR}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPSDERROR}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPFDTERMINATION}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPSDTERMINATION}"]
}
grok {
patterns_dir => "/opt/logstash/patterns"
match => ["message","%{BACKUPTERMINATION}"]
}
mutate {
add_field => { "bareos_job_elapsed_time" => 0 }
convert => { "bareos_job_elapsed_time" => "integer" }
}
ruby {
code => "
if event['bareos_job_elapsed_day'] == nil
elapsed_day = 0 ;
else
elapsed_day = event['bareos_job_elapsed_day'].to_i ;
end
if event['bareos_job_elapsed_hour'] == nil
elapsed_hour = 0 ;
else
elapsed_hour = event['bareos_job_elapsed_hour'].to_i ;
end
if event['bareos_job_elapsed_min'] == nil
elapsed_min = 0 ;
else
elapsed_min = event['bareos_job_elapsed_min'].to_i ;
end
if event['bareos_job_elapsed_sec'] == nil
elapsed_sec = 0 ;
else
elapsed_sec = event['bareos_job_elapsed_sec'].to_i ;
end
event['bareos_job_elapsed_time']=(elapsed_day*86400+elapsed_hour*3600+elapsed_min*60+elapsed_sec).to_i
;"
}
mutate {
add_tag => ["bareos_multi_grok"]
gsub => [
"bareos_fd_byte_written" , "," , "",
"bareos_fd_file_written" , "," , "",
"bareos_sd_byte_written" , "," , "",
"bareos_sd_file_written" , "," , "",
"bareos_job_last_volbyte" , "," , ""
]
# convert => { "bareos_fd_byte_written" => "integer" }
# convert => { "bareos_sd_byte_written" => "integer" }
# convert => { "bareos_job_last_volbyte" => "integer" }
}
date {
match => ["bareos_timestamp","dd-MMM HH:mm"]
target => "@timestamp"
}
}
}
