BryanDavis has uploaded a new change for review.
https://gerrit.wikimedia.org/r/169728
Change subject: logstash: reformat gelf filter config
......................................................................
logstash: reformat gelf filter config
Add whitespace and hoist comments out of filter blocks to make this
config look more like the udp2log config.
Change-Id: I649eecab2381b5c8f72b8e283faae065b6fe6fed
---
M files/logstash/filter-gelf.conf
1 file changed, 32 insertions(+), 21 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/28/169728/1
diff --git a/files/logstash/filter-gelf.conf b/files/logstash/filter-gelf.conf
index 33be766..ed39bb4 100644
--- a/files/logstash/filter-gelf.conf
+++ b/files/logstash/filter-gelf.conf
@@ -1,25 +1,30 @@
+# vim:set sw=2 ts=2 sts=2 et
+# Process GELF log output from various sources
filter {
+
if [type] == "gelf" {
+ # Tag for storage in elasticsearch and that GELF was the input protocol.
mutate {
- # tag gelf messages for storage in elasticsearch
add_tag => [ "es", "gelf" ]
}
+
+ # Convert GELF's 'facility' to Logstash's 'type'.
+ # Convert GELF's 'short_message' to Logstash's 'message'.
mutate {
- # move field names to what we're expecting
replace => [ "type", "%{facility}" ]
replace => [ "message", "%{short_message}" ]
- }
- mutate {
- # get rid of fields we moved
remove_field => [ "facility", "short_message" ]
}
- if [facility] == "Hadoop" {
+
+ if [type] == "Hadoop" {
mutate {
replace => [ "channel", "%{SourceSimpleClassName}" ]
}
+
+ # Overwrite the Thread field with a shorter more generic version, saving
+ # details to separate fields
grok {
# Oniguruma syntax for ‘named capture’: (?<field_name>the pattern here)
- # this overwrites the Thread field with a shorter more generic
version, saving details to separate fields
match => [ "Thread", "(?<Thread>IPC Server handler)
%{NUMBER:IPC_Server_handler_id} on %{NUMBER:IPC_Server_handler_port}" ]
match => [ "Thread", "(?<Thread>DeletionService)
#%{NUMBER:DeletionService_id}" ]
match => [ "Thread", "(?<Thread>LocalizerRunner) for
%{NOTSPACE:ContainerId}" ]
@@ -28,57 +33,63 @@
match => [ "Thread",
"(?<Thread>CacheReplicationMonitor)\(%{NUMBER:CacheReplicationMonitor_id}\)" ]
overwrite => [ "Thread" ]
}
+
if [message] =~ /attempt_/ {
+ # Extract attempt ID to field: attempt_1409078537822_52431_m_000009_1,
attempt_1409078537822_55176_r_000000_0
grok {
- # so that we can search by job id and find tasks and attempts:
- # extract attempt ID to field:
attempt_1409078537822_52431_m_000009_1, attempt_1409078537822_55176_r_000000_0
- match => [ "message",
"attempt_(?<attempt_id>[0-9]+_[0-9]+_[mr]_[0-9]+_[0-9]+)" ]
+ match => [ "message",
"attempt_(?<attempt_id>[0-9]+_[0-9]+_[mr]_[0-9]+_[0-9]+)" ]
tag_on_failure => ["_grokparsefailure_attempt"]
}
}
if [message] =~ /task_/ {
+ # Extract task ID to field: task_1409078537822_52431_m_000044
grok {
- # extract task ID to field: task_1409078537822_52431_m_000044
- match => [ "message",
"task_(?<task_id>[0-9]+_[0-9]+_[mr]_[0-9]+)" ]
+ match => [ "message", "task_(?<task_id>[0-9]+_[0-9]+_[mr]_[0-9]+)" ]
tag_on_failure => ["_grokparsefailure_task"]
}
}
if [message] =~ /job_/ {
+ # Extract job ID to field: job_1409078537822_52431
grok {
- # extract job ID to field: job_1409078537822_52431
- match => [ "message", "job_(?<job_id>[0-9]+_[0-9]+)" ]
+ match => [ "message", "job_(?<job_id>[0-9]+_[0-9]+)" ]
tag_on_failure => ["_grokparsefailure_job"]
}
}
+
if [task_id] != "" {
+ # Infer task ID from attempt ID
grok {
- # infer task ID from attempt ID
match => [ "attempt_id",
"(?<task_id>[0-9]+_[0-9]+_[mr]_[0-9]+)_[0-9]+" ]
tag_on_failure => ["_grokparsefailure_infer_task"]
}
}
+
if [job_id] != "" {
+ # Infer job ID from task ID
grok {
- # infer job ID from task ID
- match => [ "task_id", "(?<job_id>[0-9]+_[0-9]+)_[mr]_[0-9]+" ]
+ match => [ "task_id", "(?<job_id>[0-9]+_[0-9]+)_[mr]_[0-9]+" ]
tag_on_failure => ["_grokparsefailure_infer_job"]
}
}
+
dns {
reverse => [ "host" ]
action => "replace"
}
- }
+
+ } # end [type] == "Hadoop"
# Drop duplicated json content added by nodejs gelf-stream library
mutate {
remove_field => [ "full_message" ]
}
+ # Get rid of a whole bunch of useless _* fields that are duplicated.
+ # We must keep _id, _index, and _type
prune {
- # get rid of a whole bunch of useless _ fields that are duplicated
- # We must keep _id, _index, and _type
blacklist_names => [ "^_(?!id$|index$|type$).*" ]
}
- }
+
+ } # end [type] == "gelf"
+
}
--
To view, visit https://gerrit.wikimedia.org/r/169728
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I649eecab2381b5c8f72b8e283faae065b6fe6fed
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BryanDavis <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
