BryanDavis has uploaded a new change for review.
https://gerrit.wikimedia.org/r/112149
Change subject: logstash: Add normalized_message field to all events
......................................................................
logstash: Add normalized_message field to all events
Copy the message of all events destined for storage in Elasticsearch
into a "normalized_message" field that is truncated to 255 characters.
This can be used in dashboards as a term search to correlate common
messages.
Change-Id: I01c50456cf0e334075acacbe2aebe5d8fc941d31
---
A files/logstash/filter-add-normalized-message.conf
M manifests/role/logstash.pp
2 files changed, 40 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/49/112149/1
diff --git a/files/logstash/filter-add-normalized-message.conf
b/files/logstash/filter-add-normalized-message.conf
new file mode 100644
index 0000000..b77662a
--- /dev/null
+++ b/files/logstash/filter-add-normalized-message.conf
@@ -0,0 +1,34 @@
+# vim:set sw=2 ts=2 sts=2 et
+# Add normalized_message field to events bound for logstash
+filter {
+
+ if "es" in [tags] and ![normalized_message] {
+ mutate {
+ # Create a copy of message field that can be normalized
+ add_field => [ "normalized_message", "%{message}" ]
+ }
+ # Remove documentation anchor tags
+ mutate {
+ gsub => [
+ "normalized_message",
+ " \[<a href='[^']*'>[^<]*</a>\]",
+ ""
+ ]
+ }
+ # Trim the normalized_message to a maximum of 255 characters
+ # This is done because our Elasticsearch schema doesn't store raw fields
+ # for strings longer than 255 characters and we want something to show
+ # in terms queries even if it's shortened.
+ grok {
+ match => [
+ "normalized_message",
+ "^(?<normalized_message>.{255}).*$"
+ ]
+ overwrite => [ "normalized_message" ]
+ named_captures_only => true
+ add_tag => [ "normalized_message_trimmed" ]
+ tag_on_failure => [ "normalized_message_untrimmed" ]
+ }
+ }
+
+}
diff --git a/manifests/role/logstash.pp b/manifests/role/logstash.pp
index d17f25a..2ad37d2 100644
--- a/manifests/role/logstash.pp
+++ b/manifests/role/logstash.pp
@@ -57,7 +57,7 @@
logstash::conf { 'filter_strip_ansi_color':
source => 'puppet:///files/logstash/filter-strip-ansi-color.conf',
- priority => 50,
+ priority => 40,
}
logstash::conf { 'filter_syslog':
@@ -70,6 +70,11 @@
priority => 50,
}
+ logstash::conf { 'filter_add_normalized_message':
+ source =>
'puppet:///files/logstash/filter-add-normalized-message.conf',
+ priority => 60,
+ }
+
class { '::logstash::output::elasticsearch':
host => '127.0.0.1',
replication => 'async',
--
To view, visit https://gerrit.wikimedia.org/r/112149
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I01c50456cf0e334075acacbe2aebe5d8fc941d31
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BryanDavis <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
