BryanDavis has uploaded a new change for review.
https://gerrit.wikimedia.org/r/278315
Change subject: logstash: Make truncated MediaWiki json easier to find
......................................................................
logstash: Make truncated MediaWiki json easier to find
Some MediaWiki events sent to Logstash get mangled because their JSON
payload exceeds the length limit to a syslog UDP datagram. These
messages can be found today classified as syslog traffic. This change
would instead classify them as MediaWiki traffic and put them in a log
channel that can be searched and filtered easily.
Change-Id: I0b8cb3d37e0a678f3685d32bfa1c3fd6b30363df
---
M files/logstash/filter-syslog.conf
1 file changed, 26 insertions(+), 16 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/15/278315/1
diff --git a/files/logstash/filter-syslog.conf
b/files/logstash/filter-syslog.conf
index b8ceb8e..429e8ad 100644
--- a/files/logstash/filter-syslog.conf
+++ b/files/logstash/filter-syslog.conf
@@ -115,25 +115,35 @@
} # end [program] == "apache2"
- if [program] == "mediawiki" and [message] =~ /^{.*}$/ {
+ if [program] == "mediawiki" {
mutate {
replace => [ "type", "mediawiki" ]
- # Remove syslog added fields
- remove_field => [
- "facility",
- "facility_label",
- "logsource",
- "priority",
- "program",
- "severity",
- "severity_label",
- "timestamp"
- ]
}
- # Parse message as json to unpack logstash record
- json {
- source => "message"
+ if [message] =~ /^{.*}$/ {
+ mutate {
+ # Remove syslog added fields
+ remove_field => [
+ "facility",
+ "facility_label",
+ "logsource",
+ "priority",
+ "program",
+ "severity",
+ "severity_label",
+ "timestamp"
+ ]
+ }
+ # Parse message as json to unpack logstash record
+ json {
+ source => "message"
+ }
+ } else {
+ # Mark up the message as JSON that was cut off by the syslog transport
+ mutate {
+ add_field => { "channel" => "jsonTruncated" }
+ add_tag => [ "syslog_truncated" ]
+ }
}
- } # end [program] == "mediawiki" and [message] =~ /^{.*}$/
+ } # end [program] == "mediawiki"
}
}
--
To view, visit https://gerrit.wikimedia.org/r/278315
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I0b8cb3d37e0a678f3685d32bfa1c3fd6b30363df
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BryanDavis <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
