Yuvipanda has submitted this change and it was merged.
Change subject: logstash: Forward syslog events for apache2 + hhvm
......................................................................
logstash: Forward syslog events for apache2 + hhvm
Forward syslog events from apache2 and hhvm processes to a logstash
server if configured to do so via hiera. Forwards the same events that
would normally be sent to a udp2log relay directly to logstash.
Using the syslog input to logstash and rsylog as the transfer agent from
the individual MediaWiki hosts moves us closer to being able to
discontinue the use of log2udp forwarding to get events into logstash.
Bug: T76119
Change-Id: I5e8ea2b9c917ced306631c8092ce3149920a28f1
---
M files/logstash/filter-syslog.conf
A hieradata/eqiad/mediawiki.yaml
M modules/mediawiki/manifests/init.pp
M modules/mediawiki/templates/rsyslog.conf.erb
4 files changed, 68 insertions(+), 3 deletions(-)
Approvals:
Yuvipanda: Looks good to me, approved
Giuseppe Lavagetto: Looks good to me, but someone else must approve
jenkins-bot: Verified
diff --git a/files/logstash/filter-syslog.conf
b/files/logstash/filter-syslog.conf
index d173d2a..24c6093 100644
--- a/files/logstash/filter-syslog.conf
+++ b/files/logstash/filter-syslog.conf
@@ -1,8 +1,52 @@
filter {
if [type] == "syslog" {
+ # General syslog message cleanup
mutate {
- # tag syslog messages for storage in elasticsearch
- add_tag => [ "es" ]
+ replace => [
+ "type", "%{program}",
+ "host", "%{logsource}"
+ ]
+ add_field => { "level" => "%{severity_label}" }
+ # "\n" newline notation in substitution results in "\\n" in output.
+ # Using a string with a literal newline works as desired.
+ gsub => [ "message", "#012", '
+' ]
+ add_tag => [ "syslog", "es" ]
+ }
+
+ # Strip "message repeated" preamble
+ if [message] =~ /^message repeated \d+ times:/ {
+ grok {
+ match => [
+ "message",
+ "^message repeated %{NUMBER:repeated} times:
\[%{GREEDYDATA:message}\]$"
+ ]
+ overwrite => [ "message" ]
+ named_captures_only => true
+ }
+ }
+
+ # Strip leading newline from hhvm messages
+ if [type] == "hhvm" {
+ mutate {
+ gsub => [ "message", "^\n", "" ]
+ }
+ }
+
+ # Mark kernel messages forwared because of hhvm as hhvm messages
+ if [type] == "kernel" and [message] =~ /hhvm/ {
+ mutate {
+ replace => [ "type", "hhvm" ]
+ }
+ }
+
+ if [type] == "hhvm-fatal" {
+ # Join sequential lines into a single event
+ multiline {
+ pattern => "^Host: "
+ negate => true
+ what => "previous"
+ }
}
}
}
diff --git a/hieradata/eqiad/mediawiki.yaml b/hieradata/eqiad/mediawiki.yaml
new file mode 100644
index 0000000..d1b5ef5
--- /dev/null
+++ b/hieradata/eqiad/mediawiki.yaml
@@ -0,0 +1,3 @@
+# TODO: setup something to loadbalance the logstash hosts rather than relying
+# on a single node in the cluster.
+forward_syslog: "logstash1001.eqiad.wmnet:10514"
diff --git a/modules/mediawiki/manifests/init.pp
b/modules/mediawiki/manifests/init.pp
index b3fa32c..4e94cd8 100644
--- a/modules/mediawiki/manifests/init.pp
+++ b/modules/mediawiki/manifests/init.pp
@@ -12,8 +12,12 @@
# [*log_aggregator*]
# Udp2log aggregation server to send logs to. Default 'udplog:8420'.
#
+# [*forward_syslog*]
+# Host and port to forward syslog events to. Default undef (no forwarding).
+#
class mediawiki (
$log_aggregator = 'udplog:8420',
+ $forward_syslog = undef,
) {
include ::mediawiki::cgroup
include ::mediawiki::packages
diff --git a/modules/mediawiki/templates/rsyslog.conf.erb
b/modules/mediawiki/templates/rsyslog.conf.erb
index e7b4b0b..d5a74df 100644
--- a/modules/mediawiki/templates/rsyslog.conf.erb
+++ b/modules/mediawiki/templates/rsyslog.conf.erb
@@ -13,16 +13,30 @@
:programname, isequal, "apache2" :omfile:$apache2
& @<%= scope['::mediawiki::log_aggregator'] %>;MediaWiki
+<% if scope['::mediawiki::forward_syslog'] -%>
+& @<%= scope['::mediawiki::forward_syslog'] %>
+<% end -%>
& ~
# Forward HHVM logs and stack traces to log aggregator.
:programname, isequal, "hhvm" @<%= scope['::mediawiki::log_aggregator']
%>;MediaWiki
+<% if scope['::mediawiki::forward_syslog'] -%>
+& @<%= scope['::mediawiki::forward_syslog'] %>
+<% end -%>
& ~
:syslogtag, isequal, "hhvm-fatal:" @<%= scope['::mediawiki::log_aggregator']
%>;MediaWiki
+<% if scope['::mediawiki::forward_syslog'] -%>
+& @<%= scope['::mediawiki::forward_syslog'] %>
+<% end -%>
& ~
# Forward messages logged by the kernel and containing the string
# "hhvm" (such as warnings that the process was killed or respawned)
# to the log aggregator.
-if $msg contains "hhvm" and $programname == "kernel" then @<%=
scope['::mediawiki::log_aggregator'] %>;MediaWiki
+if $msg contains "hhvm" and $programname == "kernel" then {
+ @<%= scope['::mediawiki::log_aggregator'] %>;MediaWiki
+<% if scope['::mediawiki::forward_syslog'] -%>
+ @<%= scope['::mediawiki::forward_syslog'] %>
+<% end -%>
+}
--
To view, visit https://gerrit.wikimedia.org/r/176693
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I5e8ea2b9c917ced306631c8092ce3149920a28f1
Gerrit-PatchSet: 12
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BryanDavis <[email protected]>
Gerrit-Reviewer: BryanDavis <[email protected]>
Gerrit-Reviewer: Faidon Liambotis <[email protected]>
Gerrit-Reviewer: Gage <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: Yuvipanda <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
