Gehel has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/299825 )
Change subject: logstash: Parse nginx access logs for wdqs
......................................................................
logstash: Parse nginx access logs for wdqs
* Change type from syslog to wdqs
* Remove syslog fields
* Parse access log line and add channel = nginx
* Decode message for easier viewing
Co-Authored-By: Stanislav Malyshev <smalys...@gmail.com>
Change-Id: I30007949807099d811e197773ff25772cc5e1393
---
M modules/role/files/logstash/filter-syslog.conf
1 file changed, 46 insertions(+), 0 deletions(-)
Approvals:
jenkins-bot: Verified
Gehel: Looks good to me, approved
diff --git a/modules/role/files/logstash/filter-syslog.conf
b/modules/role/files/logstash/filter-syslog.conf
index 9710285..5fb0810 100644
--- a/modules/role/files/logstash/filter-syslog.conf
+++ b/modules/role/files/logstash/filter-syslog.conf
@@ -134,5 +134,51 @@
}
}
} # end [program] == "mediawiki"
+
+ if [program] == "wdqs" {
+ mutate {
+ replace => [ "type", "wdqs" ]
+ }
+
+ # nginx access logs
+ if [facility_label] == "local7" {
+ #
https://github.com/wikimedia/operations-puppet/blob/3218df6/modules/wdqs/templates/nginx.erb#L1-L6
+ grok {
+ match => [
+ "message",
+ "^\[%{HTTPDATE:http_date}\] .%{WORD:http_method}
%{NOTSPACE:message} HTTP/%{NUMBER:httpversion}. %{NUMBER:status}
(?:%{NUMBER:response_size}|-) %{QS:referrer} %{QS:user_agent}
%{NUMBER:request_time} %{NUMBER:upstream_time} %{IP:clientip}
%{IP:remote_addr}$"
+ ]
+ overwrite => [ "message" ]
+ named_captures_only => true
+ add_field => { "channel" => "nginx" }
+ }
+
+ if !("_grokparsefailure" in [tags]) {v
+ mutate {
+ add_field => {
+ "message_decoded" => "%{message}"
+ }
+ }
+
+ urldecode {
+ field => "message_decoded"
+ }
+ }
+
+ mutate {
+ # Remove syslog added fields
+ remove_field => [
+ "facility",
+ "facility_label",
+ "logsource",
+ "priority",
+ "program",
+ "severity",
+ "severity_label",
+ "timestamp"
+ ]
+ }
+ } # end [facility_label] == "local7"
+ } # end [program] == "wdqs"
}
}
--
To view, visit https://gerrit.wikimedia.org/r/299825
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I30007949807099d811e197773ff25772cc5e1393
Gerrit-PatchSet: 12
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BryanDavis <bda...@wikimedia.org>
Gerrit-Reviewer: BryanDavis <bda...@wikimedia.org>
Gerrit-Reviewer: Gehel <guillaume.leder...@wikimedia.org>
Gerrit-Reviewer: Smalyshev <smalys...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
