Ottomata has submitted this change and it was merged.
Change subject: Sending nginx logs to their own udp2log instance on gadolinium.
......................................................................
Sending nginx logs to their own udp2log instance on gadolinium.
Change-Id: I5152521ab0eed37fbf7122804438bf1a23346a16
---
M manifests/role/logging.pp
M templates/nginx/nginx.conf.erb
M templates/udp2log/filters.gadolinium.erb
A templates/udp2log/filters.nginx.erb
4 files changed, 59 insertions(+), 32 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
jenkins-bot: Verified
diff --git a/manifests/role/logging.pp b/manifests/role/logging.pp
index ee36217..0b09e32 100644
--- a/manifests/role/logging.pp
+++ b/manifests/role/logging.pp
@@ -72,36 +72,56 @@
class role::logging::udp2log {
include misc::udp2log,
misc::udp2log::utilities
-}
-# gadolinium udp2log instance(s)
-class role::logging::udp2log::gadolinium inherits role::logging::udp2log {
- # need file_mover account for fundraising logs
- include accounts::file_mover
-
- $log_directory = '/a/log'
- $webrequest_log_directory = "$log_directory/webrequest"
- $fundraising_log_directory = "$log_directory/fundraising"
+ $log_directory = '/a/log'
+ $webrequest_log_directory = "$log_directory/webrequest"
+ $webrequest_filter_directory = "$webrequest_log_directory/bin"
file { $log_directory:
ensure => 'directory',
}
-
- # install custom filters
- file { "$webrequest_log_directory/bin":
+ file { $webrequest_log_directory:
ensure => directory,
mode => 0755,
owner => 'udp2log',
group => 'udp2log',
}
- file { "$webrequest_log_directory/bin/vu.awk":
+
+ # install custom filters here
+ file { $webrequest_filter_directory:
+ ensure => directory,
+ mode => 0755,
+ owner => 'udp2log',
+ group => 'udp2log',
+ }
+
+ # Set up an rsync daemon module for udp2log logrotated
+ # archives. This allows stat1 to copy logs from the
+ # logrotated archive directory
+ class { 'misc::udp2log::rsyncd':
+ path => $log_directory,
+ require => File[$log_directory],
+ }
+}
+
+
+# gadolinium udp2log instance(s).
+# gadolinium hosts the 'gadolinium' udp2log instance,
+# as well as the nginx udp2log instance.
+class role::logging::udp2log::gadolinium inherits role::logging::udp2log {
+ # need file_mover account for fundraising logs
+ include accounts::file_mover
+ # gadolinium keeps fundraising logs in a subdir
+ $fundraising_log_directory = "$log_directory/fundraising"
+
+ file { "$webrequest_filter_directory/vu.awk":
ensure => 'file',
source => 'puppet:///files/udp2log/vu.awk',
mode => 0755,
owner => 'udp2log',
group => 'udp2log',
}
- file { "$webrequest_log_directory/bin/minnesota.awk":
+ file { "$webrequest_filter_directory/minnesota.awk":
ensure => 'file',
source => 'puppet:///files/udp2log/minnesota.awk',
mode => 0755,
@@ -109,7 +129,6 @@
group => 'udp2log',
}
- # gadolinium keeps fundraising logs in a subdir
file { "$fundraising_log_directory":
ensure => directory,
mode => 0775,
@@ -132,15 +151,20 @@
misc::udp2log::instance { 'gadolinium':
# gadolinium consumes from the multicast stream relay (from
oxygen)
multicast => true,
- log_directory => $log_directory,
- require => File['/a/log'],
+ log_directory => $webrequest_log_directory,
+ require => File[$webrequest_log_directory],
}
- # Set up an rsync daemon module for udp2log logrotated
- # archives. This allows stat1 to copy logs from the
- # logrotated archive directory
- class { 'misc::udp2log::rsyncd':
- path => $log_directory,
- require => Misc::Udp2log::Instance['gadolinium'],
+ # nginx machines are configured to log to
+ # gadolinium on port 8421.
+ # Since nginx logs are webrequest logs, save
+ # them in the same directory.
+ udp2log::instance { 'nginx':
+ port => '8421',
+ log_directory => $webrequest_log_directory,
+ require => File[$webrequest_log_directory],
+ # the gadolinium udp2log instance already
+ # log rotates for $webrequest_log_directory
+ log_rotate => false,
}
}
diff --git a/templates/nginx/nginx.conf.erb b/templates/nginx/nginx.conf.erb
index 1abf2b3..5c94483 100644
--- a/templates/nginx/nginx.conf.erb
+++ b/templates/nginx/nginx.conf.erb
@@ -29,8 +29,7 @@
access_log /var/log/nginx/access.log;
<% if has_variable?("nginx_use_ssl") then %>
log_format squid_combined '$hostname $udplog_sequence
$udplog_time $request_time $remote_addr -/$status $bytes_sent
$request_method $scheme://$host$uri NONE/$proxy_host $content_type
$http_referer $http_x_forwarded_for $udplog_escaped_user_agent
$http_accept_language $http_x_cs';
- access_udplog 208.80.152.138:8420 squid_combined;
- access_udplog 208.80.152.184:8420 squid_combined;
+ access_udplog 208.80.154.73:8421 squid_combined;
client_max_body_size 100m;
large_client_header_buffers 4 16k;
<% end %>
diff --git a/templates/udp2log/filters.gadolinium.erb
b/templates/udp2log/filters.gadolinium.erb
index 28d948c..811ac0b 100644
--- a/templates/udp2log/filters.gadolinium.erb
+++ b/templates/udp2log/filters.gadolinium.erb
@@ -7,22 +7,22 @@
### 0.0001 of all udp2log messages
# This log file is also on emery for redundancy
-file 1000 <%= webrequest_log_directory %>/sampled-1000.log.tsv
+file 1000 <%= webrequest_log_directory %>/sampled-1000.tsv.log
### All edits
-pipe 1 /usr/bin/udp-filter -F '\t' -p action=submit,action=edit >> <%=
webrequest_log_directory %>/edits.log.tsv
+pipe 1 /usr/bin/udp-filter -F '\t' -p action=submit,action=edit >> <%=
webrequest_log_directory %>/edits.tsv.log
### Mobile traffic filter
# TODO: use mobile varnish frontend hostnames for
# filtering rather than 'm.wikipedia.org' domain.
-pipe 100 /usr/bin/udp-filter -F '\t' -d "m.wikipedia.org" >> <%=
webrequest_log_directory %>/mobile.log.tsv
+pipe 100 /usr/bin/udp-filter -F '\t' -d "m.wikipedia.org" >> <%=
webrequest_log_directory %>/mobile.tsv.log
### Fundraising
## Landing pages
-pipe 1 /usr/bin/udp-filter -F '\t' -d
wikimediafoundation.org,donate.wikimedia.org >> <%= fundraising_log_directory
%>/logs/landingpages.log.tsv
+pipe 1 /usr/bin/udp-filter -F '\t' -d
wikimediafoundation.org,donate.wikimedia.org >> <%= fundraising_log_directory
%>/logs/landingpages.tsv.log
## Banner Impressions
-pipe 100 /usr/bin/udp-filter -F '\t' -p
Special:RecordImpression\?banner=,Special:RecordImpression\?result= >> <%=
fundraising_log_directory %>/logs/bannerImpressions-sampled100.log.tsv
+pipe 100 /usr/bin/udp-filter -F '\t' -p
Special:RecordImpression\?banner=,Special:RecordImpression\?result= >> <%=
fundraising_log_directory %>/logs/bannerImpressions-sampled100.tsv.log
@@ -47,6 +47,6 @@
# pipe 1 /a/webstats/bin/filter | log2udp -h 127.0.0.1 -p 3815
### All 5xx error responses -- domas
-# TODO: /usr/bin/udp-filter -F '\t' -r -s '^5' | awk -W interactive '$9 !~
"upload.wikimedia.org|query.php"' >> <%= webrequest_log_directory %>/5xx.log.tsv
-# pipe 1 <%= webrequest_log_directory %>/5xx-filter | awk -W interactive '$9
!~ "upload.wikimedia.org|query.php"' >> <%= webrequest_log_directory
%>/5xx.log.tsv
+# TODO: /usr/bin/udp-filter -F '\t' -r -s '^5' | awk -W interactive '$9 !~
"upload.wikimedia.org|query.php"' >> <%= webrequest_log_directory %>/5xx.tsv.log
+# pipe 1 <%= webrequest_log_directory %>/5xx-filter | awk -W interactive '$9
!~ "upload.wikimedia.org|query.php"' >> <%= webrequest_log_directory
%>/5xx.tsv.log
diff --git a/templates/udp2log/filters.nginx.erb
b/templates/udp2log/filters.nginx.erb
new file mode 100644
index 0000000..7164834
--- /dev/null
+++ b/templates/udp2log/filters.nginx.erb
@@ -0,0 +1,4 @@
+# Note: This file is managed by Puppet.
+
+# Save logs from nginx.
+file 1 <%= nginx_log_directory %>/nginx.tsv.log
--
To view, visit https://gerrit.wikimedia.org/r/55394
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I5152521ab0eed37fbf7122804438bf1a23346a16
Gerrit-PatchSet: 5
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
Gerrit-Reviewer: Diederik <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
