ArielGlenn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/379198 )
Change subject: Move datasets nginx logs rsync to dumps web manifest where it
belongs
......................................................................
Move datasets nginx logs rsync to dumps web manifest where it belongs
Only the dumps web server will ever have these logs, keep this rsync
job separate from the generic "grab this dataset" or "rsync these dumps"
jobs
Active web server does the rsync, fallback servers do not, restructure
manifests accordingly.
Bug: T175528
Change-Id: I08b8c024c1d901670795757351bcead8b78c8ccf
---
M hieradata/common.yaml
M hieradata/hosts/dataset1001.yaml
D hieradata/role/common/dumps/web/xmldumps.yaml
M manifests/site.pp
D modules/dataset/manifests/cron/rsync/nginxlogs.pp
A modules/dumps/manifests/web/rsync/nginxlogs.pp
M modules/dumps/manifests/web/xmldumps.pp
A modules/dumps/manifests/web/xmldumps_active.pp
M modules/profile/manifests/dumps/web/rsync_server.pp
D modules/profile/manifests/dumps/web/xmldumps.pp
A modules/profile/manifests/dumps/web/xmldumps_active.pp
A modules/profile/manifests/dumps/web/xmldumps_fallback.pp
M modules/role/manifests/dataset/primary.pp
D modules/role/manifests/dumps/web/xmldumps.pp
A modules/role/manifests/dumps/web/xmldumps_active.pp
A modules/role/manifests/dumps/web/xmldumps_fallback.pp
16 files changed, 117 insertions(+), 97 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 37894c0..1822f98 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -310,6 +310,32 @@
- snapshot1005.eqiad.wmnet
- snapshot1006.eqiad.wmnet
- snapshot1007.eqiad.wmnet
+dumps_web_rsync_server_clients:
+ ipv4:
+ - dataset1001.wikimedia.org
+ - mwlog1001.eqiad.wmnet
+ - mwlog2001.codfw.wmnet
+ - phab1001.eqiad.wmnet
+ - ms1001.wikimedia.org
+ - stat1005.eqiad.wmnet
+ - stat1006.eqiad.wmnet
+ - sagres.c3sl.ufpr.br
+ - odysseus.fi.muni.cz
+ - odysseus.linux.cz
+ - mirror.fr.wickedway.nl
+# disabled as of Feb 10 2017, may come back on line later
+# - wikimedia.wansec.com
+ - ftpmirror.your.org
+ - ec2-174-129-186-231.compute-1.amazonaws.com
+ - ftpmirror-ae0-4.us.your.org
+ - crcdtn01.crc.nd.edu
+ - wmrsync.crc.nd.edu
+ - wikimedia.iconic.vi
+ - poincare.acc.umu.se
+ - wikimedia.bytemark.co.uk
+ ipv6:
+ - odysseus.ip6.fi.muni.cz
+ - poincare.acc.umu.se
# Schemas names that match this regex
# will not be produced to the eventlogging-valid-mixed
diff --git a/hieradata/hosts/dataset1001.yaml b/hieradata/hosts/dataset1001.yaml
index 0640bd8..a34dd3f 100644
--- a/hieradata/hosts/dataset1001.yaml
+++ b/hieradata/hosts/dataset1001.yaml
@@ -1,2 +1,3 @@
admin::groups:
- dataset-admins
+do_acme: true
\ No newline at end of file
diff --git a/hieradata/role/common/dumps/web/xmldumps.yaml
b/hieradata/role/common/dumps/web/xmldumps.yaml
deleted file mode 100644
index 80cdcb5..0000000
--- a/hieradata/role/common/dumps/web/xmldumps.yaml
+++ /dev/null
@@ -1,26 +0,0 @@
-profile::dumps::web::rsync_server::clients:
- ipv4:
- - dataset1001.wikimedia.org
- - mwlog1001.eqiad.wmnet
- - mwlog2001.codfw.wmnet
- - phab1001.eqiad.wmnet
- - ms1001.wikimedia.org
- - stat1005.eqiad.wmnet
- - stat1006.eqiad.wmnet
- - sagres.c3sl.ufpr.br
- - odysseus.fi.muni.cz
- - odysseus.linux.cz
- - mirror.fr.wickedway.nl
-# disable Feb 10 2017, looking into dns issues
-# - wikimedia.wansec.com
- - ftpmirror.your.org
- - ec2-174-129-186-231.compute-1.amazonaws.com
- - ftpmirror-ae0-4.us.your.org
- - crcdtn01.crc.nd.edu
- - wmrsync.crc.nd.edu
- - wikimedia.iconic.vi
- - poincare.acc.umu.se
- - wikimedia.bytemark.co.uk
- ipv6:
- - odysseus.ip6.fi.muni.cz
- - poincare.acc.umu.se
diff --git a/manifests/site.pp b/manifests/site.pp
index 92d0e73..85f3f3a 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -321,7 +321,7 @@
node 'dataset1001.wikimedia.org' {
- role(dataset::primary, dumps::web::xmldumps)
+ role(dataset::primary, dumps::web::xmldumps_active)
interface::add_ip6_mapped { 'main': }
}
@@ -1558,7 +1558,7 @@
# also see dataset1001
node 'ms1001.wikimedia.org' {
- role(dataset::secondary, dumps::web::xmldumps)
+ role(dataset::secondary, dumps::web::xmldumps_fallback)
interface::add_ip6_mapped { 'main': }
}
diff --git a/modules/dataset/manifests/cron/rsync/nginxlogs.pp
b/modules/dataset/manifests/cron/rsync/nginxlogs.pp
deleted file mode 100644
index c9624fc..0000000
--- a/modules/dataset/manifests/cron/rsync/nginxlogs.pp
+++ /dev/null
@@ -1,25 +0,0 @@
-class dataset::cron::rsync::nginxlogs (
- $enable = true,
- $dest = undef,
- $user = 'root',
-)
-{
- include ::dataset::common
-
- $ensure = $enable ? {
- true => 'present',
- default => 'absent',
- }
-
- $rsync_args = '-rt --perms --chmod=go+r --bwlimit=50000'
- cron { 'rsync_nginxlogs':
- ensure => $ensure,
- user => $user,
- minute => 55,
- hour => 4,
- command => "/usr/bin/rsync ${rsync_args} /var/log/nginx/*.gz
${dest}",
- environment => '[email protected]',
- require => User[$user],
- }
-}
-
diff --git a/modules/dumps/manifests/web/rsync/nginxlogs.pp
b/modules/dumps/manifests/web/rsync/nginxlogs.pp
new file mode 100644
index 0000000..22d02c9
--- /dev/null
+++ b/modules/dumps/manifests/web/rsync/nginxlogs.pp
@@ -0,0 +1,17 @@
+class dumps::web::rsync::nginxlogs (
+ $dest = undef,
+)
+{
+ require_package('rsync')
+
+ $rsync_args = '-rt --perms --chmod=go+r --bwlimit=50000'
+ cron { 'rsync_nginxlogs':
+ ensure => 'present',
+ user => 'root',
+ minute => 55,
+ hour => 4,
+ command => "/usr/bin/rsync ${rsync_args} /var/log/nginx/*.gz
${dest}",
+ environment => '[email protected]',
+ }
+}
+
diff --git a/modules/dumps/manifests/web/xmldumps.pp
b/modules/dumps/manifests/web/xmldumps.pp
index 7cdf844..84d2421 100644
--- a/modules/dumps/manifests/web/xmldumps.pp
+++ b/modules/dumps/manifests/web/xmldumps.pp
@@ -1,6 +1,7 @@
# serve xml/sql dumps: https://wikitech.wikimedia.org/wiki/Dumps
-class dumps::web::xmldumps {
-
+class dumps::web::xmldumps(
+ $do_acme = true,
+) {
class { '::nginx':
variant => 'extras',
}
@@ -29,4 +30,28 @@
group => 'root',
mode => '0444',
}
+
+ include ::base::firewall
+
+ ferm::service { 'xmldumps_http':
+ proto => 'tcp',
+ port => '80',
+ }
+
+ ferm::service { 'xmldumps_https':
+ proto => 'tcp',
+ port => '443',
+ }
+
+ monitoring::service { 'http':
+ description => 'HTTP',
+ check_command => 'check_http'
+ }
+
+ if ($do_acme == true) {
+ monitoring::service { 'https':
+ description => 'HTTPS',
+ check_command => 'check_ssl_http_letsencrypt!dumps.wikimedia.org',
+ }
+ }
}
diff --git a/modules/dumps/manifests/web/xmldumps_active.pp
b/modules/dumps/manifests/web/xmldumps_active.pp
new file mode 100644
index 0000000..11aef6c
--- /dev/null
+++ b/modules/dumps/manifests/web/xmldumps_active.pp
@@ -0,0 +1,11 @@
+class dumps::web::xmldumps_active(
+ $do_acme = true,
+) {
+ # active web server
+ class {'::dumps::web::xmldumps': do_acme => $do_acme}
+
+ # only the active web server should be syncing nginx logs
+ class {'::dumps::web::rsync::nginxlogs':
+ dest =>
'stat1005.eqiad.wmnet::srv/log/webrequest/archive/dumps.wikimedia.org/',
+ }
+}
diff --git a/modules/profile/manifests/dumps/web/rsync_server.pp
b/modules/profile/manifests/dumps/web/rsync_server.pp
index 47f0745..1f2ff4d 100644
--- a/modules/profile/manifests/dumps/web/rsync_server.pp
+++ b/modules/profile/manifests/dumps/web/rsync_server.pp
@@ -1,5 +1,5 @@
class profile::dumps::web::rsync_server(
- $rsync_clients = hiera('profile::dumps::web::rsync_server::clients'),
+ $rsync_clients = hiera('dumps_web_rsync_server_clients'),
) {
# By default the resolve() function in ferm performs only an IPv4/A DNS
# lookup. It fails if a host only has an IPv6 address. Ferm also provides
diff --git a/modules/profile/manifests/dumps/web/xmldumps.pp
b/modules/profile/manifests/dumps/web/xmldumps.pp
deleted file mode 100644
index 45d5a18..0000000
--- a/modules/profile/manifests/dumps/web/xmldumps.pp
+++ /dev/null
@@ -1,13 +0,0 @@
-class profile::dumps::web::xmldumps {
- class {'::dumps::web::xmldumps':}
-
- ferm::service { 'xmldumps_http':
- proto => 'tcp',
- port => '80',
- }
-
- ferm::service { 'xmldumps_https':
- proto => 'tcp',
- port => '443',
- }
-}
diff --git a/modules/profile/manifests/dumps/web/xmldumps_active.pp
b/modules/profile/manifests/dumps/web/xmldumps_active.pp
new file mode 100644
index 0000000..a096cbc
--- /dev/null
+++ b/modules/profile/manifests/dumps/web/xmldumps_active.pp
@@ -0,0 +1,5 @@
+class profile::dumps::web::xmldumps_active {
+ class {'::dumps::web::xmldumps_active':
+ do_acme => hiera('do_acme'),
+ }
+}
diff --git a/modules/profile/manifests/dumps/web/xmldumps_fallback.pp
b/modules/profile/manifests/dumps/web/xmldumps_fallback.pp
new file mode 100644
index 0000000..3a26f0f
--- /dev/null
+++ b/modules/profile/manifests/dumps/web/xmldumps_fallback.pp
@@ -0,0 +1,5 @@
+class profile::dumps::web::xmldumps_fallback {
+ class {'::dumps::web::xmldumps':
+ do_acme => hiera('do_acme'),
+ }
+}
diff --git a/modules/role/manifests/dataset/primary.pp
b/modules/role/manifests/dataset/primary.pp
index 7e7a4f1..180603f 100644
--- a/modules/role/manifests/dataset/primary.pp
+++ b/modules/role/manifests/dataset/primary.pp
@@ -38,11 +38,6 @@
minute => '41',
}
- class { '::dataset::cron::rsync::nginxlogs':
- enable => true,
- dest =>
'stat1005.eqiad.wmnet::srv/log/webrequest/archive/dumps.wikimedia.org/',
- }
-
class { '::dataset::cron::wikitech_dumps':
enable => true,
url => 'https://wikitech.wikimedia.org/dumps/',
diff --git a/modules/role/manifests/dumps/web/xmldumps.pp
b/modules/role/manifests/dumps/web/xmldumps.pp
deleted file mode 100644
index 2ba12bf..0000000
--- a/modules/role/manifests/dumps/web/xmldumps.pp
+++ /dev/null
@@ -1,23 +0,0 @@
-# web server of xml/sql dumps and other datasets
-# also permits rsync from public mirrors and certain internal hosts
-# lastly, serves thse files via nfs to certain internal hosts
-class role::dumps::web::xmldumps {
- include ::profile::dumps::web::xmldumps
- include ::profile::dumps::web::rsync_server
- include ::profile::dumps::nfs_server
-
- system::role { 'role::dumps::web::xmldumps': description => 'web, nfs and
rsync server of xml/sql dumps' }
-
- monitoring::service { 'http':
- description => 'HTTP',
- check_command => 'check_http'
- }
-
- # TODO: move hiera lookup to parameter of a profile class
- if hiera('do_acme', true) {
- monitoring::service { 'https':
- description => 'HTTPS',
- check_command => 'check_ssl_http_letsencrypt!dumps.wikimedia.org',
- }
- }
-}
diff --git a/modules/role/manifests/dumps/web/xmldumps_active.pp
b/modules/role/manifests/dumps/web/xmldumps_active.pp
new file mode 100644
index 0000000..5845c40
--- /dev/null
+++ b/modules/role/manifests/dumps/web/xmldumps_active.pp
@@ -0,0 +1,11 @@
+# active web server of xml/sql dumps and other datasets
+# also permits rsync from public mirrors and certain internal hosts
+# lastly, serves thse files via nfs to certain internal hosts
+class role::dumps::web::xmldumps_active {
+ include ::standard
+ include ::profile::dumps::web::xmldumps_active
+ include ::profile::dumps::web::rsync_server
+ include ::profile::dumps::nfs_server
+
+ system::role { 'role::dumps::web::xmldumps': description => 'active web,
nfs and rsync server of xml/sql dumps' }
+}
diff --git a/modules/role/manifests/dumps/web/xmldumps_fallback.pp
b/modules/role/manifests/dumps/web/xmldumps_fallback.pp
new file mode 100644
index 0000000..b318763
--- /dev/null
+++ b/modules/role/manifests/dumps/web/xmldumps_fallback.pp
@@ -0,0 +1,11 @@
+# fallback web server of xml/sql dumps and other datasets
+# also permits rsync from public mirrors and certain internal hosts
+# lastly, serves thse files via nfs to certain internal hosts
+class role::dumps::web::xmldumps_fallback {
+ include ::standard
+ include ::profile::dumps::web::xmldumps_fallback
+ include ::profile::dumps::web::rsync_server
+ include ::profile::dumps::nfs_server
+
+ system::role { 'role::dumps::web::xmldumps': description => 'fallback web,
nfs and rsync server of xml/sql dumps' }
+}
--
To view, visit https://gerrit.wikimedia.org/r/379198
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I08b8c024c1d901670795757351bcead8b78c8ccf
Gerrit-PatchSet: 9
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
