Ottomata has submitted this change and it was merged.
Change subject: labs: Setup /srv/statistics for rsync from stats hosts
......................................................................
labs: Setup /srv/statistics for rsync from stats hosts
Is a writeable rsync module that researchers on stat*
boxes can put data into that'll show up on a separate
mount in labs instances that opt into it
Bug: T107576
Change-Id: Ief9bec2cc9b80203f3462619237e3da744884021
---
M hieradata/common.yaml
M manifests/role/labsnfs.pp
M manifests/role/statistics.pp
M manifests/site.pp
R templates/nfs/exports.labs_extras.erb
5 files changed, 48 insertions(+), 26 deletions(-)
Approvals:
Ottomata: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 575d8cd..d6fb879 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -228,3 +228,8 @@
conf1003.eqiad.wmnet: '1103'
puppetmaster: "puppet"
active_labstore_host: 'labstore1002'
+statistics_servers:
+ - stat1001.eqiad.wmnet
+ - stat1002.eqiad.wmnet
+ - stat1003.eqiad.wmnet
+ - analytics1027.eqiad.wmnet
diff --git a/manifests/role/labsnfs.pp b/manifests/role/labsnfs.pp
index 4bbf006..1803778 100644
--- a/manifests/role/labsnfs.pp
+++ b/manifests/role/labsnfs.pp
@@ -1,38 +1,52 @@
-# Class: role::labs::nfs::dumps
+# == Class role::labs::nfs::extras
#
-# The role class for the NFS server that makes dumps avaliable
-# to labs from production - it serves as a readonly server to Labs,
-# while being populated from the actual dumps server in prod.
+# The role class for the NFS server that makes dumps and other
+# datsets avaliable to labs from production - it serves as a readonly
+# server to Labs, while being populated from the actual dataset1001 server
+# in prod.
#
# The IPs of the servers allowed to populate it ($dump_servers_ips)
# must be set at the node level or via hiera.
#
-class role::labs::nfs::dumps($dump_servers_ips) {
+# This also exports /srv/statistics to allow statistics servers
+# a way to rsync public data in from production.
+#
+class role::labs::nfs::extras {
include standard
include ::labstore
include ::labstore::monitoring
include rsync::server
- # The dumps server has a simple, flat exports list
+ rsync::server::module { 'pagecounts':
+ path => '/srv/dumps/pagecounts',
+ read_only => 'no',
+ hosts_allow => $dump_servers_ips,
+ }
+
+ # Allow users to push files from statistics servers here.
+ file { '/srv/statistics':
+ ensure => 'directory',
+ }
+ $statistics_servers = hiera('statistics_servers')
+ rsync::server::module { 'statistics':
+ path => '/srv/statistics',
+ read_only => 'no',
+ hosts_allow => $statistics_servers,
+ require => File['/srv/statistics']
+ }
+
+ # This has a flat exports list
# because it only exports public data unconditionally
# and read-only
-
file { '/etc/exports':
ensure => present,
- content => template('nfs/exports.dumps.erb'),
+ content => template('nfs/exports.labs_extras.erb'),
owner => 'root',
group => 'root',
mode => '0444',
}
-
- rsync::server::module {
- 'pagecounts':
- path => '/srv/dumps/pagecounts',
- read_only => 'no',
- hosts_allow => $dump_servers_ips,
- }
-
}
+
# Class: role::labs::nfs::fileserver
#
@@ -47,4 +61,3 @@
monitor_iface => $monitor,
}
}
-
diff --git a/manifests/role/statistics.pp b/manifests/role/statistics.pp
index d80d545..1c22f3a 100644
--- a/manifests/role/statistics.pp
+++ b/manifests/role/statistics.pp
@@ -1,11 +1,14 @@
class role::statistics {
# Manually set a list of statistics servers.
- $statistics_servers = [
- 'stat1001.eqiad.wmnet',
- 'stat1002.eqiad.wmnet',
- 'stat1003.eqiad.wmnet',
- 'analytics1027.eqiad.wmnet',
- ]
+ $statistics_servers = hiera(
+ 'statistics_servers',
+ [
+ 'stat1001.eqiad.wmnet',
+ 'stat1002.eqiad.wmnet',
+ 'stat1003.eqiad.wmnet',
+ 'analytics1027.eqiad.wmnet',
+ ]
+ )
# we are attempting to stop using /a and to start using
# /srv instead. stat1002 still use
diff --git a/manifests/site.pp b/manifests/site.pp
index dc09009..b4e1e58 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -1315,7 +1315,7 @@
}
node 'labstore1003.eqiad.wmnet' {
- role labs::nfs::dumps
+ role labs::nfs::extras
}
node /labstore200[12]\.codfw\.wmnet/ {
diff --git a/templates/nfs/exports.dumps.erb
b/templates/nfs/exports.labs_extras.erb
similarity index 71%
rename from templates/nfs/exports.dumps.erb
rename to templates/nfs/exports.labs_extras.erb
index abd98a4..6f046d6 100644
--- a/templates/nfs/exports.dumps.erb
+++ b/templates/nfs/exports.labs_extras.erb
@@ -3,10 +3,11 @@
#
# THIS FILE IS MANAGED BY PUPPET
#
-# Source: files/nfs/exports.dumps
-# From: role::labs::nfs::dumps
+# Source: templates/nfs/exports.labs_extras.erb
+# From: role::labs::nfs::extras
#
/srv *(ro,fsid=0,sec=sys,sync,subtree_check,root_squash,nocrossmnt)
/srv/dumps -rw,sec=sys,no_subtree_check,no_root_squash <%=
Array(@dump_servers_ips).join(' ') %>
/srv/dumps *(ro,sec=sys,no_subtree_check,all_squash,nocrossmnt)
+/srv/statistics *(ro,sec=sys,no_subtree_check,all_squash,nocrossmnt)
--
To view, visit https://gerrit.wikimedia.org/r/229262
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ief9bec2cc9b80203f3462619237e3da744884021
Gerrit-PatchSet: 4
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: Yuvipanda <[email protected]>
Gerrit-Reviewer: coren <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
