Giuseppe Lavagetto has uploaded a new change for review.
https://gerrit.wikimedia.org/r/198721
Change subject: ganglia: autogenerate datasources from the list of clusters
......................................................................
ganglia: autogenerate datasources from the list of clusters
Change-Id: Ia32ea48028ff3b082f240cc01c602b0e9a6587f9
---
M hieradata/common.yaml
M manifests/role/ganglia.pp
A modules/ganglia_new/lib/puppet/parser/functions/ganglia_aggregator_config.rb
M modules/ganglia_new/manifests/monitor/aggregator/instance.pp
4 files changed, 157 insertions(+), 112 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/21/198721/1
diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 8863dde..1a39a0e 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -10,10 +10,14 @@
name: "LVS loadbalancers"
ip_oct: "2"
sites:
- - eqiad
- - codfw
- - esams
- - ulsfo
+ eqiad:
+ - lvs1001.wikimedia.org
+ - lvs1002.wikimedia.org
+ codfw: install2001.wikimedia.org:10649
+ esams: hooft.esams.wikimedia.org:11649
+ ulsfo:
+ - lvs4001.ulsfo.wmnet
+ - lvs4003.ulsfo.wmnet
search:
name: "Search"
ip_oct: "3"
@@ -22,110 +26,151 @@
name: "MySQL"
ip_oct: "5"
sites:
- - eqiad
- - codfw
+ eqiad:
+ - dbstore1001.eqiad.wmnet
+ - dbstore1002.eqiad.wmnet
+ codfw: install2001.wikimedia.org:10649
misc:
name: "Miscellaneous"
ip_oct: "8"
sites:
- - eqiad
- - codfw
+ eqiad:
+ - carbon.wikimedia.org
+ - ms1004.eqiad.wmnet
+ codfw: install2001.wikimedia.org:10649
appserver:
name: "Application servers"
ip_oct: "11"
sites:
- - eqiad
- - codfw
+ eqiad:
+ - mw1054.eqiad.wmnet
+ - mw1055.eqiad.wmnet
+ codfw: install2001.wikimedia.org:10649
imagescaler:
name: "Image scalers"
ip_oct: "12"
sites:
- - eqiad
- - codfw
+ eqiad:
+ - mw1153.eqiad.wmnet
+ - mw1154.eqiad.wmnet
+ codfw: install2001.wikimedia.org:10649
api_appserver:
name: "API application servers"
ip_oct: "13"
sites:
- - eqiad
- - codfw
- pdf:
+ eqiad:
+ - mw1114.eqiad.wmnet
+ - mw1115.eqiad.wmnet
+ codfw: install2001.wikimedia.org:10649
+ pdf:
name: "PDF servers"
ip_oct: "15"
sites:
- - eqiad
+ eqiad:
+ - ocg1001.eqiad.wmnet
cache_text:
name: "Text caches"
ip_oct: "20"
sites:
- - eqiad
- - esams
- - ulsfo
+ eqiad:
+ - cp1052.eqiad.wmnet
+ - cp1053.eqiad.wmnet
+ esams: hooft.esams.wikimedia.org:11649
+ ulsfo:
+ - cp4008.ulsfo.wmnet
+ - cp4016.ulsfo.wmnet
cache_bits:
name: "Bits caches"
ip_oct: "21"
sites:
- - eqiad
- - esams
- - ulsfo
+ eqiad:
+ - cp1056.eqiad.wmnet
+ - cp1057.eqiad.wmnet
+ esams: hooft.esams.wikimedia.org:11649
+ ulsfo:
+ - cp4008.ulsfo.wmnet
+ - cp4016.ulsfo.wmnet
cache_upload:
name: "Upload caches"
ip_oct: "22"
sites:
- - eqiad
- - esams
- - ulsfo
+ eqiad:
+ - cp1048.eqiad.wmnet
+ - cp1061.eqiad.wmnet
+ esams: hooft.esams.wikimedia.org:11649
+ ulsfo:
+ - cp4005.ulsfo.wmnet
+ - cp4013.ulsfo.wmnet
payments:
name: "Fundraiser payments"
ip_oct: "23"
- sites:
- - eqiad
+ sites: []
swift:
name: "Swift"
ip_oct: "27"
sites:
- - eqiad
- - codfw
- - esams
+ eqiad:
+ - ms-fe1001.eqiad.wmnet
+ - ms-fe1002.eqiad.wmnet
+ codfw: install2001.wikimedia.org:10649
+ esams: hooft.esams.wikimedia.org:11649
cache_mobile:
name: "Mobile caches"
ip_oct: "28"
sites:
- - eqiad
- - esams
- - ulsfo
+ eqiad:
+ - cp1046.eqiad.wmnet
+ - cp1047.eqiad.wmnet
+ esams: hooft.esams.wikimedia.org:11649
+ ulsfo:
+ - cp4011.ulsfo.wmnet
+ - cp4019.ulsfo.wmnet
virt:
name: "Virtualization cluster"
ip_oct: "29"
sites:
- - eqiad
+ eqiad:
+ - labnet1001.eqiad.wmnet
+ - virt1000.wikimedia.org
jobrunner:
name: "Jobrunners"
ip_oct: "31"
sites:
- - eqiad
- - codfw
+ eqiad:
+ - mw1001.eqiad.wmnet
+ - mw1002.eqiad.wmnet
+ codfw: install2001.wikimedia.org:10649
analytics:
name: "Analytics cluster"
ip_oct: "32"
sites:
- - eqiad
+ eqiad:
+ - analytics1013.eqiad.wmnet
+ - analytics1014.eqiad.wmnet
+ - analytics1019.eqiad.wmnet
memcached:
name: "Memcached"
ip_oct: "33"
sites:
- - eqiad
- - codfw
+ eqiad:
+ - mc1001.eqiad.wmnet
+ - mc1002.eqiad.wmnet
+ codfw: install2001.wikimedia.org:10649
videoscaler:
name: "Video scalers"
ip_oct: "34"
sites:
- - eqiad
- - codfw
+ eqiad:
+ - tmh1001.eqiad.wmnet
+ - tmh1002.eqiad.wmnet
+ codfw: install2001.wikimedia.org:10649
fundraising:
name: "Fundraising"
ip_oct: "35"
sites:
- - eqiad
+ eqiad:
+ - pay-lvs1001.frack.eqiad.wmnet
+ - pay-lvs1002.frack.eqiad.wmnet
ceph: # Not used anymore
name: "Ceph"
ip_oct: "36"
@@ -134,57 +179,77 @@
name: "Parsoid"
ip_oct: "37"
sites:
- - eqiad
+ eqiad:
+ - wtp1001.eqiad.wmnet
+ - wtp1002.eqiad.wmnet
cache_parsoid:
name: "Parsoid Varnish"
ip_oct: "38"
sites:
- - eqiad
- - esams
- - ulsfo
+ eqiad:
+ - cp1045.eqiad.wmnet
+ - cp1058.eqiad.wmnet
redis:
name: "Redis"
ip_oct: "39"
sites:
- - eqiad
- - codfw
+ eqiad:
+ - rdb1001.eqiad.wmnet
+ - rdb1002.eqiad.wmnet
+ codfw: install2001.wikimedia.org:10649
labsnfs:
name: "Labs NFS cluster"
ip_oct: "40"
sites:
- - eqiad
+ eqiad:
+ - labstore1001.eqiad.wmnet
+ - labstore1003.eqiad.wmnet
cache_misc:
name: "Misc Web caching cluster"
ip_oct: "41"
sites:
- - eqiad
+ eqiad:
+ - cp1043.eqiad.wmnet
+ - cp1044.eqiad.wmnet
elasticsearch:
name: "Elasticsearch cluster"
ip_oct: "42"
sites:
- - eqiad
- logstash:
+ eqiad:
+ - elastic1001.eqiad.wmnet
+ - elastic1007.eqiad.wmnet
+ - elastic1013.eqiad.wmnet
+ logstash:
name: "Logstash cluster"
ip_oct: "43"
sites:
- - eqiad
+ eqiad:
+ - logstash1001.eqiad.wmnet
+ - logstash1003.eqiad.wmnet
rcstream:
name: "RCStream cluster"
ip_oct: "44"
sites:
- - eqiad
+ eqiad:
+ - rcs1001.eqiad.wmnet
analytics_kafka:
name: "Analytics Kafka cluster"
ip_oct: "45"
sites:
- - eqiad
+ eqiad:
+ - analytics1012.eqiad.wmnet
+ - analytics1018.eqiad.wmnet
+ - analytics1022.eqiad.wmnet
sca:
name: "Service Cluster A"
ip_oct: "46"
sites:
- - eqiad
+ eqiad:
+ - sca1001.eqiad.wmnet
+ - sca1002.eqiad.wmnet
openldap_corp_mirror:
name: "Corp OIT LDAP mirror"
ip_oct: "47"
sites:
- - eqiad
+ eqiad:
+ - plutonium.wikimedia.org
diff --git a/manifests/role/ganglia.pp b/manifests/role/ganglia.pp
index 9c45a8f..52bc665 100644
--- a/manifests/role/ganglia.pp
+++ b/manifests/role/ganglia.pp
@@ -1,60 +1,7 @@
class role::ganglia::config {
# TODO: hiera this
$rra_sizes = '"RRA:AVERAGE:0.5:1:360" "RRA:AVERAGE:0.5:24:245"
"RRA:AVERAGE:0.5:168:241" "RRA:AVERAGE:0.5:672:241" "RRA:AVERAGE:0.5:5760:371"'
-
- $data_sources = {
- 'Video scalers eqiad' => 'tmh1001.eqiad.wmnet
tmh1002.eqiad.wmnet',
- 'Video scalers codfw' => 'install2001.wikimedia.org:10673',
- 'Image scalers eqiad' => 'mw1153.eqiad.wmnet
mw1154.eqiad.wmnet',
- 'Image scalers codfw' => 'install2001.wikimedia.org:10661',
- 'API application servers eqiad' => 'mw1114.eqiad.wmnet
mw1115.eqiad.wmnet',
- 'API application servers codfw' => 'install2001.wikimedia.org:10662',
- 'Application servers eqaid' => 'mw1054.eqiad.wmnet
mw1055.eqiad.wmnet',
- 'Application servers codfw' => 'install2001.wikimedia.org:10660',
- 'Jobrunners eqiad' => 'mw1001.eqiad.wmnet
mw1002.eqiad.wmnet',
- 'Jobrunners codfw' => 'install2001.wikimedia.org:10680
mw2001.codfw.wmnet mw2080.codfw.wmnet',
- 'MySQL' => 'db1050.eqiad.wmnet',
- 'PDF servers eqiad' => 'ocg1001.eqiad.wmnet',
- 'Fundraising eqiad' => 'pay-lvs1001.frack.eqiad.wmnet
pay-lvs1002.frack.eqiad.wmnet',
- 'Virtualization cluster eqiad' => 'labnet1001.eqiad.wmnet
virt1000.wikimedia.org',
- 'Labs NFS cluster eqiad' => 'labstore1001.eqiad.wmnet
labstore1003.eqiad.wmnet',
- 'MySQL eqiad' => 'dbstore1001.eqiad.wmnet
dbstore1002.eqiad.wmnet',
- 'LVS loadbalancers eqiad' => 'lvs1001.wikimedia.org
lvs1002.wikimedia.org',
- 'LVS loadbalancers codfw' => 'install2001.wikimedia.org:10651
lvs2001.codfw.wmnet lvs2002.codfw.wmnet',
- 'Miscellaneous eqiad' => 'carbon.wikimedia.org
ms1004.eqiad.wmnet',
- 'Miscellaneous codfw' => 'install2001.wikimedia.org:10657',
- 'Mobile caches eqiad' => 'cp1046.eqiad.wmnet
cp1047.eqiad.wmnet',
- 'Mobile caches esams' => 'hooft.esams.wikimedia.org:11677',
- 'Bits caches eqiad' => 'cp1056.eqiad.wmnet
cp1057.eqiad.wmnet',
- 'Upload caches eqiad' => 'cp1048.eqiad.wmnet
cp1061.eqiad.wmnet',
- 'Swift eqiad' => 'ms-fe1001.eqiad.wmnet
ms-fe1002.eqiad.wmnet',
- 'Swift esams' => 'hooft.esams.wikimedia.org:11676',
- 'Swift codfw' => 'install2001.wikimedia.org:10676',
- 'Bits caches esams' => 'hooft.esams.wikimedia.org:11670
cp3019.esams.wmnet cp3020.esams.wmnet',
- 'LVS loadbalancers esams' => 'hooft.esams.wikimedia.org:11651
lvs3001.esams.wmnet lvs3002.esams.wmnet',
- 'Miscellaneous esams' => 'hooft.esams.wikimedia.org:11657',
- 'Analytics cluster eqiad' => 'analytics1013.eqiad.wmnet
analytics1014.eqiad.wmnet analytics1019.eqiad.wmnet',
- 'Memcached eqiad' => 'mc1001.eqiad.wmnet
mc1002.eqiad.wmnet',
- 'Text caches esams' => 'hooft.esams.wikimedia.org:11669',
- 'Upload caches esams' => 'hooft.esams.wikimedia.org:11671
cp3003.esams.wmnet cp3004.esams.wmnet',
- 'Parsoid eqiad' => 'wtp1001.eqiad.wmnet
wtp1002.eqiad.wmnet',
- 'Parsoid Varnish eqiad' => 'cp1045.eqiad.wmnet
cp1058.eqiad.wmnet',
- 'Redis eqiad' => 'rdb1001.eqiad.wmnet
rdb1002.eqiad.wmnet',
- 'Text caches eqiad' => 'cp1052.eqiad.wmnet
cp1053.eqiad.wmnet',
- 'Misc Web caches eqiad' => 'cp1043.eqiad.wmnet
cp1044.eqiad.wmnet',
- 'LVS loadbalancers ulsfo' => 'lvs4001.ulsfo.wmnet
lvs4003.ulsfo.wmnet',
- 'Bits caches ulsfo' => 'cp4001.ulsfo.wmnet
cp4003.ulsfo.wmnet',
- 'Upload caches ulsfo' => 'cp4005.ulsfo.wmnet
cp4013.ulsfo.wmnet',
- 'Mobile caches ulsfo' => 'cp4011.ulsfo.wmnet
cp4019.ulsfo.wmnet',
- 'Text caches ulsfo' => 'cp4008.ulsfo.wmnet
cp4016.ulsfo.wmnet',
- 'Elasticsearch eqiad' => 'elastic1001.eqiad.wmnet
elastic1007.eqiad.wmnet elastic1013.eqiad.wmnet',
- 'Logstash eqiad' => 'logstash1001.eqiad.wmnet
logstash1003.eqiad.wmnet',
- 'RCStream eqiad' => 'rcs1001.eqiad.wmnet',
- 'Analytics Kafka cluster eqiad' => 'analytics1012.eqiad.wmnet
analytics1018.eqiad.wmnet analytics1022.eqiad.wmnet',
- 'Service Cluster A eqiad' => 'sca1001.eqiad.wmnet
sca1002.eqiad.wmnet',
- 'Corp OIT LDAP mirror eqiad' => 'plutonium.wikimedia.org',
- 'Corp OIT LDAP mirror codfw' => 'pollux.wikimedia.org',
- }
+ $data_sources = ganglia_aggregator_config()
}
# A role that includes all the needed stuff to run a ganglia web frontend
diff --git
a/modules/ganglia_new/lib/puppet/parser/functions/ganglia_aggregator_config.rb
b/modules/ganglia_new/lib/puppet/parser/functions/ganglia_aggregator_config.rb
new file mode 100644
index 0000000..ee26a3a
--- /dev/null
+++
b/modules/ganglia_new/lib/puppet/parser/functions/ganglia_aggregator_config.rb
@@ -0,0 +1,33 @@
+#
+# ganglia_aggregator_config - generates the aggregator config
+#
+def calc_url(aggregator, ip_octect)
+ url, port = aggregator.split(':')
+ port = port.to_i + ip_octect.to_i
+ return sprintf("%s:%d", url, port)
+end
+
+
+module Puppet::Parser::Functions
+ newfunction(:ganglia_aggregator_config, :type => :rvalue) do |args|
+ config = {}
+ site_config = {}
+ clusters = function_hiera(['ganglia_clusters'])
+ clusters.each do |cluster, data|
+ data['sites'].each do |site, aggregators|
+ name = sprintf("%s %s", data['name'], site)
+ unless site_config.include? site
+ site_config[site] = function_hiera(['ganglia_class'], nil, [site])
+ end
+ if site_config[site] == 'old'
+ aggregator = aggregators.join(' ')
+ else
+ # Compute the port to use
+ aggregator = calc_url(aggregators, data['ip_oct'])
+ end
+ config[name] = aggregator
+ end
+ end
+ return config
+ end
+end
diff --git a/modules/ganglia_new/manifests/monitor/aggregator/instance.pp
b/modules/ganglia_new/manifests/monitor/aggregator/instance.pp
index 83746ea..5e228b2 100644
--- a/modules/ganglia_new/manifests/monitor/aggregator/instance.pp
+++ b/modules/ganglia_new/manifests/monitor/aggregator/instance.pp
@@ -9,7 +9,7 @@
$cluster = regsubst($title, '^(.*)_[^_]+$', '\1')
if has_key($ganglia_new::configuration::clusters[$cluster], 'sites') {
- $sites = $ganglia_new::configuration::clusters[$cluster]['sites']
+ $sites = keys($ganglia_new::configuration::clusters[$cluster]['sites'])
} else {
$sites = $ganglia_new::configuration::default_sites
}
--
To view, visit https://gerrit.wikimedia.org/r/198721
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia32ea48028ff3b082f240cc01c602b0e9a6587f9
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Giuseppe Lavagetto <glavage...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
