Giuseppe Lavagetto has submitted this change and it was merged.
Change subject: ganglia: autogenerate datasources from the list of clusters
......................................................................
ganglia: autogenerate datasources from the list of clusters
Bug:T93776
Change-Id: Ia32ea48028ff3b082f240cc01c602b0e9a6587f9
---
M hieradata/codfw.yaml
M hieradata/common.yaml
M hieradata/esams.yaml
M manifests/role/ganglia.pp
A modules/ganglia_new/lib/puppet/parser/functions/ganglia_aggregator_config.rb
M modules/ganglia_new/manifests/monitor/aggregator/instance.pp
6 files changed, 174 insertions(+), 125 deletions(-)
Approvals:
Giuseppe Lavagetto: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/codfw.yaml b/hieradata/codfw.yaml
index ebb6b56..c23a88a 100644
--- a/hieradata/codfw.yaml
+++ b/hieradata/codfw.yaml
@@ -25,3 +25,4 @@
- '10.192.32.23:11211:1 "shard16"'
jobrunner_state: 'stopped'
ganglia_class: "new"
+ganglia_aggregators: install2001.wikimedia.org:10649
diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index a87bf6a..8732306 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -5,195 +5,258 @@
decommissioned:
name: "Decommissioned servers"
id: 1
- sites: []
+ sites: {}
lvs:
name: "LVS loadbalancers"
id: 2
sites:
- - eqiad
- - codfw
- - esams
- - ulsfo
+ eqiad:
+ - lvs1001.wikimedia.org
+ - lvs1002.wikimedia.org
+ codfw: []
+ esams: []
+ ulsfo:
+ - lvs4001.ulsfo.wmnet
+ - lvs4003.ulsfo.wmnet
search:
name: "Search"
id: 4
- sites: []
+ sites: {}
mysql:
name: "MySQL"
id: 5
sites:
- - eqiad
- - codfw
+ eqiad:
+ - dbstore1001.eqiad.wmnet
+ - dbstore1002.eqiad.wmnet
+ codfw: []
misc:
name: "Miscellaneous"
id: 8
sites:
- - eqiad
- - codfw
- - esams
- - ulsfo
+ eqiad:
+ - carbon.wikimedia.org
+ - ms1004.eqiad.wmnet
+ codfw: []
+ esams: []
appserver:
name: "Application servers"
id: 11
sites:
- - eqiad
- - codfw
+ eqiad:
+ - mw1054.eqiad.wmnet
+ - mw1055.eqiad.wmnet
+ codfw: []
imagescaler:
name: "Image scalers"
id: 12
sites:
- - eqiad
- - codfw
+ eqiad:
+ - mw1153.eqiad.wmnet
+ - mw1154.eqiad.wmnet
+ codfw: []
api_appserver:
name: "API application servers"
id: 13
sites:
- - eqiad
- - codfw
+ eqiad:
+ - mw1114.eqiad.wmnet
+ - mw1115.eqiad.wmnet
+ codfw: []
pdf:
name: "PDF servers"
id: 15
sites:
- - eqiad
+ eqiad:
+ - ocg1001.eqiad.wmnet
cache_text:
name: "Text caches"
id: 20
sites:
- - eqiad
- - esams
- - ulsfo
+ eqiad:
+ - cp1052.eqiad.wmnet
+ - cp1053.eqiad.wmnet
+ esams: []
+ ulsfo:
+ - cp4008.ulsfo.wmnet
+ - cp4016.ulsfo.wmnet
cache_bits:
name: "Bits caches"
id: 21
sites:
- - eqiad
- - esams
- - ulsfo
+ eqiad:
+ - cp1056.eqiad.wmnet
+ - cp1057.eqiad.wmnet
+ esams: []
+ ulsfo:
+ - cp4008.ulsfo.wmnet
+ - cp4016.ulsfo.wmnet
cache_upload:
name: "Upload caches"
id: 22
sites:
- - eqiad
- - esams
- - ulsfo
+ eqiad:
+ - cp1048.eqiad.wmnet
+ - cp1061.eqiad.wmnet
+ esams: []
+ ulsfo:
+ - cp4005.ulsfo.wmnet
+ - cp4013.ulsfo.wmnet
payments:
name: "Fundraiser payments"
+
id: 23
- sites:
- - eqiad
- - codfw
+ sites: []
ssl:
name: "SSL cluster"
id: 26
- sites:
- - eqiad
- - esams
+ sites: {}
swift:
name: "Swift"
id: 27
sites:
- - eqiad
- - codfw
- - esams
+ eqiad:
+ - ms-fe1001.eqiad.wmnet
+ - ms-fe1002.eqiad.wmnet
+ codfw: []
+ esams: []
cache_mobile:
name: "Mobile caches"
id: 28
sites:
- - eqiad
- - esams
- - ulsfo
- virt:
+ eqiad:
+ - cp1046.eqiad.wmnet
+ - cp1047.eqiad.wmnet
+ esams: []
+ ulsfo:
+ - cp4011.ulsfo.wmnet
+ - cp4019.ulsfo.wmnet
+ virt:
name: "Virtualization cluster"
id: 29
sites:
- - eqiad
+ eqiad:
+ - labnet1001.eqiad.wmnet
+ - virt1000.wikimedia.org
jobrunner:
name: "Jobrunners"
id: 31
sites:
- - eqiad
- - codfw
- analytics:
+ eqiad:
+ - mw1001.eqiad.wmnet
+ - mw1002.eqiad.wmnet
+ codfw: []
+ analytics:
name: "Analytics cluster"
id: 32
sites:
- - eqiad
+ eqiad:
+ - analytics1013.eqiad.wmnet
+ - analytics1014.eqiad.wmnet
+ - analytics1019.eqiad.wmnet
memcached:
name: "Memcached"
id: 33
sites:
- - eqiad
- - codfw
- videoscaler:
+ eqiad:
+ - mc1001.eqiad.wmnet
+ - mc1002.eqiad.wmnet
+ codfw: []
+ videoscaler:
name: "Video scalers"
id: 34
sites:
- - eqiad
- - codfw
+ eqiad:
+ - tmh1001.eqiad.wmnet
+ - tmh1002.eqiad.wmnet
+ codfw: []
fundraising:
name: "Fundraising"
id: 35
sites:
- - eqiad
- - codfw
+ eqiad:
+ - pay-lvs1001.frack.eqiad.wmnet
+ - pay-lvs1002.frack.eqiad.wmnet
ceph: # Not used anymore
name: "Ceph"
id: 36
- sites: []
+ sites: {}
parsoid:
name: "Parsoid"
id: 37
sites:
- - eqiad
- cache_parsoid:
+ eqiad:
+ - wtp1001.eqiad.wmnet
+ - wtp1002.eqiad.wmnet
+ cache_parsoid:
name: "Parsoid Varnish"
id: 38
sites:
- - eqiad
- - codfw
+ eqiad:
+ - cp1045.eqiad.wmnet
+ - cp1058.eqiad.wmnet
+ codfw: []
redis:
name: "Redis"
id: 39
sites:
- - eqiad
- - codfw
- labsnfs:
+ eqiad:
+ - rdb1001.eqiad.wmnet
+ - rdb1002.eqiad.wmnet
+ codfw: []
+ labsnfs:
name: "Labs NFS cluster"
id: 40
sites:
- - eqiad
+ eqiad:
+ - labstore1001.eqiad.wmnet
+ - labstore1003.eqiad.wmnet
cache_misc:
name: "Misc Web caching cluster"
id: 41
sites:
- - eqiad
+ eqiad:
+ - cp1043.eqiad.wmnet
+ - cp1044.eqiad.wmnet
elasticsearch:
name: "Elasticsearch cluster"
id: 42
sites:
- - eqiad
+ eqiad:
+ - elastic1001.eqiad.wmnet
+ - elastic1007.eqiad.wmnet
+ - elastic1013.eqiad.wmnet
logstash:
name: "Logstash cluster"
id: 43
sites:
- - eqiad
- rcstream:
+ eqiad:
+ - logstash1001.eqiad.wmnet
+ - logstash1003.eqiad.wmnet
+ rcstream:
name: "RCStream cluster"
id: 44
sites:
- - eqiad
+ eqiad:
+ - rcs1001.eqiad.wmnet
analytics_kafka:
name: "Analytics Kafka cluster"
id: 45
sites:
- - eqiad
+ eqiad:
+ - analytics1012.eqiad.wmnet
+ - analytics1018.eqiad.wmnet
+ - analytics1022.eqiad.wmnet
sca:
name: "Service Cluster A"
id: 46
sites:
- - eqiad
+ eqiad:
+ - sca1001.eqiad.wmnet
+ - sca1002.eqiad.wmnet
openldap_corp_mirror:
name: "Corp OIT LDAP mirror"
id: 47
sites:
- - eqiad
+ eqiad:
+ - plutonium.wikimedia.org
diff --git a/hieradata/esams.yaml b/hieradata/esams.yaml
index af849b7..2a2a4a1 100644
--- a/hieradata/esams.yaml
+++ b/hieradata/esams.yaml
@@ -1 +1,2 @@
ganglia_class: "new"
+ganglia_aggregators: hooft.esams.wikimedia.org:11649
diff --git a/manifests/role/ganglia.pp b/manifests/role/ganglia.pp
index 9c45a8f..52bc665 100644
--- a/manifests/role/ganglia.pp
+++ b/manifests/role/ganglia.pp
@@ -1,60 +1,7 @@
class role::ganglia::config {
# TODO: hiera this
$rra_sizes = '"RRA:AVERAGE:0.5:1:360" "RRA:AVERAGE:0.5:24:245"
"RRA:AVERAGE:0.5:168:241" "RRA:AVERAGE:0.5:672:241" "RRA:AVERAGE:0.5:5760:371"'
-
- $data_sources = {
- 'Video scalers eqiad' => 'tmh1001.eqiad.wmnet
tmh1002.eqiad.wmnet',
- 'Video scalers codfw' => 'install2001.wikimedia.org:10673',
- 'Image scalers eqiad' => 'mw1153.eqiad.wmnet
mw1154.eqiad.wmnet',
- 'Image scalers codfw' => 'install2001.wikimedia.org:10661',
- 'API application servers eqiad' => 'mw1114.eqiad.wmnet
mw1115.eqiad.wmnet',
- 'API application servers codfw' => 'install2001.wikimedia.org:10662',
- 'Application servers eqaid' => 'mw1054.eqiad.wmnet
mw1055.eqiad.wmnet',
- 'Application servers codfw' => 'install2001.wikimedia.org:10660',
- 'Jobrunners eqiad' => 'mw1001.eqiad.wmnet
mw1002.eqiad.wmnet',
- 'Jobrunners codfw' => 'install2001.wikimedia.org:10680
mw2001.codfw.wmnet mw2080.codfw.wmnet',
- 'MySQL' => 'db1050.eqiad.wmnet',
- 'PDF servers eqiad' => 'ocg1001.eqiad.wmnet',
- 'Fundraising eqiad' => 'pay-lvs1001.frack.eqiad.wmnet
pay-lvs1002.frack.eqiad.wmnet',
- 'Virtualization cluster eqiad' => 'labnet1001.eqiad.wmnet
virt1000.wikimedia.org',
- 'Labs NFS cluster eqiad' => 'labstore1001.eqiad.wmnet
labstore1003.eqiad.wmnet',
- 'MySQL eqiad' => 'dbstore1001.eqiad.wmnet
dbstore1002.eqiad.wmnet',
- 'LVS loadbalancers eqiad' => 'lvs1001.wikimedia.org
lvs1002.wikimedia.org',
- 'LVS loadbalancers codfw' => 'install2001.wikimedia.org:10651
lvs2001.codfw.wmnet lvs2002.codfw.wmnet',
- 'Miscellaneous eqiad' => 'carbon.wikimedia.org
ms1004.eqiad.wmnet',
- 'Miscellaneous codfw' => 'install2001.wikimedia.org:10657',
- 'Mobile caches eqiad' => 'cp1046.eqiad.wmnet
cp1047.eqiad.wmnet',
- 'Mobile caches esams' => 'hooft.esams.wikimedia.org:11677',
- 'Bits caches eqiad' => 'cp1056.eqiad.wmnet
cp1057.eqiad.wmnet',
- 'Upload caches eqiad' => 'cp1048.eqiad.wmnet
cp1061.eqiad.wmnet',
- 'Swift eqiad' => 'ms-fe1001.eqiad.wmnet
ms-fe1002.eqiad.wmnet',
- 'Swift esams' => 'hooft.esams.wikimedia.org:11676',
- 'Swift codfw' => 'install2001.wikimedia.org:10676',
- 'Bits caches esams' => 'hooft.esams.wikimedia.org:11670
cp3019.esams.wmnet cp3020.esams.wmnet',
- 'LVS loadbalancers esams' => 'hooft.esams.wikimedia.org:11651
lvs3001.esams.wmnet lvs3002.esams.wmnet',
- 'Miscellaneous esams' => 'hooft.esams.wikimedia.org:11657',
- 'Analytics cluster eqiad' => 'analytics1013.eqiad.wmnet
analytics1014.eqiad.wmnet analytics1019.eqiad.wmnet',
- 'Memcached eqiad' => 'mc1001.eqiad.wmnet
mc1002.eqiad.wmnet',
- 'Text caches esams' => 'hooft.esams.wikimedia.org:11669',
- 'Upload caches esams' => 'hooft.esams.wikimedia.org:11671
cp3003.esams.wmnet cp3004.esams.wmnet',
- 'Parsoid eqiad' => 'wtp1001.eqiad.wmnet
wtp1002.eqiad.wmnet',
- 'Parsoid Varnish eqiad' => 'cp1045.eqiad.wmnet
cp1058.eqiad.wmnet',
- 'Redis eqiad' => 'rdb1001.eqiad.wmnet
rdb1002.eqiad.wmnet',
- 'Text caches eqiad' => 'cp1052.eqiad.wmnet
cp1053.eqiad.wmnet',
- 'Misc Web caches eqiad' => 'cp1043.eqiad.wmnet
cp1044.eqiad.wmnet',
- 'LVS loadbalancers ulsfo' => 'lvs4001.ulsfo.wmnet
lvs4003.ulsfo.wmnet',
- 'Bits caches ulsfo' => 'cp4001.ulsfo.wmnet
cp4003.ulsfo.wmnet',
- 'Upload caches ulsfo' => 'cp4005.ulsfo.wmnet
cp4013.ulsfo.wmnet',
- 'Mobile caches ulsfo' => 'cp4011.ulsfo.wmnet
cp4019.ulsfo.wmnet',
- 'Text caches ulsfo' => 'cp4008.ulsfo.wmnet
cp4016.ulsfo.wmnet',
- 'Elasticsearch eqiad' => 'elastic1001.eqiad.wmnet
elastic1007.eqiad.wmnet elastic1013.eqiad.wmnet',
- 'Logstash eqiad' => 'logstash1001.eqiad.wmnet
logstash1003.eqiad.wmnet',
- 'RCStream eqiad' => 'rcs1001.eqiad.wmnet',
- 'Analytics Kafka cluster eqiad' => 'analytics1012.eqiad.wmnet
analytics1018.eqiad.wmnet analytics1022.eqiad.wmnet',
- 'Service Cluster A eqiad' => 'sca1001.eqiad.wmnet
sca1002.eqiad.wmnet',
- 'Corp OIT LDAP mirror eqiad' => 'plutonium.wikimedia.org',
- 'Corp OIT LDAP mirror codfw' => 'pollux.wikimedia.org',
- }
+ $data_sources = ganglia_aggregator_config()
}
# A role that includes all the needed stuff to run a ganglia web frontend
diff --git
a/modules/ganglia_new/lib/puppet/parser/functions/ganglia_aggregator_config.rb
b/modules/ganglia_new/lib/puppet/parser/functions/ganglia_aggregator_config.rb
new file mode 100644
index 0000000..5906e14
--- /dev/null
+++
b/modules/ganglia_new/lib/puppet/parser/functions/ganglia_aggregator_config.rb
@@ -0,0 +1,37 @@
+#
+# ganglia_aggregator_config - generates the aggregator config
+#
+def calc_url(aggregator, ip_octet)
+ url, port = aggregator.split(':')
+ port = port.to_i + ip_octet.to_i
+ return sprintf("%s:%d", url, port)
+end
+
+
+module Puppet::Parser::Functions
+ newfunction(:ganglia_aggregator_config, :type => :rvalue) do |args|
+ config = {}
+ site_config = {}
+ site_wide_aggregators = {}
+ clusters = function_hiera(['ganglia_clusters'])
+ clusters.each do |cluster, data|
+ data['sites'].each do |site, aggregators|
+ name = sprintf("%s %s", data['name'], site)
+ unless site_config.include? site
+ site_config[site] = function_hiera(['ganglia_class', nil, site])
+ end
+ if site_config[site] == 'old'
+ aggregator = aggregators.join(' ')
+ else
+ unless site_wide_aggregators.include? site
+ site_wide_aggregators[site] =
function_hiera(['ganglia_aggregators', nil, site])
+ end
+ # Compute the port to use
+ aggregator = calc_url(site_wide_aggregators[site], data['id'])
+ end
+ config[name] = aggregator
+ end
+ end
+ return config
+ end
+end
diff --git a/modules/ganglia_new/manifests/monitor/aggregator/instance.pp
b/modules/ganglia_new/manifests/monitor/aggregator/instance.pp
index 83746ea..5e228b2 100644
--- a/modules/ganglia_new/manifests/monitor/aggregator/instance.pp
+++ b/modules/ganglia_new/manifests/monitor/aggregator/instance.pp
@@ -9,7 +9,7 @@
$cluster = regsubst($title, '^(.*)_[^_]+$', '\1')
if has_key($ganglia_new::configuration::clusters[$cluster], 'sites') {
- $sites = $ganglia_new::configuration::clusters[$cluster]['sites']
+ $sites = keys($ganglia_new::configuration::clusters[$cluster]['sites'])
} else {
$sites = $ganglia_new::configuration::default_sites
}
--
To view, visit https://gerrit.wikimedia.org/r/198721
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ia32ea48028ff3b082f240cc01c602b0e9a6587f9
Gerrit-PatchSet: 7
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: Dzahn <dz...@wikimedia.org>
Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
