Rush has submitted this change and it was merged.
Change subject: diamond: wdqs and blazegraph collectors tweaks
......................................................................
diamond: wdqs and blazegraph collectors tweaks
* remove some debugging cruft
* call self.publish from collect
* file names are standard lower and snakecase
* we use a standard default prefix
(no need for special prefixing)
* move collectors to files/monitor
* set interval at default for aggregation integrity
Change-Id: I2bc918a177f42408b954875e623db03a7811d17c
---
R modules/wdqs/files/monitor/blazegraph.py
R modules/wdqs/files/monitor/wdqs_updater.py
M modules/wdqs/manifests/monitor/blazegraph.pp
M modules/wdqs/manifests/monitor/updater.pp
4 files changed, 24 insertions(+), 33 deletions(-)
Approvals:
Smalyshev: Looks good to me, but someone else must approve
Rush: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/wdqs/files/BlazegraphCollector.py
b/modules/wdqs/files/monitor/blazegraph.py
similarity index 86%
rename from modules/wdqs/files/BlazegraphCollector.py
rename to modules/wdqs/files/monitor/blazegraph.py
index cb8ee08..6b3a700 100644
--- a/modules/wdqs/files/BlazegraphCollector.py
+++ b/modules/wdqs/files/monitor/blazegraph.py
@@ -13,7 +13,6 @@
chelp.update({
'url': 'URL of the blazegraph instance',
'counters': 'List of counters to report',
- 'prefix': 'Metric prefix',
})
return chelp
@@ -25,7 +24,6 @@
config.update({
'url': 'http://localhost:9999/bigdata/',
'counters': ["/Query Engine/queryDoneCount"],
- 'prefix': 'blazegraph',
})
return config
@@ -34,19 +32,8 @@
if isinstance(self.config['counters'], basestring):
self.config['counters'] = [self.config['counters']]
- def collect(self):
- """
- Overrides the Collector.collect method
- """
-
- for counter in self.config['counters']:
- metric_name = self.config['prefix'] + self.query_to_metric(counter)
- metric_value = self.get_counter(counter)
- if metric_value is not None:
- self.publish(metric_name, metric_value)
-
def query_to_metric(self, qname):
- return qname.replace(' ', '_').replace('/', '.')
+ return qname.replace(' ', '_').replace('/', '.').lstrip('.')
def get_counter(self, cnt_name):
# Not sure why we need depth but some counters don't work without it
@@ -64,3 +51,10 @@
if cnt.attrib['name'] == last_name:
return cnt.attrib['value']
return None
+
+ def collect(self):
+ for counter in self.config['counters']:
+ metric_name = self.query_to_metric(counter)
+ metric_value = self.get_counter(counter)
+ if metric_value is not None:
+ self.publish(metric_name, metric_value)
diff --git a/modules/wdqs/files/WDQSUpdaterCollector.py
b/modules/wdqs/files/monitor/wdqs_updater.py
similarity index 77%
rename from modules/wdqs/files/WDQSUpdaterCollector.py
rename to modules/wdqs/files/monitor/wdqs_updater.py
index 8ce87a4..43304ce 100644
--- a/modules/wdqs/files/WDQSUpdaterCollector.py
+++ b/modules/wdqs/files/monitor/wdqs_updater.py
@@ -1,4 +1,9 @@
# coding=utf-8
+"""
+Requires:
+
+sudo for runner script
+"""
import diamond.collector
import urllib2
@@ -13,7 +18,6 @@
chelp = super(WDQSUpdaterCollector, self).get_default_config_help()
chelp.update({
'runner': 'Path to Jolokia runner',
- 'prefix': 'Metric prefix',
'counters': 'List of counters to collect',
'sudo_user': 'The user to use if using sudo',
})
@@ -26,7 +30,6 @@
config = super(WDQSUpdaterCollector, self).get_default_config()
config.update({
'runner': '/srv/wdqs/blazegraph/jolokia.sh',
- 'prefix': 'wdqs-updater',
'counters': ["updates/MeanRate", "batch-progress/MeanRate"],
'sudo_user': 'blazegraph',
})
@@ -43,28 +46,24 @@
subprocess.call(cmdline)
def query_to_metric(self, qname):
- name = qname.replace(' ', '_').replace('/', '.')
- return self.config['prefix']+'.'+name
+ return qname.replace(' ', '_').replace('/', '.')
def get_data(self, metric):
url = "%sread/metrics:name=%s" % (self.url, metric)
- #print url
req = urllib2.Request(url)
response = urllib2.urlopen(req)
- #print response
data = json.loads(response.read())
- #print data
- if 'value' not in data:
- return
- self.publish(self.query_to_metric(metric), data['value'])
+ if 'value' in data:
+ return data['value']
+ self.log.error("No value found in data")
def collect(self):
- """
- Overrides the Collector.collect method
- """
self.start_jolokia()
try:
for counter in self.config['counters']:
- self.get_data(counter)
+ data = self.get_data(counter)
+ if data:
+ self.publish(self.query_to_metric(counter),
+ data)
finally:
self.stop_jolokia()
diff --git a/modules/wdqs/manifests/monitor/blazegraph.pp
b/modules/wdqs/manifests/monitor/blazegraph.pp
index a81e0cf..36628ac 100644
--- a/modules/wdqs/manifests/monitor/blazegraph.pp
+++ b/modules/wdqs/manifests/monitor/blazegraph.pp
@@ -21,9 +21,8 @@
'"/JVM/Memory/Runtime Max Memory"',
'"/JVM/Memory/Runtime Total Memory"',
],
- interval => 300,
},
- source => 'puppet:///modules/wdqs/BlazegraphCollector.py',
+ source => 'puppet:///modules/wdqs/monitor/blazegraph.py',
}
# TODO: add monitoring of the http and https endpoints, and of the service
diff --git a/modules/wdqs/manifests/monitor/updater.pp
b/modules/wdqs/manifests/monitor/updater.pp
index afc227d..b8e0fa6 100644
--- a/modules/wdqs/manifests/monitor/updater.pp
+++ b/modules/wdqs/manifests/monitor/updater.pp
@@ -7,6 +7,7 @@
$username=$::wdqs::username,
) {
require ::wdqs::updater
+
diamond::collector { 'WDQSUpdater':
settings => {
runner => "${package_dir}/jolokia.sh",
@@ -15,9 +16,8 @@
'"batch-progress/MeanRate"',
],
sudo_user => $username,
- interval => 300,
},
- source => 'puppet:///modules/wdqs/WDQSUpdaterCollector.py',
+ source => 'puppet:///modules/wdqs/monitor/wdqs_updater.py',
}
sudo::user { 'diamond_to_blazegraph':
@@ -26,5 +26,4 @@
"ALL=(${username}) NOPASSWD: ${package_dir}/jolokia.sh"
],
}
-
}
--
To view, visit https://gerrit.wikimedia.org/r/235770
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I2bc918a177f42408b954875e623db03a7811d17c
Gerrit-PatchSet: 2
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Rush <r...@wikimedia.org>
Gerrit-Reviewer: Rush <r...@wikimedia.org>
Gerrit-Reviewer: Smalyshev <smalys...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
