[MediaWiki-commits] [Gerrit] diamond: wdqs and blazegraph collectors tweaks - change (operations/puppet)

[Rush (Code Review)]

Rush has submitted this change and it was merged.

Change subject: diamond: wdqs and blazegraph collectors tweaks
..


diamond: wdqs and blazegraph collectors tweaks

* remove some debugging cruft
* call self.publish from collect
* file names are standard lower and snakecase
* we use a standard default prefix
(no need for special prefixing)
* move collectors to files/monitor
* set interval at default for aggregation integrity

Change-Id: I2bc918a177f42408b954875e623db03a7811d17c
---
R modules/wdqs/files/monitor/blazegraph.py
R modules/wdqs/files/monitor/wdqs_updater.py
M modules/wdqs/manifests/monitor/blazegraph.pp
M modules/wdqs/manifests/monitor/updater.pp
4 files changed, 24 insertions(+), 33 deletions(-)

Approvals:
  Smalyshev: Looks good to me, but someone else must approve
  Rush: Looks good to me, approved
  jenkins-bot: Verified



diff --git a/modules/wdqs/files/BlazegraphCollector.py 
b/modules/wdqs/files/monitor/blazegraph.py
similarity index 86%
rename from modules/wdqs/files/BlazegraphCollector.py
rename to modules/wdqs/files/monitor/blazegraph.py
index cb8ee08..6b3a700 100644
--- a/modules/wdqs/files/BlazegraphCollector.py
+++ b/modules/wdqs/files/monitor/blazegraph.py
@@ -13,7 +13,6 @@
 chelp.update({
 'url': 'URL of the blazegraph instance',
 'counters': 'List of counters to report',
-'prefix': 'Metric prefix',
 })
 return chelp
 
@@ -25,7 +24,6 @@
 config.update({
 'url': 'http://localhost:/bigdata/',
 'counters': ["/Query Engine/queryDoneCount"],
-'prefix': 'blazegraph',
 })
 return config
 
@@ -34,19 +32,8 @@
 if isinstance(self.config['counters'], basestring):
 self.config['counters'] = [self.config['counters']]
 
-def collect(self):
-"""
-Overrides the Collector.collect method
-"""
-
-for counter in self.config['counters']:
-metric_name = self.config['prefix'] + self.query_to_metric(counter)
-metric_value = self.get_counter(counter)
-if metric_value is not None:
-self.publish(metric_name, metric_value)
-
 def query_to_metric(self, qname):
-return qname.replace(' ', '_').replace('/', '.')
+return qname.replace(' ', '_').replace('/', '.').lstrip('.')
 
 def get_counter(self, cnt_name):
 # Not sure why we need depth but some counters don't work without it
@@ -64,3 +51,10 @@
 if cnt.attrib['name'] == last_name:
 return cnt.attrib['value']
 return None
+
+def collect(self):
+for counter in self.config['counters']:
+metric_name = self.query_to_metric(counter)
+metric_value = self.get_counter(counter)
+if metric_value is not None:
+self.publish(metric_name, metric_value)
diff --git a/modules/wdqs/files/WDQSUpdaterCollector.py 
b/modules/wdqs/files/monitor/wdqs_updater.py
similarity index 77%
rename from modules/wdqs/files/WDQSUpdaterCollector.py
rename to modules/wdqs/files/monitor/wdqs_updater.py
index 8ce87a4..43304ce 100644
--- a/modules/wdqs/files/WDQSUpdaterCollector.py
+++ b/modules/wdqs/files/monitor/wdqs_updater.py
@@ -1,4 +1,9 @@
 # coding=utf-8
+"""
+Requires:
+
+sudo for runner script
+"""
 
 import diamond.collector
 import urllib2
@@ -13,7 +18,6 @@
 chelp = super(WDQSUpdaterCollector, self).get_default_config_help()
 chelp.update({
 'runner': 'Path to Jolokia runner',
-'prefix': 'Metric prefix',
 'counters': 'List of counters to collect',
 'sudo_user': 'The user to use if using sudo',
 })
@@ -26,7 +30,6 @@
 config = super(WDQSUpdaterCollector, self).get_default_config()
 config.update({
 'runner': '/srv/wdqs/blazegraph/jolokia.sh',
-'prefix': 'wdqs-updater',
 'counters': ["updates/MeanRate", "batch-progress/MeanRate"],
 'sudo_user': 'blazegraph',
 })
@@ -43,28 +46,24 @@
 subprocess.call(cmdline)
 
 def query_to_metric(self, qname):
-name = qname.replace(' ', '_').replace('/', '.')
-return self.config['prefix']+'.'+name
+return qname.replace(' ', '_').replace('/', '.')
 
 def get_data(self, metric):
 url = "%sread/metrics:name=%s" % (self.url, metric)
-#print url
 req = urllib2.Request(url)
 response = urllib2.urlopen(req)
-#print response
 data = json.loads(response.read())
-#print data
-if 'value' not in data:
-return
-self.publish(self.query_to_metric(metric), data['value'])
+if 'value' in data:
+return data['value']
+self.log.error("No value found in data")
 
 def collect(self):
-"""
-Overrides the Collector.collect method
-

[MediaWiki-commits] [Gerrit] diamond: wdqs and blazegraph collectors tweaks - change (operations/puppet)

[Rush (Code Review)]

Rush has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/235770

Change subject: diamond: wdqs and blazegraph collectors tweaks
..

diamond: wdqs and blazegraph collectors tweaks

* remove some debugging cruft
* call self.publish from collect
* file names are standard lower and snakecase
* we use a standard default prefix
(no need for special prefixing)
* move collectors to files/monitor
* set interval at default for aggregation integrity

Change-Id: I2bc918a177f42408b954875e623db03a7811d17c
---
R modules/wdqs/files/monitor/blazegraph.py
R modules/wdqs/files/monitor/wdqs_updater.py
M modules/wdqs/manifests/monitor/blazegraph.pp
M modules/wdqs/manifests/monitor/updater.pp
4 files changed, 24 insertions(+), 33 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/70/235770/1

diff --git a/modules/wdqs/files/BlazegraphCollector.py 
b/modules/wdqs/files/monitor/blazegraph.py
similarity index 86%
rename from modules/wdqs/files/BlazegraphCollector.py
rename to modules/wdqs/files/monitor/blazegraph.py
index cb8ee08..6b3a700 100644
--- a/modules/wdqs/files/BlazegraphCollector.py
+++ b/modules/wdqs/files/monitor/blazegraph.py
@@ -13,7 +13,6 @@
 chelp.update({
 'url': 'URL of the blazegraph instance',
 'counters': 'List of counters to report',
-'prefix': 'Metric prefix',
 })
 return chelp
 
@@ -25,7 +24,6 @@
 config.update({
 'url': 'http://localhost:/bigdata/',
 'counters': ["/Query Engine/queryDoneCount"],
-'prefix': 'blazegraph',
 })
 return config
 
@@ -34,19 +32,8 @@
 if isinstance(self.config['counters'], basestring):
 self.config['counters'] = [self.config['counters']]
 
-def collect(self):
-"""
-Overrides the Collector.collect method
-"""
-
-for counter in self.config['counters']:
-metric_name = self.config['prefix'] + self.query_to_metric(counter)
-metric_value = self.get_counter(counter)
-if metric_value is not None:
-self.publish(metric_name, metric_value)
-
 def query_to_metric(self, qname):
-return qname.replace(' ', '_').replace('/', '.')
+return qname.replace(' ', '_').replace('/', '.').lstrip('.')
 
 def get_counter(self, cnt_name):
 # Not sure why we need depth but some counters don't work without it
@@ -64,3 +51,10 @@
 if cnt.attrib['name'] == last_name:
 return cnt.attrib['value']
 return None
+
+def collect(self):
+for counter in self.config['counters']:
+metric_name = self.query_to_metric(counter)
+metric_value = self.get_counter(counter)
+if metric_value is not None:
+self.publish(metric_name, metric_value)
diff --git a/modules/wdqs/files/WDQSUpdaterCollector.py 
b/modules/wdqs/files/monitor/wdqs_updater.py
similarity index 77%
rename from modules/wdqs/files/WDQSUpdaterCollector.py
rename to modules/wdqs/files/monitor/wdqs_updater.py
index 8ce87a4..43304ce 100644
--- a/modules/wdqs/files/WDQSUpdaterCollector.py
+++ b/modules/wdqs/files/monitor/wdqs_updater.py
@@ -1,4 +1,9 @@
 # coding=utf-8
+"""
+Requires:
+
+sudo for runner script
+"""
 
 import diamond.collector
 import urllib2
@@ -13,7 +18,6 @@
 chelp = super(WDQSUpdaterCollector, self).get_default_config_help()
 chelp.update({
 'runner': 'Path to Jolokia runner',
-'prefix': 'Metric prefix',
 'counters': 'List of counters to collect',
 'sudo_user': 'The user to use if using sudo',
 })
@@ -26,7 +30,6 @@
 config = super(WDQSUpdaterCollector, self).get_default_config()
 config.update({
 'runner': '/srv/wdqs/blazegraph/jolokia.sh',
-'prefix': 'wdqs-updater',
 'counters': ["updates/MeanRate", "batch-progress/MeanRate"],
 'sudo_user': 'blazegraph',
 })
@@ -43,28 +46,24 @@
 subprocess.call(cmdline)
 
 def query_to_metric(self, qname):
-name = qname.replace(' ', '_').replace('/', '.')
-return self.config['prefix']+'.'+name
+return qname.replace(' ', '_').replace('/', '.')
 
 def get_data(self, metric):
 url = "%sread/metrics:name=%s" % (self.url, metric)
-#print url
 req = urllib2.Request(url)
 response = urllib2.urlopen(req)
-#print response
 data = json.loads(response.read())
-#print data
-if 'value' not in data:
-return
-self.publish(self.query_to_metric(metric), data['value'])
+if 'value' in data:
+return data['value']
+self.log.error("No value found in data")
 
 def collect(self):
-"""
-Overrides the Collector.collect method
-"""