[MediaWiki-commits] [Gerrit] analytics/reportupdater[master]: Support passing the exploded values by file path
Milimetric has submitted this change and it was merged.
Change subject: Support passing the exploded values by file path
..
Support passing the exploded values by file path
If there's only 1 value in an explode_by field, it will be
considered a path to a file that contains all values in
separate lines. The path should be relative to the query
folder.
This still supports the old way of exploding by wiki.
Bug: T132481
Change-Id: I825347635acde51acab5bacfda46593512f94a18
---
M reportupdater/reader.py
M test/fixtures/config/reportupdater_test4.yaml
M test/reader_test.py
M test/reportupdater_test.py
4 files changed, 43 insertions(+), 21 deletions(-)
Approvals:
Milimetric: Verified; Looks good to me, approved
diff --git a/reportupdater/reader.py b/reportupdater/reader.py
index 3f4a52f..0a0d408 100644
--- a/reportupdater/reader.py
+++ b/reportupdater/reader.py
@@ -60,7 +60,7 @@
report.lag = self.get_lag(report_config)
report.is_funnel = self.get_is_funnel(report_config)
report.first_date = self.get_first_date(report_config)
-report.explode_by = self.get_explode_by(report_config)
+report.explode_by = self.get_explode_by(report_config, query_folder)
report.max_data_points = self.get_max_data_points(report_config)
executable = self.get_executable(report_config) or report_key
if report.type == 'sql':
@@ -144,14 +144,24 @@
return os.path.join(query_folder, report_key)
-def get_explode_by(self, report_config):
+def get_explode_by(self, report_config, query_folder):
explode_by = {}
if 'by_wiki' in report_config and report_config['by_wiki'] is True:
explode_by['wiki'] = get_wikis(self.config) + ['all']
if 'explode_by' in report_config:
for placeholder, values_str in
report_config['explode_by'].iteritems():
values = [value.strip() for value in values_str.split(',')]
-explode_by[placeholder] = values
+if len(values) == 1:
+explode_path = os.path.join(query_folder, values[0])
+try:
+with io.open(explode_path, encoding='utf-8') as
explode_file:
+read_values = [v.strip() for v in
explode_file.readlines()]
+if (len(read_values) > 0):
+explode_by[placeholder] = read_values
+except IOError, e:
+explode_by[placeholder] = values
+elif len(values) > 1:
+explode_by[placeholder] = values
return explode_by
diff --git a/test/fixtures/config/reportupdater_test4.yaml
b/test/fixtures/config/reportupdater_test4.yaml
index c8e148f..4de6c34 100644
--- a/test/fixtures/config/reportupdater_test4.yaml
+++ b/test/fixtures/config/reportupdater_test4.yaml
@@ -10,6 +10,6 @@
reportupdater_test4:
granularity: days
starts: 2015-01-01
-by_wiki: true
explode_by:
+wiki: ../wikis.txt
editor: "visualeditor, wikitext"
diff --git a/test/reader_test.py b/test/reader_test.py
index 102af69..3046495 100644
--- a/test/reader_test.py
+++ b/test/reader_test.py
@@ -219,17 +219,32 @@
self.assertEqual(result, expected)
-def test_get_explode_by_wiki(self):
-self.config['wikis_path'] = 'test/fixtures/wikis.txt'
-result = self.reader.get_explode_by({})
-self.assertNotIn('wiki', result)
-result = self.reader.get_explode_by({'by_wiki': ('not', 'a', 'bool')})
-self.assertNotIn('wiki', result)
-result = self.reader.get_explode_by({'by_wiki': False})
-self.assertNotIn('wiki', result)
-result = self.reader.get_explode_by({'by_wiki': True})
-self.assertIn('wiki', result)
-self.assertEqual(result['wiki'], ['wiki1', 'wiki2', 'wiki3', 'all'])
+def test_get_explode_by_using_file(self):
+report_config = {
+'explode_by': {
+'wiki': '../wikis.txt'
+}
+}
+query_folder = self.config['query_folder']
+result = self.reader.get_explode_by(report_config, query_folder)
+expected = {
+'wiki': ['wiki1', 'wiki2', 'wiki3']
+}
+self.assertEqual(result, expected)
+
+
+def test_get_explode_by_with_one_element_that_is_not_a_file(self):
+report_config = {
+'explode_by': {
+'wiki': 'somewiki'
+}
+}
+query_folder = self.config['query_folder']
+result = self.reader.get_explode_by(report_config, query_folder)
+expected = {
+'wiki': ['somewiki']
+}
+self.assertEqual(result, expected)
def test_get_explode_by(self):
@@ -239,7 +254,8 @@
'language': 'en, de, fr'
}
}
-
[MediaWiki-commits] [Gerrit] analytics/reportupdater[master]: Support passing the exploded values by file path
Mforns has uploaded a new change for review.
https://gerrit.wikimedia.org/r/306966
Change subject: Support passing the exploded values by file path
..
Support passing the exploded values by file path
If there's only 1 value in an explode_by field, it will be
considered a path to a file that contains all values in
separate lines. The path should be relative to the query
folder.
This still supports the old way of exploding by wiki.
Bug: T132481
Change-Id: I825347635acde51acab5bacfda46593512f94a18
---
M reportupdater/reader.py
M test/fixtures/config/reportupdater_test4.yaml
M test/reader_test.py
M test/reportupdater_test.py
4 files changed, 29 insertions(+), 21 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/reportupdater
refs/changes/66/306966/1
diff --git a/reportupdater/reader.py b/reportupdater/reader.py
index 3f4a52f..605bed2 100644
--- a/reportupdater/reader.py
+++ b/reportupdater/reader.py
@@ -60,7 +60,7 @@
report.lag = self.get_lag(report_config)
report.is_funnel = self.get_is_funnel(report_config)
report.first_date = self.get_first_date(report_config)
-report.explode_by = self.get_explode_by(report_config)
+report.explode_by = self.get_explode_by(report_config, query_folder)
report.max_data_points = self.get_max_data_points(report_config)
executable = self.get_executable(report_config) or report_key
if report.type == 'sql':
@@ -144,14 +144,24 @@
return os.path.join(query_folder, report_key)
-def get_explode_by(self, report_config):
+def get_explode_by(self, report_config, query_folder):
explode_by = {}
if 'by_wiki' in report_config and report_config['by_wiki'] is True:
explode_by['wiki'] = get_wikis(self.config) + ['all']
if 'explode_by' in report_config:
for placeholder, values_str in
report_config['explode_by'].iteritems():
values = [value.strip() for value in values_str.split(',')]
-explode_by[placeholder] = values
+if len(values) == 1:
+explode_path = os.path.join(query_folder, values[0])
+try:
+with io.open(explode_path, encoding='utf-8') as
explode_file:
+read_values = [v.strip() for v in
explode_file.readlines()]
+except IOError, e:
+raise IOError('Could not read the explode by file (' +
str(e) + ').')
+if (len(read_values) > 0):
+explode_by[placeholder] = read_values
+elif len(values) > 1:
+explode_by[placeholder] = values
return explode_by
diff --git a/test/fixtures/config/reportupdater_test4.yaml
b/test/fixtures/config/reportupdater_test4.yaml
index c8e148f..4de6c34 100644
--- a/test/fixtures/config/reportupdater_test4.yaml
+++ b/test/fixtures/config/reportupdater_test4.yaml
@@ -10,6 +10,6 @@
reportupdater_test4:
granularity: days
starts: 2015-01-01
-by_wiki: true
explode_by:
+wiki: ../wikis.txt
editor: "visualeditor, wikitext"
diff --git a/test/reader_test.py b/test/reader_test.py
index 102af69..986638f 100644
--- a/test/reader_test.py
+++ b/test/reader_test.py
@@ -219,17 +219,18 @@
self.assertEqual(result, expected)
-def test_get_explode_by_wiki(self):
-self.config['wikis_path'] = 'test/fixtures/wikis.txt'
-result = self.reader.get_explode_by({})
-self.assertNotIn('wiki', result)
-result = self.reader.get_explode_by({'by_wiki': ('not', 'a', 'bool')})
-self.assertNotIn('wiki', result)
-result = self.reader.get_explode_by({'by_wiki': False})
-self.assertNotIn('wiki', result)
-result = self.reader.get_explode_by({'by_wiki': True})
-self.assertIn('wiki', result)
-self.assertEqual(result['wiki'], ['wiki1', 'wiki2', 'wiki3', 'all'])
+def test_get_explode_by_using_file(self):
+report_config = {
+'explode_by': {
+'wiki': '../wikis.txt'
+}
+}
+query_folder = self.config['query_folder']
+result = self.reader.get_explode_by(report_config, query_folder)
+expected = {
+'wiki': ['wiki1', 'wiki2', 'wiki3']
+}
+self.assertEqual(result, expected)
def test_get_explode_by(self):
@@ -239,7 +240,8 @@
'language': 'en, de, fr'
}
}
-result = self.reader.get_explode_by(report_config)
+query_folder = self.config['query_folder']
+result = self.reader.get_explode_by(report_config, query_folder)
expected = {
'editor': ['visualeditor', 'wikitext'],
'language': ['en', 'de', 'fr']
diff --git a/test/reportupdater_test.py
