Milimetric has submitted this change and it was merged.
Change subject: Support passing the exploded values by file path
......................................................................
Support passing the exploded values by file path
If there's only 1 value in an explode_by field, it will be
considered a path to a file that contains all values in
separate lines. The path should be relative to the query
folder.
This still supports the old way of exploding by wiki.
Bug: T132481
Change-Id: I825347635acde51acab5bacfda46593512f94a18
---
M reportupdater/reader.py
M test/fixtures/config/reportupdater_test4.yaml
M test/reader_test.py
M test/reportupdater_test.py
4 files changed, 43 insertions(+), 21 deletions(-)
Approvals:
Milimetric: Verified; Looks good to me, approved
diff --git a/reportupdater/reader.py b/reportupdater/reader.py
index 3f4a52f..0a0d408 100644
--- a/reportupdater/reader.py
+++ b/reportupdater/reader.py
@@ -60,7 +60,7 @@
report.lag = self.get_lag(report_config)
report.is_funnel = self.get_is_funnel(report_config)
report.first_date = self.get_first_date(report_config)
- report.explode_by = self.get_explode_by(report_config)
+ report.explode_by = self.get_explode_by(report_config, query_folder)
report.max_data_points = self.get_max_data_points(report_config)
executable = self.get_executable(report_config) or report_key
if report.type == 'sql':
@@ -144,14 +144,24 @@
return os.path.join(query_folder, report_key)
- def get_explode_by(self, report_config):
+ def get_explode_by(self, report_config, query_folder):
explode_by = {}
if 'by_wiki' in report_config and report_config['by_wiki'] is True:
explode_by['wiki'] = get_wikis(self.config) + ['all']
if 'explode_by' in report_config:
for placeholder, values_str in
report_config['explode_by'].iteritems():
values = [value.strip() for value in values_str.split(',')]
- explode_by[placeholder] = values
+ if len(values) == 1:
+ explode_path = os.path.join(query_folder, values[0])
+ try:
+ with io.open(explode_path, encoding='utf-8') as
explode_file:
+ read_values = [v.strip() for v in
explode_file.readlines()]
+ if (len(read_values) > 0):
+ explode_by[placeholder] = read_values
+ except IOError, e:
+ explode_by[placeholder] = values
+ elif len(values) > 1:
+ explode_by[placeholder] = values
return explode_by
diff --git a/test/fixtures/config/reportupdater_test4.yaml
b/test/fixtures/config/reportupdater_test4.yaml
index c8e148f..4de6c34 100644
--- a/test/fixtures/config/reportupdater_test4.yaml
+++ b/test/fixtures/config/reportupdater_test4.yaml
@@ -10,6 +10,6 @@
reportupdater_test4:
granularity: days
starts: 2015-01-01
- by_wiki: true
explode_by:
+ wiki: ../wikis.txt
editor: "visualeditor, wikitext"
diff --git a/test/reader_test.py b/test/reader_test.py
index 102af69..3046495 100644
--- a/test/reader_test.py
+++ b/test/reader_test.py
@@ -219,17 +219,32 @@
self.assertEqual(result, expected)
- def test_get_explode_by_wiki(self):
- self.config['wikis_path'] = 'test/fixtures/wikis.txt'
- result = self.reader.get_explode_by({})
- self.assertNotIn('wiki', result)
- result = self.reader.get_explode_by({'by_wiki': ('not', 'a', 'bool')})
- self.assertNotIn('wiki', result)
- result = self.reader.get_explode_by({'by_wiki': False})
- self.assertNotIn('wiki', result)
- result = self.reader.get_explode_by({'by_wiki': True})
- self.assertIn('wiki', result)
- self.assertEqual(result['wiki'], ['wiki1', 'wiki2', 'wiki3', 'all'])
+ def test_get_explode_by_using_file(self):
+ report_config = {
+ 'explode_by': {
+ 'wiki': '../wikis.txt'
+ }
+ }
+ query_folder = self.config['query_folder']
+ result = self.reader.get_explode_by(report_config, query_folder)
+ expected = {
+ 'wiki': ['wiki1', 'wiki2', 'wiki3']
+ }
+ self.assertEqual(result, expected)
+
+
+ def test_get_explode_by_with_one_element_that_is_not_a_file(self):
+ report_config = {
+ 'explode_by': {
+ 'wiki': 'somewiki'
+ }
+ }
+ query_folder = self.config['query_folder']
+ result = self.reader.get_explode_by(report_config, query_folder)
+ expected = {
+ 'wiki': ['somewiki']
+ }
+ self.assertEqual(result, expected)
def test_get_explode_by(self):
@@ -239,7 +254,8 @@
'language': 'en, de, fr'
}
}
- result = self.reader.get_explode_by(report_config)
+ query_folder = self.config['query_folder']
+ result = self.reader.get_explode_by(report_config, query_folder)
expected = {
'editor': ['visualeditor', 'wikitext'],
'language': ['en', 'de', 'fr']
diff --git a/test/reportupdater_test.py b/test/reportupdater_test.py
index 91c4df2..06e6671 100644
--- a/test/reportupdater_test.py
+++ b/test/reportupdater_test.py
@@ -223,12 +223,10 @@
pymysql.connect = MagicMock(return_value=connection_mock)
config_path = os.path.join(self.config_folder,
'reportupdater_test4.yaml')
- wikis_path = 'test/fixtures/wikis.txt'
reportupdater.run(
config_path=config_path,
query_folder=self.query_folder,
- output_folder=self.output_folder,
- wikis_path=wikis_path
+ output_folder=self.output_folder
)
output_folder = os.path.join(self.output_folder, 'reportupdater_test4')
@@ -238,11 +236,9 @@
'visualeditor/wiki1.tsv',
'visualeditor/wiki2.tsv',
'visualeditor/wiki3.tsv',
- 'visualeditor/all.tsv',
'wikitext/wiki1.tsv',
'wikitext/wiki2.tsv',
'wikitext/wiki3.tsv',
- 'wikitext/all.tsv'
]
for output_filename in output_filenames:
output_path = os.path.join(output_folder, output_filename)
--
To view, visit https://gerrit.wikimedia.org/r/306966
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I825347635acde51acab5bacfda46593512f94a18
Gerrit-PatchSet: 3
Gerrit-Project: analytics/reportupdater
Gerrit-Branch: master
Gerrit-Owner: Mforns <mfo...@wikimedia.org>
Gerrit-Reviewer: Milimetric <dandree...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
