Mforns has uploaded a new change for review.
https://gerrit.wikimedia.org/r/280201
Change subject: [WIP] Remove dynamic last data point
......................................................................
[WIP] Remove dynamic last data point
... details coming soon ...
Bug: T131049
Change-Id: I855731f0e132bf5c537f84ee72d187c82841e9e3
---
M reportupdater/executor.py
M reportupdater/reader.py
M reportupdater/report.py
M reportupdater/reportupdater.py
M reportupdater/selector.py
M reportupdater/writer.py
M test/executor_test.py
M test/fixtures/config/reportupdater_test1.yaml
M test/fixtures/config/reportupdater_test2.yaml
M test/fixtures/config/reportupdater_test3.yaml
M test/fixtures/config/reportupdater_test4.yaml
M test/fixtures/config/reportupdater_test5.yaml
M test/reader_test.py
M test/report_test.py
M test/reportupdater_test.py
M test/selector_test.py
16 files changed, 45 insertions(+), 336 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/reportupdater
refs/changes/01/280201/1
diff --git a/reportupdater/executor.py b/reportupdater/executor.py
index 7052e7e..c44d0aa 100644
--- a/reportupdater/executor.py
+++ b/reportupdater/executor.py
@@ -61,10 +61,10 @@
def instantiate_sql(self, report):
- values = {}
- if report.is_timeboxed:
- values['from_timestamp'] = report.start.strftime(TIMESTAMP_FORMAT)
- values['to_timestamp'] = report.end.strftime(TIMESTAMP_FORMAT)
+ values = {
+ 'from_timestamp': report.start.strftime(TIMESTAMP_FORMAT),
+ 'to_timestamp': report.end.strftime(TIMESTAMP_FORMAT)
+ }
values.update(report.explode_by)
try:
return report.sql_template.format(**values)
diff --git a/reportupdater/reader.py b/reportupdater/reader.py
index 60b0220..60772bf 100644
--- a/reportupdater/reader.py
+++ b/reportupdater/reader.py
@@ -56,12 +56,10 @@
report = Report()
report.key = report_key
report.type = self.get_type(report_config)
- report.frequency = self.get_frequency(report_config)
report.granularity = self.get_granularity(report_config)
report.lag = self.get_lag(report_config)
- report.is_timeboxed = self.get_is_timeboxed(report_config)
report.is_funnel = self.get_is_funnel(report_config)
- report.first_date = self.get_first_date(report_config,
report.is_timeboxed)
+ report.first_date = self.get_first_date(report_config)
report.explode_by = self.get_explode_by(report_config)
if report.type == 'sql':
report.db_key = self.get_db_key(report_config)
@@ -76,15 +74,6 @@
if report_type not in ['sql', 'script']:
raise ValueError('Report type is not valid.')
return report_type
-
-
- def get_frequency(self, report_config):
- if 'frequency' not in report_config:
- raise KeyError('Report frequency is not specified.')
- frequency = report_config['frequency']
- if frequency not in ['hours', 'days', 'weeks', 'months']:
- raise ValueError('Report frequency is not valid.')
- return frequency
def get_granularity(self, report_config):
@@ -105,15 +94,11 @@
return lag
- def get_is_timeboxed(self, report_config):
- return 'timeboxed' in report_config and report_config['timeboxed'] is
True
-
-
def get_is_funnel(self, report_config):
return 'funnel' in report_config and report_config['funnel'] is True
- def get_first_date(self, report_config, is_timeboxed):
+ def get_first_date(self, report_config):
if 'starts' in report_config:
first_date = report_config['starts']
if isinstance(first_date, date):
@@ -126,10 +111,8 @@
except ValueError:
raise ValueError('Report starts does not match date
format')
return first_date
- elif is_timeboxed:
- raise ValueError('Timeboxed report does not specify starts.')
else:
- return None
+ raise ValueError('Report does not specify starts.')
def get_db_key(self, report_config):
diff --git a/reportupdater/report.py b/reportupdater/report.py
index 5f82114..c58c4ea 100644
--- a/reportupdater/report.py
+++ b/reportupdater/report.py
@@ -17,10 +17,8 @@
def __init__(self):
self.key = None
self.type = None
- self.frequency = None
self.granularity = None
self.lag = 0
- self.is_timeboxed = False
self.is_funnel = False
self.first_date = None
self.start = None
@@ -37,10 +35,8 @@
'<Report' +
' key=' + str(self.key) +
' type=' + str(self.type) +
- ' frequency=' + str(self.frequency) +
' granularity=' + str(self.granularity) +
' lag=' + str(self.lag) +
- ' is_timeboxed=' + str(self.is_timeboxed) +
' is_funnel=' + str(self.is_funnel) +
' first_date=' + self.format_date(self.first_date) +
' start=' + self.format_date(self.start) +
diff --git a/reportupdater/reportupdater.py b/reportupdater/reportupdater.py
index 3198722..8ed5183 100644
--- a/reportupdater/reportupdater.py
+++ b/reportupdater/reportupdater.py
@@ -37,11 +37,9 @@
write_pid_file(params) # create lock to avoid concurrent executions
current_exec_time = utcnow()
- last_exec_time = replace_exec_time(current_exec_time,
params['history_path'])
config = load_config(params['config_path'])
config['current_exec_time'] = current_exec_time
- config['last_exec_time'] = last_exec_time
config['query_folder'] = params['query_folder']
config['output_folder'] = params['output_folder']
config['wikis_path'] = params['wikis_path']
@@ -63,7 +61,6 @@
query_folder = passed_params.pop('query_folder',
os.path.join(project_root, 'queries'))
process_params = {
'pid_file_path': os.path.join(query_folder, '.reportupdater.pid'),
- 'history_path': os.path.join(query_folder, '.reportupdater.history'),
'config_path': os.path.join(query_folder, 'config.yaml'),
'output_folder': os.path.join(project_root, 'output'),
'wikis_path': os.path.join(project_root, 'wikis.txt'),
@@ -139,21 +136,6 @@
os.remove(params['pid_file_path'])
except OSError, e:
logging.error('Unable to delete the pid file (' + str(e) + ').')
-
-
-def replace_exec_time(current_time, history_path):
- # Writes the current execution time to the history file.
- # If the file contains the last execution time, it is returned.
- if os.path.exists(history_path):
- with io.open(history_path) as history_file:
- last_time_str = history_file.read().strip()
- last_time = datetime.strptime(last_time_str, DATE_AND_TIME_FORMAT)
- else:
- last_time = None
- with io.open(history_path, 'w') as history_file:
- current_time_str = current_time.strftime(DATE_AND_TIME_FORMAT)
- history_file.write(unicode(current_time_str))
- return last_time
def load_config(config_path):
diff --git a/reportupdater/selector.py b/reportupdater/selector.py
index ce96468..c402b1f 100644
--- a/reportupdater/selector.py
+++ b/reportupdater/selector.py
@@ -4,7 +4,7 @@
# 1. The time that has passed sinnce the last execution
# 2. If the report data is up to date or not
#
-# It also divides timeboxed reports in intervals of one time unit.
+# It also divides reports in intervals of one time unit.
# For example, if the report in question has a monthly granularity,
# divides a 3-month report into 3 1-month reports.
@@ -32,38 +32,20 @@
def run(self):
if 'current_exec_time' not in self.config:
raise_critical(KeyError, 'Current exec time is not in config.')
- if 'last_exec_time' not in self.config:
- raise_critical(KeyError, 'Last exec time is not in config.')
now = self.config['current_exec_time']
- last_exec_time = self.config['last_exec_time']
if not isinstance(now, datetime):
raise_critical(ValueError, 'Current exec time is not a date.')
- if last_exec_time and last_exec_time > now:
- raise_critical(ValueError, 'Last exec time is greater than current
exec time.')
for report in self.reader.run():
logging.debug('Triaging "{report}"...'.format(report=str(report)))
try:
- if self.is_time_to_execute(last_exec_time, now,
report.frequency):
- for exploded_report in self.explode(report):
- if report.is_timeboxed:
- for interval_report in
self.get_interval_reports(exploded_report, now):
- yield interval_report
- else:
- yield exploded_report
+ for exploded_report in self.explode(report):
+ for interval_report in
self.get_interval_reports(exploded_report, now):
+ yield interval_report
except Exception, e:
message = ('Report "{report_key}" could not be triaged for
execution '
'because of error: {error}')
logging.error(message.format(report_key=report.key,
error=str(e)))
-
-
- def is_time_to_execute(self, last_exec_time, now, frequency):
- if last_exec_time:
- t1 = self.truncate_date(last_exec_time, frequency)
- else:
- t1 = None
- t2 = self.truncate_date(now, frequency)
- return t1 != t2
def get_interval_reports(self, report, now):
@@ -74,16 +56,15 @@
raise ValueError('Output folder is not a string.')
first_date = self.truncate_date(report.first_date, report.granularity)
- frequency_increment = self.get_increment(report.frequency)
lag_increment = relativedelta(seconds=report.lag)
granularity_increment = self.get_increment(report.granularity)
- relative_now = now - frequency_increment - lag_increment
+ relative_now = now - lag_increment - granularity_increment
last_date = self.truncate_date(relative_now, report.granularity)
previous_results = get_previous_results(report, output_folder)
already_done_dates = previous_results['data'].keys()
for start in self.get_all_start_dates(first_date, last_date,
granularity_increment):
- if start == last_date or start not in already_done_dates:
+ if start not in already_done_dates:
report_copy = deepcopy(report)
report_copy.start = start
report_copy.end = start + granularity_increment
diff --git a/reportupdater/writer.py b/reportupdater/writer.py
index 9f24601..00dc290 100644
--- a/reportupdater/writer.py
+++ b/reportupdater/writer.py
@@ -2,9 +2,6 @@
# This module is the last step of the pipeline.
# It gets the results passed from the executor,
# and updates the report's corresponding file.
-#
-# In the case of timeboxed reports, it handles the
-# update of previous results consistently.
import os
diff --git a/test/executor_test.py b/test/executor_test.py
index c41dbce..f5569d1 100644
--- a/test/executor_test.py
+++ b/test/executor_test.py
@@ -38,7 +38,6 @@
self.report = Report()
self.report.type = 'sql'
self.report.script = '/some/path'
- self.report.is_timeboxed = True
self.report.start = datetime(2015, 1, 1)
self.report.end = datetime(2015, 1, 2)
self.report.db_key = self.db_key
@@ -47,13 +46,13 @@
'AND date < {to_timestamp};')
- def
test_instantiate_sql_when_report_is_timeboxed_and_format_raises_error(self):
+ def test_instantiate_sql_when_format_raises_error(self):
self.report.sql_template = 'SOME sql WITH AN {unknown} placeholder;'
with self.assertRaises(ValueError):
self.executor.instantiate_sql(self.report)
- def test_instantiate_sql_when_report_is_timeboxed(self):
+ def test_instantiate_sql(self):
result = self.executor.instantiate_sql(self.report)
expected = self.report.sql_template.format(
from_timestamp=self.report.start.strftime(TIMESTAMP_FORMAT),
@@ -62,15 +61,7 @@
self.assertEqual(result, expected)
- def test_instantiate_sql_when_report_is_not_timeboxed(self):
- self.report.is_timeboxed = False
- self.report.sql_template = 'SOME sql CODE;'
- sql_query = self.executor.instantiate_sql(self.report)
- self.assertEqual(sql_query, self.report.sql_template)
-
-
def test_instantiate_sql_when_exploded_by_wiki(self):
- self.report.is_timeboxed = False
self.report.explode_by = {'wiki': 'wiki'}
self.report.sql_template = 'SOME sql WITH "{wiki}";'
sql_query = self.executor.instantiate_sql(self.report)
diff --git a/test/fixtures/config/reportupdater_test1.yaml
b/test/fixtures/config/reportupdater_test1.yaml
index 5d86410..beb95e1 100644
--- a/test/fixtures/config/reportupdater_test1.yaml
+++ b/test/fixtures/config/reportupdater_test1.yaml
@@ -8,7 +8,5 @@
db: reportupdater_db
reports:
reportupdater_test1:
- frequency: hours
granularity: days
- timeboxed: true
starts: 2015-01-01
diff --git a/test/fixtures/config/reportupdater_test2.yaml
b/test/fixtures/config/reportupdater_test2.yaml
index 35a74fb..acb444a 100644
--- a/test/fixtures/config/reportupdater_test2.yaml
+++ b/test/fixtures/config/reportupdater_test2.yaml
@@ -8,7 +8,5 @@
db: reportupdater_db
reports:
reportupdater_test2:
- frequency: days
granularity: days
- timeboxed: true
starts: 2015-01-01
diff --git a/test/fixtures/config/reportupdater_test3.yaml
b/test/fixtures/config/reportupdater_test3.yaml
index 744683f..4231bf5 100644
--- a/test/fixtures/config/reportupdater_test3.yaml
+++ b/test/fixtures/config/reportupdater_test3.yaml
@@ -8,8 +8,6 @@
db: reportupdater_db
reports:
reportupdater_test3:
- frequency: hours
granularity: days
- timeboxed: true
funnel: true
starts: 2015-01-01
diff --git a/test/fixtures/config/reportupdater_test4.yaml
b/test/fixtures/config/reportupdater_test4.yaml
index 6ba35a9..c8e148f 100644
--- a/test/fixtures/config/reportupdater_test4.yaml
+++ b/test/fixtures/config/reportupdater_test4.yaml
@@ -8,7 +8,6 @@
db: reportupdater_db
reports:
reportupdater_test4:
- frequency: hours
granularity: days
starts: 2015-01-01
by_wiki: true
diff --git a/test/fixtures/config/reportupdater_test5.yaml
b/test/fixtures/config/reportupdater_test5.yaml
index b60802c..7563617 100644
--- a/test/fixtures/config/reportupdater_test5.yaml
+++ b/test/fixtures/config/reportupdater_test5.yaml
@@ -1,7 +1,5 @@
reports:
reportupdater_test5:
- frequency: days
granularity: days
- timeboxed: true
starts: 2015-01-01
type: script
diff --git a/test/reader_test.py b/test/reader_test.py
index 755f054..7545990 100644
--- a/test/reader_test.py
+++ b/test/reader_test.py
@@ -15,8 +15,6 @@
self.report_key = 'reader_test'
self.report_config = {
'starts': '2015-01-01',
- 'timeboxed': True,
- 'frequency': 'hours',
'granularity': 'days'
}
self.config = {
@@ -51,30 +49,21 @@
self.assertEqual(result, 'script')
- def test_get_frequency_and_granularity_when_value_is_not_in_config(self):
+ def test_get_granularity_when_value_is_not_in_config(self):
report_config = {}
- with self.assertRaises(KeyError):
- self.reader.get_frequency(report_config)
with self.assertRaises(KeyError):
self.reader.get_granularity(report_config)
- def test_get_frequency_and_granularity_when_value_is_not_valid(self):
+ def test_get_granularity_when_value_is_not_valid(self):
report_config = {
- 'frequency': 'wrong',
'granularity': 'wrong'
}
with self.assertRaises(ValueError):
- self.reader.get_frequency(report_config)
- with self.assertRaises(ValueError):
self.reader.get_granularity(report_config)
- def test_get_frequency_and_granularity(self):
- for frequency in ['hours', 'days', 'weeks', 'months']:
- report_config = {'frequency': frequency}
- result = self.reader.get_frequency(report_config)
- self.assertEqual(result, frequency)
+ def test_get_granularity(self):
for granularity in ['days', 'weeks', 'months']:
report_config = {'granularity': granularity}
result = self.reader.get_granularity(report_config)
@@ -102,25 +91,6 @@
self.assertEqual(result, 10)
- def test_get_is_timeboxed_when_report_timeboxed_is_not_in_config(self):
- report_config = {}
- is_timeboxed = self.reader.get_is_timeboxed(report_config)
- self.assertFalse(is_timeboxed)
-
-
- def test_get_is_timeboxed_when_report_timeboxed_is_not_true(self):
- for value in [False, None, 0]:
- report_config = {'timeboxed': value}
- is_timeboxed = self.reader.get_is_timeboxed(report_config)
- self.assertFalse(is_timeboxed)
-
-
- def test_get_is_timeboxed_when_report_timeboxed_is_true(self):
- report_config = {'timeboxed': True}
- is_timeboxed = self.reader.get_is_timeboxed(report_config)
- self.assertTrue(is_timeboxed)
-
-
def test_get_is_funnel_when_report_funnel_is_not_in_config(self):
report_config = {}
is_funnel = self.reader.get_is_funnel(report_config)
@@ -142,37 +112,26 @@
def test_get_first_date_when_report_starts_is_not_a_string(self):
report_config = {'starts': ('not', 'a', 'string')}
- is_timeboxed = True
with self.assertRaises(TypeError):
- self.reader.get_first_date(report_config, is_timeboxed)
+ self.reader.get_first_date(report_config)
def
test_get_first_date_when_report_starts_does_not_match_date_format(self):
report_config = {'starts': 'no match'}
- is_timeboxed = True
with self.assertRaises(ValueError):
- self.reader.get_first_date(report_config, is_timeboxed)
-
-
- def
test_get_first_date_when_report_starts_is_not_in_timeboxed_config(self):
- report_config = {}
- is_timeboxed = True
- with self.assertRaises(ValueError):
- self.reader.get_first_date(report_config, is_timeboxed)
+ self.reader.get_first_date(report_config)
def test_get_first_date_when_report_starts_is_not_in_config(self):
report_config = {}
- is_timeboxed = False
- first_date = self.reader.get_first_date(report_config, is_timeboxed)
- self.assertEqual(first_date, None)
+ with self.assertRaises(ValueError):
+ self.reader.get_first_date(report_config)
def test_get_first_date(self):
date_str = '2015-01-01'
report_config = {'starts': date_str}
- is_timeboxed = True
- result = self.reader.get_first_date(report_config, is_timeboxed)
+ result = self.reader.get_first_date(report_config)
expected = datetime.strptime(date_str, DATE_FORMAT)
self.assertEqual(result, expected)
@@ -309,9 +268,7 @@
def test_create_sql_report(self):
self.reader.get_type = MagicMock(return_value='sql')
self.reader.get_first_date = MagicMock(return_value='first_date')
- self.reader.get_frequency = MagicMock(return_value='frequency')
self.reader.get_granularity = MagicMock(return_value='granularity')
- self.reader.get_is_timeboxed = MagicMock(return_value='is_timeboxed')
self.reader.get_is_funnel = MagicMock(return_value='is_funnel')
self.reader.get_db_key = MagicMock(return_value='db_key')
self.reader.get_sql_template = MagicMock(return_value='sql_template')
@@ -320,9 +277,7 @@
self.assertEqual(report.key, self.report_key)
self.assertEqual(report.type, 'sql')
self.assertEqual(report.first_date, 'first_date')
- self.assertEqual(report.frequency, 'frequency')
self.assertEqual(report.granularity, 'granularity')
- self.assertEqual(report.is_timeboxed, 'is_timeboxed')
self.assertEqual(report.is_funnel, 'is_funnel')
self.assertEqual(report.db_key, 'db_key')
self.assertEqual(report.sql_template, 'sql_template')
@@ -336,9 +291,7 @@
def test_create_script_report(self):
self.reader.get_type = MagicMock(return_value='script')
self.reader.get_first_date = MagicMock(return_value='first_date')
- self.reader.get_frequency = MagicMock(return_value='frequency')
self.reader.get_granularity = MagicMock(return_value='granularity')
- self.reader.get_is_timeboxed = MagicMock(return_value='is_timeboxed')
self.reader.get_is_funnel = MagicMock(return_value='is_funnel')
self.reader.get_db_key = MagicMock(return_value='db_key')
self.reader.get_sql_template = MagicMock(return_value='sql_template')
@@ -347,9 +300,7 @@
self.assertEqual(report.key, self.report_key)
self.assertEqual(report.type, 'script')
self.assertEqual(report.first_date, 'first_date')
- self.assertEqual(report.frequency, 'frequency')
self.assertEqual(report.granularity, 'granularity')
- self.assertEqual(report.is_timeboxed, 'is_timeboxed')
self.assertEqual(report.is_funnel, 'is_funnel')
self.assertEqual(report.db_key, None)
self.assertEqual(report.sql_template, None)
diff --git a/test/report_test.py b/test/report_test.py
index 23104f8..134f9a0 100644
--- a/test/report_test.py
+++ b/test/report_test.py
@@ -11,10 +11,8 @@
self.report = Report()
self.report.key = 'report_test'
self.report.type = 'sql'
- self.report.frequency = 'hours'
self.report.granularity = 'days'
self.report.lag = 0
- self.report.is_timeboxed = True
self.report.is_funnel = True
self.report.first_date = datetime(2015, 1, 1)
self.report.start = datetime(2015, 1, 2)
diff --git a/test/reportupdater_test.py b/test/reportupdater_test.py
index 634d04b..9518a85 100644
--- a/test/reportupdater_test.py
+++ b/test/reportupdater_test.py
@@ -24,7 +24,6 @@
self.query_folder = 'test/fixtures/queries'
self.output_folder = 'test/fixtures/output'
self.pid_file_path = 'test/fixtures/queries/.reportupdater.pid'
- self.history_path = 'test/fixtures/reportupdater_test.history'
self.paths_to_clean = [self.pid_file_path]
@@ -39,38 +38,6 @@
shutil.rmtree(path)
except:
pass
-
-
- def
test_when_current_exec_time_and_last_exec_time_are_within_the_same_hour(self):
- last_exec_time = datetime(2015, 1, 2, 3, 4, 5)
- self.write_time_to_history(last_exec_time)
- reportupdater.utcnow = MagicMock(return_value=datetime(2015, 1, 2, 3,
40, 50))
- reportupdater.run(
- config_path=os.path.join(self.config_folder,
'reportupdater_test1.yaml'),
- query_folder=self.query_folder,
- output_folder=self.output_folder,
- history_path=self.history_path
- )
- # The report should not be computed because it has already been
computed
- # within this hour. So the output file should not exist.
- output_path = os.path.join(self.output_folder,
'reportupdater_test1.tsv')
- self.assertFalse(os.path.exists(output_path))
-
-
- def
test_when_current_exec_time_and_last_exec_time_are_within_the_same_day(self):
- last_exec_time = datetime(2015, 1, 2, 3, 4, 5)
- self.write_time_to_history(last_exec_time)
- reportupdater.utcnow = MagicMock(return_value=datetime(2015, 1, 2, 13,
14, 15))
- reportupdater.run(
- config_path=os.path.join(self.config_folder,
'reportupdater_test2.yaml'),
- query_folder=self.query_folder,
- output_folder=self.output_folder,
- history_path=self.history_path
- )
- # The report should not be computed because it has already been
computed
- # within this day. So the output file should not exist.
- output_path = os.path.join(self.output_folder,
'reportupdater_test2.tsv')
- self.assertFalse(os.path.exists(output_path))
def test_when_two_threads_run_reportupdater_in_parallel(self):
@@ -88,14 +55,12 @@
MySQLdb.connect = MagicMock(wraps=connect_with_lag)
# The first thread should execute normally and output the results.
- history_path1 = 'test/fixtures/reportupdater_test1.history'
output_path1 = os.path.join(self.output_folder,
'reportupdater_test1.tsv')
- self.paths_to_clean.extend([history_path1, output_path1])
+ self.paths_to_clean.extend([output_path1])
args1 = {
'config_path': os.path.join(self.config_folder,
'reportupdater_test1.yaml'),
'query_folder': self.query_folder,
- 'output_folder': self.output_folder,
- 'history_path': history_path1
+ 'output_folder': self.output_folder
}
thread1 = Thread(target=reportupdater.run, kwargs=args1)
thread1.start()
@@ -103,17 +68,13 @@
# The second thread will start when the first thread is still running,
# so it should be discarded by the pidfile control
# and no output should be written.
- # Note that the history file is different, so that
- # the frequency control does not discard this thread.
time.sleep(0.1)
- history_path2 = 'test/fixtures/reportupdater_test2.history'
output_path2 = os.path.join(self.output_folder,
'reportupdater_test2.tsv')
- self.paths_to_clean.extend([history_path2, output_path2])
+ self.paths_to_clean.extend([output_path2])
args2 = {
'config_path': os.path.join(self.config_folder,
'reportupdater_test2.yaml'),
'query_folder': self.query_folder,
- 'output_folder': self.output_folder,
- 'history_path': history_path2
+ 'output_folder': self.output_folder
}
thread2 = Thread(target=reportupdater.run, kwargs=args2)
thread2.start()
@@ -127,7 +88,7 @@
self.assertFalse(os.path.exists(output_path2))
- def test_hourly_timeboxed_report_without_previous_results(self):
+ def test_hourly_report_without_previous_results(self):
def fetchall_callback():
# This method will return a subsequent row with each call.
try:
@@ -145,13 +106,11 @@
config_path = os.path.join(self.config_folder,
'reportupdater_test1.yaml')
output_path = os.path.join(self.output_folder,
'reportupdater_test1.tsv')
- history_path = 'test/fixtures/reportupdater_test1.history'
- self.paths_to_clean.extend([output_path, history_path])
+ self.paths_to_clean.extend([output_path])
reportupdater.run(
config_path=config_path,
query_folder=self.query_folder,
- output_folder=self.output_folder,
- history_path=history_path
+ output_folder=self.output_folder
)
self.assertTrue(os.path.exists(output_path))
with io.open(output_path, 'r', encoding='utf-8') as output_file:
@@ -169,7 +128,7 @@
expected_value += 1
- def test_hourly_funnel_timeboxed_report_without_previous_results(self):
+ def test_hourly_funnel_report_without_previous_results(self):
def fetchall_callback():
# This method will return a subsequent row with each call.
try:
@@ -188,13 +147,11 @@
config_path = os.path.join(self.config_folder,
'reportupdater_test3.yaml')
output_path = os.path.join(self.output_folder,
'reportupdater_test3.tsv')
- history_path = 'test/fixtures/reportupdater_test3.history'
- self.paths_to_clean.extend([output_path, history_path])
+ self.paths_to_clean.extend([output_path])
reportupdater.run(
config_path=config_path,
query_folder=self.query_folder,
- output_folder=self.output_folder,
- history_path=history_path
+ output_folder=self.output_folder
)
self.assertTrue(os.path.exists(output_path))
with io.open(output_path, 'r', encoding='utf-8') as output_file:
@@ -215,7 +172,7 @@
expected_value = 1
- def test_daily_timeboxed_report_with_previous_results(self):
+ def test_daily_report_with_previous_results(self):
def fetchall_callback():
# This method will return a subsequent row with each call.
try:
@@ -234,15 +191,13 @@
config_path = os.path.join(self.config_folder,
'reportupdater_test2.yaml')
output_path = os.path.join(self.output_folder,
'reportupdater_test2.tsv')
- history_path = 'test/fixtures/reportupdater_test2.history'
with io.open(output_path, 'w') as output_file:
output_file.write(unicode('date\tvalue\n2015-01-01\t1\n2015-02-01\t2\n'))
- self.paths_to_clean.extend([output_path, history_path])
+ self.paths_to_clean.extend([output_path])
reportupdater.run(
config_path=config_path,
query_folder=self.query_folder,
- output_folder=self.output_folder,
- history_path=history_path
+ output_folder=self.output_folder
)
self.assertTrue(os.path.exists(output_path))
with io.open(output_path, 'r', encoding='utf-8') as output_file:
@@ -260,7 +215,7 @@
expected_value += 1
- def
test_daily_not_timeboxed_report_without_previous_results_with_explode_by(self):
+ def test_daily_report_without_previous_results_with_explode_by(self):
def fetchall_callback():
return [[datetime(2015, 1, 1), str(1)]]
header = ['date', 'value']
@@ -268,18 +223,16 @@
MySQLdb.connect = MagicMock(return_value=connection_mock)
config_path = os.path.join(self.config_folder,
'reportupdater_test4.yaml')
- history_path = 'test/fixtures/reportupdater_test4.history'
wikis_path = 'test/fixtures/wikis.txt'
reportupdater.run(
config_path=config_path,
query_folder=self.query_folder,
output_folder=self.output_folder,
- history_path=history_path,
wikis_path=wikis_path
)
output_folder = os.path.join(self.output_folder, 'reportupdater_test4')
- self.paths_to_clean.extend([output_folder, history_path])
+ self.paths_to_clean.extend([output_folder])
output_filenames = [
'visualeditor/wiki1.tsv',
@@ -301,17 +254,15 @@
self.assertEqual(output_lines[1], '2015-01-01\t1\n')
- def test_daily_timeboxed_script_report_without_previous_results(self):
+ def test_daily_script_report_without_previous_results(self):
config_path = os.path.join(self.config_folder,
'reportupdater_test5.yaml')
- history_path = 'test/fixtures/reportupdater_test5.history'
reportupdater.run(
config_path=config_path,
query_folder=self.query_folder,
- output_folder=self.output_folder,
- history_path=history_path
+ output_folder=self.output_folder
)
output_path = os.path.join(self.output_folder,
'reportupdater_test5.tsv')
- self.paths_to_clean.extend([output_path, history_path])
+ self.paths_to_clean.extend([output_path])
self.assertTrue(os.path.exists(output_path))
with io.open(output_path, 'r', encoding='utf-8') as output_file:
@@ -327,10 +278,3 @@
self.assertEqual(date_str, expected_date_str)
self.assertEqual(type(value), unicode)
expected_date += relativedelta(days=+1)
-
-
- def write_time_to_history(self, last_exec_time):
- last_exec_time_str = last_exec_time.strftime(DATE_AND_TIME_FORMAT)
- with io.open(self.history_path, 'w') as history_file:
- history_file.write(unicode(last_exec_time_str))
- self.paths_to_clean.append(self.history_path)
diff --git a/test/selector_test.py b/test/selector_test.py
index 2a252f9..eb41d0e 100644
--- a/test/selector_test.py
+++ b/test/selector_test.py
@@ -16,7 +16,6 @@
def setUp(self):
self.config = {
'output_folder': 'test/fixtures/output',
- 'last_exec_time': datetime(2015, 1, 2, 23, 50, 30),
'current_exec_time': datetime(2015, 1, 3, 1, 20, 30)
}
reader = Reader(self.config)
@@ -25,65 +24,7 @@
self.report = Report()
self.report.key = 'selector_test'
self.report.first_date = datetime(2015, 1, 1)
- self.report.frequency = 'hours'
self.report.granularity = 'days'
- self.report.is_timeboxed = True
-
-
- def test_is_time_to_execute_when_last_exec_time_is_none(self):
- last_exec_time = None
- now = datetime.now()
- frequency = 'hours'
- is_time = self.selector.is_time_to_execute(last_exec_time, now,
frequency)
- self.assertTrue(is_time)
-
-
- def test_is_time_to_execute_when_both_dates_are_in_the_same_hour(self):
- last_exec_time = datetime(2015, 1, 1, 3, 30, 0)
- now = datetime(2015, 1, 1, 3, 40, 0)
- frequency = 'hours'
- is_time = self.selector.is_time_to_execute(last_exec_time, now,
frequency)
- self.assertFalse(is_time)
-
-
- def test_is_time_to_execute_when_both_dates_are_in_different_hours(self):
- last_exec_time = datetime(2015, 1, 1, 3, 30, 0)
- now = datetime(2015, 1, 1, 4, 20, 0)
- frequency = 'hours'
- is_time = self.selector.is_time_to_execute(last_exec_time, now,
frequency)
- self.assertTrue(is_time)
-
-
- def test_is_time_to_execute_when_both_dates_are_in_the_same_day(self):
- last_exec_time = datetime(2015, 1, 1, 3, 30, 0)
- now = datetime(2015, 1, 1, 10, 40, 0)
- frequency = 'days'
- is_time = self.selector.is_time_to_execute(last_exec_time, now,
frequency)
- self.assertFalse(is_time)
-
-
- def test_is_time_to_execute_when_both_dates_are_in_different_days(self):
- last_exec_time = datetime(2015, 1, 1, 3, 30, 0)
- now = datetime(2015, 1, 2, 4, 20, 0)
- frequency = 'days'
- is_time = self.selector.is_time_to_execute(last_exec_time, now,
frequency)
- self.assertTrue(is_time)
-
-
- def test_is_time_to_execute_when_both_dates_are_in_the_same_week(self):
- last_exec_time = datetime(2015, 1, 1, 3, 30, 0)
- now = datetime(2015, 1, 3, 10, 40, 0)
- frequency = 'weeks'
- is_time = self.selector.is_time_to_execute(last_exec_time, now,
frequency)
- self.assertFalse(is_time)
-
-
- def test_is_time_to_execute_when_both_dates_are_in_different_weeks(self):
- last_exec_time = datetime(2015, 1, 1, 3, 30, 0)
- now = datetime(2015, 1, 5, 4, 20, 0)
- frequency = 'days'
- is_time = self.selector.is_time_to_execute(last_exec_time, now,
frequency)
- self.assertTrue(is_time)
def test_get_interval_reports_when_previous_results_is_empty(self):
@@ -112,15 +53,12 @@
# see: test/fixtures/output/selector_test2.tsv
now = datetime(2015, 1, 3)
reports = list(self.selector.get_interval_reports(self.report, now))
- self.assertEqual(len(reports), 1)
- self.assertEqual(reports[0].start, datetime(2015, 1, 2))
- self.assertEqual(reports[0].end, datetime(2015, 1, 3))
+ self.assertEqual(len(reports), 0)
def test_get_interval_reports_when_lag_is_set(self):
# Note no previous results tsv exists for default report.
now = datetime(2015, 1, 3)
- self.report.frequency = 'days'
self.report.lag = 100 # 1 minute and 40 seconds
reports = list(self.selector.get_interval_reports(self.report, now))
self.assertEqual(len(reports), 1)
@@ -248,48 +186,20 @@
for report in reports:
self.assertEqual(report.key, self.report.key)
self.assertEqual(report.first_date, self.report.first_date)
- self.assertEqual(report.frequency, self.report.frequency)
self.assertEqual(report.granularity, self.report.granularity)
- self.assertEqual(report.is_timeboxed, self.report.is_timeboxed)
self.assertIn(report.explode_by, expected_values)
expected_values.remove(report.explode_by)
-
-
- def test_run_when_last_exec_time_is_greater_than_current_exec_time(self):
- self.config['last_exec_time'] = datetime(2015, 1, 2)
- self.config['current_exec_time'] = datetime(2015, 1, 1)
- with self.assertRaises(ValueError):
- list(self.selector.run())
def test_run_when_helper_method_raises_error(self):
read = [self.report]
self.selector.reader.run = MagicMock(return_value=read)
- self.selector.is_time_to_execute = MagicMock(side_effect=Exception())
+ self.selector.get_interval_reports = MagicMock(side_effect=Exception())
selected = list(self.selector.run())
self.assertEqual(len(selected), 0)
- def test_run_when_not_is_time_to_execute(self):
- read = [self.report]
- self.selector.reader.run = MagicMock(return_value=read)
- self.selector.is_time_to_execute = MagicMock(return_value=False)
- selected = list(self.selector.run())
- self.assertEqual(len(selected), 0)
-
-
- def test_run_when_not_is_timeboxed(self):
- self.report.is_timeboxed = False
- read = [self.report]
- self.selector.reader.run = MagicMock(return_value=read)
- self.selector.is_time_to_execute = MagicMock(return_value=True)
- selected = list(self.selector.run())
- self.assertEqual(len(selected), 1)
- self.assertEqual(selected[0], self.report)
-
-
- def test_run_when_is_timeboxed(self):
- self.report.is_timeboxed = True
+ def test_run(self):
read = [self.report]
self.selector.reader.run = MagicMock(return_value=read)
self.selector.is_time_to_execute = MagicMock(return_value=True)
@@ -300,9 +210,8 @@
self.assertEqual(selected[1], self.report)
- def test_run_when_should_explode_and_is_timeboxed(self):
+ def test_run_when_should_explode(self):
self.report.explode_by = {'wiki': ['enwiki', 'dewiki', 'all']}
- self.report.is_timeboxed = True
def get_interval_reports_mock(report, now):
yield report
@@ -311,20 +220,6 @@
self.selector.reader.run = MagicMock(return_value=read)
self.selector.is_time_to_execute = MagicMock(return_value=True)
self.selector.get_interval_reports =
MagicMock(wraps=get_interval_reports_mock)
- selected = list(self.selector.run())
- self.assertEqual(len(selected), 3)
- self.assertEqual(selected[0].explode_by, {'wiki': 'enwiki'})
- self.assertEqual(selected[1].explode_by, {'wiki': 'dewiki'})
- self.assertEqual(selected[2].explode_by, {'wiki': 'all'})
-
-
- def test_run_when_should_explode_and_is_not_timeboxed(self):
- self.report.explode_by = {'wiki': ['enwiki', 'dewiki', 'all']}
- self.report.is_timeboxed = False
-
- read = [self.report]
- self.selector.reader.run = MagicMock(return_value=read)
- self.selector.is_time_to_execute = MagicMock(return_value=True)
selected = list(self.selector.run())
self.assertEqual(len(selected), 3)
self.assertEqual(selected[0].explode_by, {'wiki': 'enwiki'})
--
To view, visit https://gerrit.wikimedia.org/r/280201
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I855731f0e132bf5c537f84ee72d187c82841e9e3
Gerrit-PatchSet: 1
Gerrit-Project: analytics/reportupdater
Gerrit-Branch: master
Gerrit-Owner: Mforns <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
