Milimetric has submitted this change and it was merged.
Change subject: Add lag option to reportupdater
......................................................................
Add lag option to reportupdater
Adds a 'lag' option to the report configuration (yaml)
that permits waiting for a given time before running
a report. This is useful for example if the source db
has lag.
Bug: T117091
Change-Id: I9b01c608bd912e80e31cdb372cb9e45441f96f76
---
M reportupdater/reader.py
M reportupdater/report.py
M reportupdater/selector.py
M test/reader_test.py
M test/report_test.py
M test/selector_test.py
6 files changed, 50 insertions(+), 3 deletions(-)
Approvals:
Milimetric: Verified; Looks good to me, approved
diff --git a/reportupdater/reader.py b/reportupdater/reader.py
index 1a7245f..866d39f 100644
--- a/reportupdater/reader.py
+++ b/reportupdater/reader.py
@@ -58,6 +58,7 @@
report.type = self.get_type(report_config)
report.frequency = self.get_frequency(report_config)
report.granularity = self.get_granularity(report_config)
+ report.lag = self.get_lag(report_config)
report.is_timeboxed = self.get_is_timeboxed(report_config)
report.is_funnel = self.get_is_funnel(report_config)
report.first_date = self.get_first_date(report_config,
report.is_timeboxed)
@@ -81,7 +82,7 @@
if 'frequency' not in report_config:
raise KeyError('Report frequency is not specified.')
frequency = report_config['frequency']
- if frequency not in ['hours', 'days', 'weeks']:
+ if frequency not in ['hours', 'days', 'weeks', 'months']:
raise ValueError('Report frequency is not valid.')
return frequency
@@ -95,6 +96,15 @@
return granularity
+ def get_lag(self, report_config):
+ if 'lag' not in report_config:
+ return 0
+ lag = report_config['lag']
+ if type(lag) != int or lag < 0:
+ raise ValueError('Report lag is not valid.')
+ return lag
+
+
def get_is_timeboxed(self, report_config):
return 'timeboxed' in report_config and report_config['timeboxed'] is
True
diff --git a/reportupdater/report.py b/reportupdater/report.py
index 3a34228..5f82114 100644
--- a/reportupdater/report.py
+++ b/reportupdater/report.py
@@ -19,6 +19,7 @@
self.type = None
self.frequency = None
self.granularity = None
+ self.lag = 0
self.is_timeboxed = False
self.is_funnel = False
self.first_date = None
@@ -38,6 +39,7 @@
' type=' + str(self.type) +
' frequency=' + str(self.frequency) +
' granularity=' + str(self.granularity) +
+ ' lag=' + str(self.lag) +
' is_timeboxed=' + str(self.is_timeboxed) +
' is_funnel=' + str(self.is_funnel) +
' first_date=' + self.format_date(self.first_date) +
diff --git a/reportupdater/selector.py b/reportupdater/selector.py
index 0fab90c..ce96468 100644
--- a/reportupdater/selector.py
+++ b/reportupdater/selector.py
@@ -75,8 +75,10 @@
first_date = self.truncate_date(report.first_date, report.granularity)
frequency_increment = self.get_increment(report.frequency)
+ lag_increment = relativedelta(seconds=report.lag)
granularity_increment = self.get_increment(report.granularity)
- last_date = self.truncate_date(now - frequency_increment,
report.granularity)
+ relative_now = now - frequency_increment - lag_increment
+ last_date = self.truncate_date(relative_now, report.granularity)
previous_results = get_previous_results(report, output_folder)
already_done_dates = previous_results['data'].keys()
diff --git a/test/reader_test.py b/test/reader_test.py
index b11f626..445e740 100644
--- a/test/reader_test.py
+++ b/test/reader_test.py
@@ -71,7 +71,7 @@
def test_get_frequency_and_granularity(self):
- for frequency in ['hours', 'days', 'weeks']:
+ for frequency in ['hours', 'days', 'weeks', 'months']:
report_config = {'frequency': frequency}
result = self.reader.get_frequency(report_config)
self.assertEqual(result, frequency)
@@ -81,6 +81,27 @@
self.assertEqual(result, granularity)
+ def test_get_lag_when_value_is_not_in_config(self):
+ report_config = {}
+ result = self.reader.get_lag(report_config)
+ self.assertEqual(result, 0)
+
+
+ def test_get_lag_when_value_is_not_valid(self):
+ report_config = {'lag': 'not an int'}
+ with self.assertRaises(ValueError):
+ self.reader.get_lag(report_config)
+ report_config = {'lag': -1}
+ with self.assertRaises(ValueError):
+ self.reader.get_lag(report_config)
+
+
+ def test_get_lag(self):
+ report_config = {'lag': 10}
+ result = self.reader.get_lag(report_config)
+ self.assertEqual(result, 10)
+
+
def test_get_is_timeboxed_when_report_timeboxed_is_not_in_config(self):
report_config = {}
is_timeboxed = self.reader.get_is_timeboxed(report_config)
diff --git a/test/report_test.py b/test/report_test.py
index 52309ed..8f8a068 100644
--- a/test/report_test.py
+++ b/test/report_test.py
@@ -13,6 +13,7 @@
self.report.type = 'sql'
self.report.frequency = 'hours'
self.report.granularity = 'days'
+ self.report.lag = 0
self.report.is_timeboxed = True
self.report.is_funnel = True
self.report.first_date = datetime(2015, 1, 1)
diff --git a/test/selector_test.py b/test/selector_test.py
index 39ce9db..c766fc8 100644
--- a/test/selector_test.py
+++ b/test/selector_test.py
@@ -117,6 +117,17 @@
self.assertEqual(reports[0].end, datetime(2015, 1, 3))
+ def test_get_interval_reports_when_lag_is_set(self):
+ # Note no previous results tsv exists for default report.
+ now = datetime(2015, 1, 3)
+ self.report.frequency = 'days'
+ self.report.lag = 100 # 1 minute and 40 seconds
+ reports = list(self.selector.get_interval_reports(self.report, now))
+ self.assertEqual(len(reports), 1)
+ self.assertEqual(reports[0].start, datetime(2015, 1, 1))
+ self.assertEqual(reports[0].end, datetime(2015, 1, 2))
+
+
def test_truncate_date_when_period_is_hours(self):
date = datetime(2015, 1, 5, 10, 20, 30)
result = self.selector.truncate_date(date, 'hours')
--
To view, visit https://gerrit.wikimedia.org/r/249813
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I9b01c608bd912e80e31cdb372cb9e45441f96f76
Gerrit-PatchSet: 1
Gerrit-Project: analytics/limn-mobile-data
Gerrit-Branch: master
Gerrit-Owner: Mforns <[email protected]>
Gerrit-Reviewer: Milimetric <[email protected]>
Gerrit-Reviewer: Nuria <[email protected]>
Gerrit-Reviewer: Yuvipanda <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
