Milimetric has submitted this change and it was merged.
Change subject: timeseries ordering and labeling fixes
......................................................................
timeseries ordering and labeling fixes
Change-Id: Ifc272b374806e38bc6b0d5e5da1bf10c15530e1f
---
M tests/test_metrics/test_bytes_added.py
M tests/test_metrics/test_namespace_edits.py
M tests/test_metrics/test_timeseries.py
M tests/test_models/test_run_report.py
M wikimetrics/config/celery_config.yaml
M wikimetrics/configurables.py
M wikimetrics/metrics/timeseries_metric.py
M wikimetrics/models/report_nodes/aggregate_report.py
M wikimetrics/run.py
9 files changed, 126 insertions(+), 33 deletions(-)
Approvals:
Milimetric: Verified; Looks good to me, approved
diff --git a/tests/test_metrics/test_bytes_added.py
b/tests/test_metrics/test_bytes_added.py
index 43f80c5..7da10f4 100644
--- a/tests/test_metrics/test_bytes_added.py
+++ b/tests/test_metrics/test_bytes_added.py
@@ -176,7 +176,7 @@
results = metric(list(self.cohort), self.mwSession)
expected1 = {
'net_sum': {
- '2012-12-31 00:00:00' : 0,
+ '2012-12-31 09:00:00' : 0,
'2013-01-01 00:00:00' : 100,
'2013-01-02 00:00:00' : 0,
'2013-01-03 00:00:00' : 0,
diff --git a/tests/test_metrics/test_namespace_edits.py
b/tests/test_metrics/test_namespace_edits.py
index 2f01cb7..9d48d7b 100644
--- a/tests/test_metrics/test_namespace_edits.py
+++ b/tests/test_metrics/test_namespace_edits.py
@@ -206,7 +206,7 @@
def test_timeseries_day(self):
metric = NamespaceEdits(
namespaces=[0],
- start_date='2012-12-31 00:00:00',
+ start_date='2012-12-31 10:00:00',
end_date='2013-01-02 00:00:00',
timeseries=TimeseriesChoices.DAY,
)
@@ -216,7 +216,7 @@
assert_equal(
results[self.editors[0].user_id]['edits'],
{
- '2012-12-31 00:00:00' : 1,
+ '2012-12-31 10:00:00' : 1,
'2013-01-01 00:00:00' : 2,
}
)
diff --git a/tests/test_metrics/test_timeseries.py
b/tests/test_metrics/test_timeseries.py
index be03f4f..8c6925a 100644
--- a/tests/test_metrics/test_timeseries.py
+++ b/tests/test_metrics/test_timeseries.py
@@ -37,7 +37,7 @@
)
assert_equals(t4, '2010-01-02 03:00:00')
- def test_fill_in_missing_datetimes_hour(self):
+ def normalize_datetime_slices(self):
m = TimeseriesMetric(
start_date='2013-01-01 23:00:00',
end_date='2013-01-03 00:00:00',
@@ -52,7 +52,7 @@
}
}
}
- r = m.fill_in_missing_datetimes(results, [('test', 1, 0)])
+ r = m.normalize_datetime_slices(results, [('test', 1, 0)])
assert_equals(r, {
1: {
'test': {
@@ -85,7 +85,7 @@
}
})
- def test_fill_in_missing_datetimes_day(self):
+ def test_normalize_datetime_slices_day(self):
m = TimeseriesMetric(
start_date='2013-01-01 00:00:00',
end_date='2013-01-05 00:00:00',
@@ -106,7 +106,7 @@
}
}
}
- r = m.fill_in_missing_datetimes(results, [('test', 1, 0)])
+ r = m.normalize_datetime_slices(results, [('test', 1, 0)])
assert_equals(r, {
1: {
@@ -127,7 +127,7 @@
}
})
- def test_fill_in_missing_datetimes_month(self):
+ def test_normalize_datetime_slices_month(self):
m = TimeseriesMetric(
start_date='2013-01-02 00:00:00',
end_date='2013-03-05 00:00:00',
@@ -142,22 +142,19 @@
}
},
}
- r = m.fill_in_missing_datetimes(results, [('test', 1, 0)])
-
- from pprint import pprint
- pprint(r)
+ r = m.normalize_datetime_slices(results, [('test', 1, 0)])
assert_equals(r, {
1: {
'test': {
- '2013-01-01 00:00:00': 12,
+ '2013-01-02 00:00:00': 12,
'2013-02-01 00:00:00': 0,
'2013-03-01 00:00:00': 1,
}
},
})
- def test_fill_in_missing_datetimes_year(self):
+ def test_normalize_datetime_slices_year(self):
m = TimeseriesMetric(
start_date='2013-03-10 00:00:00',
end_date='2015-03-05 00:00:00',
@@ -171,16 +168,42 @@
}
},
}
- r = m.fill_in_missing_datetimes(results, [('test', 1, 0)])
- from pprint import pprint
- pprint(r)
+ r = m.normalize_datetime_slices(results, [('test', 1, 0)])
assert_equals(r, {
1: {
'test': {
- '2013-01-01 00:00:00': 12,
+ '2013-03-10 00:00:00': 12,
'2014-01-01 00:00:00': 0,
'2015-01-01 00:00:00': 0,
}
},
})
+
+ def test_normalize_datetime_slices_start_date_forces_first_interval(self):
+ m = TimeseriesMetric(
+ start_date='2013-01-02 00:00:00',
+ end_date='2013-03-05 00:00:00',
+ timeseries=TimeseriesChoices.MONTH,
+ )
+
+ results = {
+ 1: {
+ 'test': {
+ '2013-01-01 00:00:00': 12,
+ '2013-03-01 00:00:00': 1,
+ }
+ },
+ }
+ r = m.normalize_datetime_slices(results, [('test', 1, 0)])
+
+ assert_equals(r, {
+ 1: {
+ 'test': {
+ # The first interval starts on the 2nd and not the 1st
+ '2013-01-02 00:00:00': 12,
+ '2013-02-01 00:00:00': 0,
+ '2013-03-01 00:00:00': 1,
+ }
+ },
+ })
diff --git a/tests/test_models/test_run_report.py
b/tests/test_models/test_run_report.py
index 2409012..f74ff24 100644
--- a/tests/test_models/test_run_report.py
+++ b/tests/test_models/test_run_report.py
@@ -1,9 +1,10 @@
from nose.tools import assert_equals, assert_true, raises
from celery.exceptions import SoftTimeLimitExceeded
+from tests.fixtures import QueueDatabaseTest
from wikimetrics.models import (
RunReport, Aggregation, PersistentReport
)
-from ..fixtures import QueueDatabaseTest
+from wikimetrics.metrics import TimeseriesChoices
class RunReportTest(QueueDatabaseTest):
@@ -82,6 +83,52 @@
5,
)
+ def test_aggregated_response_namespace_edits_with_timeseries(self):
+ desired_responses = [{
+ 'name': 'Edits - test',
+ 'cohort': {
+ 'id': self.test_cohort_id,
+ },
+ 'metric': {
+ 'name': 'NamespaceEdits',
+ 'namespaces': [0, 1, 2],
+ 'start_date': '2013-05-01 10:00:00',
+ 'end_date': '2013-09-01 00:00:00',
+ 'timeseries': TimeseriesChoices.MONTH,
+ 'individualResults': True,
+ 'aggregateResults': True,
+ 'aggregateSum': True,
+ 'aggregateAverage': False,
+ 'aggregateStandardDeviation': False,
+ },
+ }]
+ jr = RunReport(desired_responses, user_id=self.test_user_id)
+ results = jr.task.delay(jr).get()
+ self.session.commit()
+ result_key = self.session.query(PersistentReport)\
+ .filter(PersistentReport.id == jr.children[0].persistent_id)\
+ .one()\
+ .result_key
+ results = results[result_key]
+
+ user_id = self.test_mediawiki_user_id
+ key = results[Aggregation.IND][0][user_id]['edits'].items()[0][0]
+ assert_equals(key, '2013-05-01 10:00:00')
+
+ assert_equals(
+ results[Aggregation.SUM]['edits'].items()[0][0],
+ '2013-05-01 10:00:00',
+ )
+
+ assert_equals(
+ results[Aggregation.SUM]['edits']['2013-05-01 10:00:00'],
+ 0,
+ )
+ assert_equals(
+ results[Aggregation.SUM]['edits']['2013-06-01 00:00:00'],
+ 2,
+ )
+
def test_aggregated_response_bytes_added(self):
desired_responses = [{
'name': 'Edits - test',
diff --git a/wikimetrics/config/celery_config.yaml
b/wikimetrics/config/celery_config.yaml
index 0b28098..df8d682 100644
--- a/wikimetrics/config/celery_config.yaml
+++ b/wikimetrics/config/celery_config.yaml
@@ -7,3 +7,4 @@
CELERYD_TASK_TIME_LIMIT : 60
CELERYD_TASK_SOFT_TIME_LIMIT : 30
DEBUG : True
+LOG_LEVEL : 'WARNING'
diff --git a/wikimetrics/configurables.py b/wikimetrics/configurables.py
index 08aa60a..a48ff0e 100644
--- a/wikimetrics/configurables.py
+++ b/wikimetrics/configurables.py
@@ -96,16 +96,25 @@
def get_absolute_path():
return os.path.dirname(os.path.abspath(__file__)) + '/'
+
def get_wikimetrics_version():
"""
Returns
a tuple of the form (pretty version string, latest commit sha)
"""
path = get_absolute_path()
- orig_wd = os.getcwd() # remember our original working directory
+ orig_wd = os.getcwd() # remember our original working directory
try:
os.chdir(path)
- cmd = ['git', 'log', '--date', 'relative', "--pretty=format:'%an %ar
%h'", '-n', '1']
+ cmd = [
+ 'git',
+ 'log',
+ '--date',
+ 'relative',
+ "--pretty=format:'%an %ar %h'",
+ '-n',
+ '1',
+ ]
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE)
version, err = p.communicate()
if err is not None:
diff --git a/wikimetrics/metrics/timeseries_metric.py
b/wikimetrics/metrics/timeseries_metric.py
index 2834dc7..b83c04d 100644
--- a/wikimetrics/metrics/timeseries_metric.py
+++ b/wikimetrics/metrics/timeseries_metric.py
@@ -1,3 +1,4 @@
+from collections import OrderedDict
from sqlalchemy import func
from datetime import datetime
from dateutil.relativedelta import relativedelta
@@ -130,7 +131,7 @@
}
# in timeseries results, fill in missing date-times
- results = self.fill_in_missing_datetimes(results, submetrics)
+ results = self.normalize_datetime_slices(results, submetrics)
return results
def submetrics_by_user(self, query, submetrics, date_index=None):
@@ -139,17 +140,17 @@
the query_results list.
"""
query_results = query.all()
+ results = OrderedDict()
# handle simple cases (no results or no timeseries)
if not query_results:
- return dict()
+ return results
# get results by user and by date
- results = {}
for row in query_results:
user_id = row[0]
if not user_id in results:
- results[user_id] = {}
+ results[user_id] = OrderedDict()
date_slice = None
if self.timeseries.data != TimeseriesChoices.NONE:
@@ -165,11 +166,12 @@
return results
- def fill_in_missing_datetimes(self, results_by_user, submetrics):
+ def normalize_datetime_slices(self, results_by_user, submetrics):
"""
Starting from a sparse set of timeseries results, fill in default
values
- for the specified list of sub-metrics. If self.timeseries is NONE,
this
- is a simple identity function.
+ for the specified list of sub-metrics. Also make sure the
chronological
+ first timeseries slice is >= self.start_date.
+ If self.timeseries is NONE, this is a simple identity function.
Parameters
results_by_user : dictionary of submetrics dictionaries by user
@@ -182,8 +184,13 @@
return results_by_user
slice_delta = self.get_delta_from_choice()
- timeseries_slices = dict()
- slice_to_default = self.get_first_slice()
+ timeseries_slices = OrderedDict()
+ start_slice_key = format_pretty_date(self.start_date.data)
+ timeseries_slices[start_slice_key] = None
+
+ first_slice = self.get_first_slice()
+ first_slice_key = format_pretty_date(first_slice)
+ slice_to_default = first_slice
while slice_to_default < self.end_date.data:
date_key = format_pretty_date(slice_to_default)
timeseries_slices[date_key] = None
@@ -198,6 +205,10 @@
for k, v in defaults.iteritems():
if not v:
defaults[k] = default
+
+ # coerce the first datetime slice to be self.start_date
+ defaults[start_slice_key] = defaults.pop(first_slice_key)
+
user_submetrics[label] = defaults
return results_by_user
diff --git a/wikimetrics/models/report_nodes/aggregate_report.py
b/wikimetrics/models/report_nodes/aggregate_report.py
index 1d09f8a..1dfa16a 100644
--- a/wikimetrics/models/report_nodes/aggregate_report.py
+++ b/wikimetrics/models/report_nodes/aggregate_report.py
@@ -1,8 +1,10 @@
+from collections import OrderedDict
from decimal import Decimal
+from celery.utils.log import get_task_logger
+
from wikimetrics.utils import stringify
from report import ReportNode
from multi_project_metric_report import MultiProjectMetricReport
-from celery.utils.log import get_task_logger
__all__ = ['AggregateReport', 'Aggregation']
@@ -106,7 +108,7 @@
# handle timeseries aggregation
if isinstance(value, dict):
if not key in aggregation:
- aggregation[key] = dict()
+ aggregation[key] = OrderedDict()
helper[key] = dict()
for subkey in value:
diff --git a/wikimetrics/run.py b/wikimetrics/run.py
index d56e569..bab607f 100644
--- a/wikimetrics/run.py
+++ b/wikimetrics/run.py
@@ -22,7 +22,7 @@
def run_celery():
from configurables import queue
- queue.start(argv=['celery', 'worker', '-l', 'DEBUG'])
+ queue.start(argv=['celery', 'worker', '-l', queue.conf['LOG_LEVEL']])
def setup_parser():
--
To view, visit https://gerrit.wikimedia.org/r/86751
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ifc272b374806e38bc6b0d5e5da1bf10c15530e1f
Gerrit-PatchSet: 1
Gerrit-Project: analytics/wikimetrics
Gerrit-Branch: master
Gerrit-Owner: Milimetric <[email protected]>
Gerrit-Reviewer: Milimetric <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
