Joal has submitted this change and it was merged.
Change subject: Add aggregation across projects
......................................................................
Add aggregation across projects
With this change, if the flag --all-projects is used,
a new file 'all.csv' will be created for the raw_daily
data as well as for each additional aggregator
(daily, weekly_rescaled, etc.). This file will contain
an aggregation across all projects.
Bug: T95339
Change-Id: Ieee012e60d2286e450779cabf253d6a1f11578b4
---
M aggregator/projectcounts.py
M aggregator/util.py
M bin/aggregate_projectcounts
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-010000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-020000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-030000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-040000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-050000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-060000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-070000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-080000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-090000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-100000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-110000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-120000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-130000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-140000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-150000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-160000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-170000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-180000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-190000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-200000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-210000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-220000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-230000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-000000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-010000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-020000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-030000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-040000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-050000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-060000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-070000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-080000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-090000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-100000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-110000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-120000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-130000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-140000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-150000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-160000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-170000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-180000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-190000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-200000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-210000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-220000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-230000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-000000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-010000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-020000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-030000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-040000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-050000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-060000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-070000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-080000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-090000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-100000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-110000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-120000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-130000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-140000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-150000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-160000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-170000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-180000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-190000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-200000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-210000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-220000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-230000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141104-000000
M tests/test_projectcounts/test_per_project_aggregation.py
M tests/test_util.py
77 files changed, 377 insertions(+), 17 deletions(-)
Approvals:
Joal: Verified; Looks good to me, approved
diff --git a/aggregator/projectcounts.py b/aggregator/projectcounts.py
index 75c7f5e..e261f6e 100644
--- a/aggregator/projectcounts.py
+++ b/aggregator/projectcounts.py
@@ -465,7 +465,8 @@
def update_per_project_csvs_for_dates(
source_dir_abs, target_dir_abs, first_date, last_date,
- bad_dates=[], additional_aggregators=[], force_recomputation=False):
+ bad_dates=[], additional_aggregators=[], force_recomputation=False,
+ compute_all_projects=False):
"""Updates per project CSVs from hourly projectcounts files.
The existing per project CSV files in the daily_raw subdirectory of
@@ -496,13 +497,23 @@
through. (Default: [])
:param force_recomputation: If True, recompute data for the given days,
even if it is already in the CSV. (Default: False)
+ :param compute_all_projects: If True, compute counts for all projects
+ into a file named 'all.csv'.
"""
+ # Contains the aggregation of all data across projects indexed by date.
+ all_projects_data = {}
+
for csv_file_abs in sorted(glob.glob(os.path.join(
target_dir_abs, 'daily_raw', '*.csv'))):
- logging.info("Updating csv '%s'" % (csv_file_abs))
-
dbname = os.path.basename(csv_file_abs)
dbname = dbname.rsplit('.csv', 1)[0]
+
+ if dbname == 'all':
+ # 'all.csv' is an aggregation across all projects
+ # and should not be processed.
+ continue
+
+ logging.info("Updating csv '%s'" % (csv_file_abs))
csv_data = util.parse_csv_to_first_column_dict(csv_file_abs)
@@ -545,20 +556,71 @@
count_mobile if date >= DATE_MOBILE_ADDED else None,
count_zero if date >= DATE_MOBILE_ADDED else None)
- util.write_dict_values_sorted_to_csv(
- csv_file_abs,
+ _write_raw_and_aggregated_csv_data(
+ target_dir_abs,
+ dbname,
csv_data,
- header=CSV_HEADER)
+ first_date,
+ last_date,
+ additional_aggregators,
+ bad_dates,
+ force_recomputation)
- for additional_aggregator in additional_aggregators:
- additional_aggregator(
- target_dir_abs,
- dbname,
- csv_data,
- first_date,
- last_date,
- bad_dates=bad_dates,
- force_recomputation=force_recomputation)
+ # Aggregates values across all projects
+ if compute_all_projects:
+ util.merge_sum_csv_data_dict(all_projects_data, csv_data)
+
+ # Writes aggregations across all projects
+ if compute_all_projects:
+ _write_raw_and_aggregated_csv_data(
+ target_dir_abs,
+ 'all',
+ all_projects_data,
+ first_date,
+ last_date,
+ additional_aggregators,
+ bad_dates,
+ force_recomputation)
+
+
+def _write_raw_and_aggregated_csv_data(
+ target_dir_abs, dbname, csv_data, first_date, last_date,
+ additional_aggregators, bad_dates, force_recomputation):
+ """
+ Writes the data passed in the csv_data dict to various destinations:
+
+ 1. Writes the raw data (as it comes in the csv_data dict), to:
+ <target_dir_abs>/daily_raw/<dbname>.csv
+
+ 2. Uses each aggregator in additional_aggregators to write the data
+ to the aggregator's specific location. Note: Some of this method's
+ parameters are just forwarded to these aggregators.
+
+ :param target_dir_abs: Absolute directory of the per project CSVs.
+ :param dbname: The database name of the wiki to consider (E.g.: 'enwiki')
+ :param csv_data: The dict containing the data to be written.
+ :param first_date: The first date to write non-existing data for.
+ :param last_date: The last date to write non-existing data for.
+ :param additional_aggregators: See update_per_project_csvs_for_dates.
+ :param bad_dates: List of dates considered having bad data.
+ :param force_recomputation: If True, recompute data for the given days.
+ """
+ csv_file_abs = os.path.join(target_dir_abs, 'daily_raw', dbname + '.csv')
+
+ util.write_dict_values_sorted_to_csv(
+ csv_file_abs,
+ csv_data,
+ header=CSV_HEADER)
+
+ for additional_aggregator in additional_aggregators:
+ additional_aggregator(
+ target_dir_abs,
+ dbname,
+ csv_data,
+ first_date,
+ last_date,
+ bad_dates=bad_dates,
+ force_recomputation=force_recomputation)
def _get_validity_issues_for_aggregated_projectcounts_generic(
diff --git a/aggregator/util.py b/aggregator/util.py
index 9527b6d..947a65f 100644
--- a/aggregator/util.py
+++ b/aggregator/util.py
@@ -21,6 +21,7 @@
import datetime
import os
+from operator import add
WEBSTATSCOLLECTOR_WHITELISTED_WIKIMEDIA_WIKIS = [
'commons',
@@ -205,3 +206,35 @@
line += ',%s' % ('' if column is None else column)
csv_data[first_column] = line
return csv_data
+
+
+def merge_sum_csv_data_dict(csv_data_1, csv_data_2):
+ """
+ Merge csv_data_2 into csv_data_1, accumulating values by date.
+
+ The two dicts have to be of the form dict<key: str, value: str>
+ and their value format should be 'date,int,int,int,int'.
+ If any of those is incorrect, a ValueError is raised.
+
+ The merged values will be the cross sum of all integer operators,
+ with the exception of the first integer column, that will hold
+ the sum of the other integer columns:
+ 'date,int1a,int1b,int1c,int1d' + 'date,int2a,int2b,int2c,int2d' =
+ 'date,<sumOfTheOtherInts>,int1b+int2b,int1c+int2c,int1d+int2d'
+ Note that the values are aggregated by date.
+
+ :param csv_data_1: The data dict to be merged on.
+ :param csv_data_2: The data dict to be added from.
+ """
+ parse = lambda vs: [int(v.strip()) for v in vs.split(',')[2:5]]
+ build = lambda d, vs: ','.join([d, str(sum(vs))] + [str(v) for v in vs])
+
+ for date in csv_data_2:
+
+ value_1 = parse(csv_data_1[date]) if date in csv_data_1 else [0, 0, 0]
+ value_2 = parse(csv_data_2[date])
+ if len(value_1) != 3 or len(value_2) != 3:
+ raise ValueError('Cannot parse CSV data dict value.')
+
+ summed_value = map(add, value_1, value_2)
+ csv_data_1[date] = build(date, summed_value)
diff --git a/bin/aggregate_projectcounts b/bin/aggregate_projectcounts
index b34eb96..a8d0efd 100755
--- a/bin/aggregate_projectcounts
+++ b/bin/aggregate_projectcounts
@@ -18,7 +18,8 @@
Usage: aggregate_projectcounts [--source SOURCE_DIR] [--target TARGET_DIR]
[--first-date FIRST_DATE] [--last-date LAST_DATE] [--date DATE]
- [--log LOG_FILE] [--force] [--push-target] [-v ...] [--help]
+ [--log LOG_FILE] [--force] [--all-projects] [--push-target]
+ [-v ...] [--help]
Options:
-h, --help Show this help message and exit.
@@ -45,6 +46,8 @@
--last-date)
--force Force recomputation of given days, even if the CSV
would already contain that data.
+ --all-projects Compute aggregation across projects into a file
+ named 'all.csv'.
--push-target Assumes the target directory is a git repository,
and automatically hard reset it before the
aggregation, and commit and push after the
@@ -180,6 +183,7 @@
(first_date, last_date))
force_recomputation = arguments['--force']
+ compute_all_projects = arguments['--all-projects']
if not all_parameters_ok:
logging.error("Parameters could not get parsed")
@@ -211,7 +215,8 @@
last_date,
bad_dates=bad_dates,
additional_aggregators=additional_aggregators,
- force_recomputation=force_recomputation
+ force_recomputation=force_recomputation,
+ compute_all_projects=compute_all_projects
)
if arguments["--push-target"]:
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-010000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-010000
new file mode 100644
index 0000000..20a5299
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-010000
@@ -0,0 +1,3 @@
+en - 3 100
+de - 2 100
+fr - 5 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-020000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-020000
new file mode 100644
index 0000000..db54213
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-020000
@@ -0,0 +1,2 @@
+en - 8 100
+fr - 9 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-030000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-030000
new file mode 100644
index 0000000..2ba29f4
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-030000
@@ -0,0 +1 @@
+de - 9 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-040000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-040000
new file mode 100644
index 0000000..f7e2efc
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-040000
@@ -0,0 +1,2 @@
+de - 7 100
+fr - 4 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-050000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-050000
new file mode 100644
index 0000000..f0f4ceb
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-050000
@@ -0,0 +1,3 @@
+en - 10 100
+de - 3 100
+fr - 4 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-060000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-060000
new file mode 100644
index 0000000..550d23f
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-060000
@@ -0,0 +1,2 @@
+de - 5 100
+fr - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-070000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-070000
new file mode 100644
index 0000000..8cd13be
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-070000
@@ -0,0 +1,3 @@
+en - 3 100
+de - 10 100
+fr - 10 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-080000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-080000
new file mode 100644
index 0000000..143f271
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-080000
@@ -0,0 +1,3 @@
+en - 6 100
+de - 5 100
+fr - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-090000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-090000
new file mode 100644
index 0000000..4bdbca3
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-090000
@@ -0,0 +1,3 @@
+en - 2 100
+de - 4 100
+fr - 5 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-100000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-100000
new file mode 100644
index 0000000..d7441c7
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-100000
@@ -0,0 +1,2 @@
+en - 9 100
+fr - 4 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-110000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-110000
new file mode 100644
index 0000000..28320c1
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-110000
@@ -0,0 +1,3 @@
+en - 8 100
+de - 7 100
+fr - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-120000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-120000
new file mode 100644
index 0000000..a411fc0
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-120000
@@ -0,0 +1,2 @@
+en - 6 100
+de - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-130000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-130000
new file mode 100644
index 0000000..79d9344
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-130000
@@ -0,0 +1,3 @@
+en - 1 100
+de - 10 100
+fr - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-140000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-140000
new file mode 100644
index 0000000..0cc611c
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-140000
@@ -0,0 +1,3 @@
+en - 7 100
+de - 6 100
+fr - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-150000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-150000
new file mode 100644
index 0000000..5c990f9
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-150000
@@ -0,0 +1 @@
+de - 10 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-160000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-160000
new file mode 100644
index 0000000..5cbbd36
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-160000
@@ -0,0 +1,2 @@
+de - 7 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-170000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-170000
new file mode 100644
index 0000000..ab12bb5
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-170000
@@ -0,0 +1,2 @@
+en - 10 100
+fr - 5 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-180000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-180000
new file mode 100644
index 0000000..fa98519
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-180000
@@ -0,0 +1,3 @@
+en - 2 100
+de - 4 100
+fr - 2 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-190000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-190000
new file mode 100644
index 0000000..5d9eeee
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-190000
@@ -0,0 +1,2 @@
+de - 4 100
+fr - 3 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-200000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-200000
new file mode 100644
index 0000000..f61ad79
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-200000
@@ -0,0 +1,3 @@
+en - 9 100
+de - 5 100
+fr - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-210000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-210000
new file mode 100644
index 0000000..42cd30b
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-210000
@@ -0,0 +1,3 @@
+en - 4 100
+de - 9 100
+fr - 4 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-220000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-220000
new file mode 100644
index 0000000..f599cc7
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-220000
@@ -0,0 +1,3 @@
+en - 7 100
+de - 1 100
+fr - 4 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-230000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-230000
new file mode 100644
index 0000000..07ec149
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-230000
@@ -0,0 +1,2 @@
+en - 8 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-000000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-000000
new file mode 100644
index 0000000..ee33a23
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-000000
@@ -0,0 +1,2 @@
+de - 6 100
+fr - 10 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-010000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-010000
new file mode 100644
index 0000000..b6aee19
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-010000
@@ -0,0 +1,2 @@
+de - 9 100
+fr - 2 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-020000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-020000
new file mode 100644
index 0000000..342f8bb
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-020000
@@ -0,0 +1,2 @@
+en - 3 100
+fr - 10 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-030000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-030000
new file mode 100644
index 0000000..b5d3008
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-030000
@@ -0,0 +1,3 @@
+en - 2 100
+de - 7 100
+fr - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-040000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-040000
new file mode 100644
index 0000000..9b19dea
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-040000
@@ -0,0 +1,3 @@
+en - 5 100
+de - 10 100
+fr - 3 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-050000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-050000
new file mode 100644
index 0000000..1126eaa
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-050000
@@ -0,0 +1,3 @@
+en - 5 100
+de - 5 100
+fr - 3 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-060000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-060000
new file mode 100644
index 0000000..4b01cb0
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-060000
@@ -0,0 +1,2 @@
+de - 1 100
+fr - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-070000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-070000
new file mode 100644
index 0000000..c49e31f
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-070000
@@ -0,0 +1,3 @@
+en - 10 100
+de - 10 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-080000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-080000
new file mode 100644
index 0000000..fe39ef4
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-080000
@@ -0,0 +1,3 @@
+en - 10 100
+de - 6 100
+fr - 5 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-090000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-090000
new file mode 100644
index 0000000..4032e72
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-090000
@@ -0,0 +1,2 @@
+en - 2 100
+fr - 10 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-100000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-100000
new file mode 100644
index 0000000..f1b8aaf
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-100000
@@ -0,0 +1,3 @@
+en - 10 100
+de - 2 100
+fr - 3 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-110000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-110000
new file mode 100644
index 0000000..5d9eeee
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-110000
@@ -0,0 +1,2 @@
+de - 4 100
+fr - 3 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-120000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-120000
new file mode 100644
index 0000000..a3122e3
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-120000
@@ -0,0 +1,3 @@
+en - 9 100
+de - 8 100
+fr - 9 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-130000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-130000
new file mode 100644
index 0000000..a9d5d52
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-130000
@@ -0,0 +1,3 @@
+en - 7 100
+de - 10 100
+fr - 2 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-140000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-140000
new file mode 100644
index 0000000..5ae14a5
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-140000
@@ -0,0 +1 @@
+de - 2 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-150000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-150000
new file mode 100644
index 0000000..d9f9e99
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-150000
@@ -0,0 +1,2 @@
+en - 1 100
+fr - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-160000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-160000
new file mode 100644
index 0000000..5d280f6
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-160000
@@ -0,0 +1,3 @@
+en - 8 100
+de - 4 100
+fr - 8 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-170000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-170000
new file mode 100644
index 0000000..f3b4231
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-170000
@@ -0,0 +1 @@
+en - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-180000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-180000
new file mode 100644
index 0000000..3015d99
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-180000
@@ -0,0 +1,3 @@
+en - 8 100
+de - 1 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-190000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-190000
new file mode 100644
index 0000000..3481f50
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-190000
@@ -0,0 +1,2 @@
+de - 4 100
+fr - 9 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-200000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-200000
new file mode 100644
index 0000000..5b84afe
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-200000
@@ -0,0 +1,2 @@
+en - 2 100
+de - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-210000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-210000
new file mode 100644
index 0000000..fb80dc3
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-210000
@@ -0,0 +1,2 @@
+en - 6 100
+fr - 10 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-220000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-220000
new file mode 100644
index 0000000..fca1e7a
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-220000
@@ -0,0 +1,3 @@
+en - 6 100
+de - 2 100
+fr - 8 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-230000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-230000
new file mode 100644
index 0000000..07dfda7
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-230000
@@ -0,0 +1,2 @@
+en - 3 100
+de - 2 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-000000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-000000
new file mode 100644
index 0000000..b9a25ff
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-000000
@@ -0,0 +1,3 @@
+en - 5 100
+de - 6 100
+fr - 2 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-010000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-010000
new file mode 100644
index 0000000..7a7cc98
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-010000
@@ -0,0 +1,3 @@
+en - 4 100
+de - 7 100
+fr - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-020000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-020000
new file mode 100644
index 0000000..e2bfebb
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-020000
@@ -0,0 +1,2 @@
+en - 4 100
+fr - 3 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-030000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-030000
new file mode 100644
index 0000000..769a819
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-030000
@@ -0,0 +1,2 @@
+en - 6 100
+de - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-040000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-040000
new file mode 100644
index 0000000..6f8555c
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-040000
@@ -0,0 +1,2 @@
+en - 4 100
+de - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-050000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-050000
new file mode 100644
index 0000000..7882d2a
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-050000
@@ -0,0 +1,3 @@
+en - 10 100
+de - 9 100
+fr - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-060000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-060000
new file mode 100644
index 0000000..d606e2f
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-060000
@@ -0,0 +1 @@
+de - 3 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-070000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-070000
new file mode 100644
index 0000000..c74e5be
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-070000
@@ -0,0 +1,2 @@
+en - 9 100
+de - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-080000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-080000
new file mode 100644
index 0000000..2466e50
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-080000
@@ -0,0 +1,3 @@
+en - 8 100
+de - 4 100
+fr - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-090000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-090000
new file mode 100644
index 0000000..e37c329
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-090000
@@ -0,0 +1,3 @@
+en - 3 100
+de - 10 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-100000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-100000
new file mode 100644
index 0000000..8d2fa32
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-100000
@@ -0,0 +1,3 @@
+en - 6 100
+de - 1 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-110000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-110000
new file mode 100644
index 0000000..a411fc0
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-110000
@@ -0,0 +1,2 @@
+en - 6 100
+de - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-120000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-120000
new file mode 100644
index 0000000..97476cb
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-120000
@@ -0,0 +1,3 @@
+en - 10 100
+de - 4 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-130000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-130000
new file mode 100644
index 0000000..f193e51
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-130000
@@ -0,0 +1,3 @@
+en - 3 100
+de - 3 100
+fr - 9 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-140000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-140000
new file mode 100644
index 0000000..eaef622
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-140000
@@ -0,0 +1,3 @@
+en - 5 100
+de - 3 100
+fr - 5 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-150000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-150000
new file mode 100644
index 0000000..6ead5be
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-150000
@@ -0,0 +1,3 @@
+en - 6 100
+de - 5 100
+fr - 9 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-160000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-160000
new file mode 100644
index 0000000..88e3715
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-160000
@@ -0,0 +1,2 @@
+en - 1 100
+de - 5 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-170000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-170000
new file mode 100644
index 0000000..25b81aa
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-170000
@@ -0,0 +1,2 @@
+en - 1 100
+de - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-180000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-180000
new file mode 100644
index 0000000..fd852b1
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-180000
@@ -0,0 +1,3 @@
+en - 6 100
+de - 7 100
+fr - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-190000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-190000
new file mode 100644
index 0000000..47a0c2d
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-190000
@@ -0,0 +1 @@
+en - 2 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-200000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-200000
new file mode 100644
index 0000000..57c813d
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-200000
@@ -0,0 +1,2 @@
+de - 5 100
+fr - 4 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-210000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-210000
new file mode 100644
index 0000000..a5e80dd
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-210000
@@ -0,0 +1,3 @@
+en - 2 100
+de - 2 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-220000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-220000
new file mode 100644
index 0000000..bbb3bab
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-220000
@@ -0,0 +1,3 @@
+en - 5 100
+de - 8 100
+fr - 9 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-230000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-230000
new file mode 100644
index 0000000..5f1c8ef
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-230000
@@ -0,0 +1,3 @@
+en - 8 100
+de - 5 100
+fr - 8 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141104-000000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141104-000000
new file mode 100644
index 0000000..4792cdf
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141104-000000
@@ -0,0 +1,2 @@
+de - 2 100
+fr - 7 100
diff --git a/tests/test_projectcounts/test_per_project_aggregation.py
b/tests/test_projectcounts/test_per_project_aggregation.py
index c75f400..b5f66c3 100644
--- a/tests/test_projectcounts/test_per_project_aggregation.py
+++ b/tests/test_projectcounts/test_per_project_aggregation.py
@@ -338,3 +338,31 @@
bad_dates,
True
]])
+
+ def test_update_per_project_compute_all_projects(self):
+ fixture = self.get_fixture_dir_abs(
+ '2014-11-3projects-for-aggregation')
+
+ first_date = datetime.date(2014, 11, 1)
+ last_date = datetime.date(2014, 11, 3)
+
+ enwiki_file_abs = os.path.join(self.daily_raw_dir_abs, 'enwiki.csv')
+ dewiki_file_abs = os.path.join(self.daily_raw_dir_abs, 'dewiki.csv')
+ frwiki_file_abs = os.path.join(self.daily_raw_dir_abs, 'frwiki.csv')
+ self.create_empty_file(enwiki_file_abs)
+ self.create_empty_file(dewiki_file_abs)
+ self.create_empty_file(frwiki_file_abs)
+
+ aggregator.update_per_project_csvs_for_dates(
+ fixture,
+ self.data_dir_abs,
+ first_date,
+ last_date,
+ compute_all_projects=True)
+
+ all_file_abs = os.path.join(self.daily_raw_dir_abs, 'all.csv')
+ self.assert_file_content_equals(all_file_abs, [
+ '2014-11-01,323,323,0,0',
+ '2014-11-02,321,321,0,0',
+ '2014-11-03,310,310,0,0',
+ ])
diff --git a/tests/test_util.py b/tests/test_util.py
index 75e4011..34ba5b9 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -408,3 +408,60 @@
"2014-05-12,1,2",
"2014-05-13,3,4",
])
+
+ def test_merge_sum_csv_data_dict_with_empty_add_dict(self):
+ dict_1 = {
+ '2014-01-01': '2014-01-01,6,3,2,1',
+ '2014-01-02': '2014-01-02,9,4,3,2',
+ }
+ dict_2 = {}
+ aggregator.merge_sum_csv_data_dict(dict_1, dict_2)
+ self.assertEqual(dict_1, dict_1)
+
+ def test_merge_sum_csv_data_dict_with_empty_base_dict(self):
+ dict_1 = {}
+ dict_2 = {
+ '2014-01-01': '2014-01-01,6,3,2,1',
+ '2014-01-02': '2014-01-02,9,4,3,2',
+ }
+ aggregator.merge_sum_csv_data_dict(dict_1, dict_2)
+ self.assertEqual(dict_1, dict_2)
+
+ def test_merge_sum_csv_data_dict_with_update(self):
+ dict_1 = {
+ '2014-01-01': '2014-01-01,6,3,2,1',
+ '2014-01-03': '2014-01-03,12,5,4,3',
+ }
+ dict_2 = {
+ '2014-01-01': '2014-01-01,6,2,3,1',
+ '2014-01-02': '2014-01-02,6,4,2,3',
+ }
+ aggregator.merge_sum_csv_data_dict(dict_1, dict_2)
+ expected = {
+ '2014-01-01': '2014-01-01,12,5,5,2',
+ '2014-01-02': '2014-01-02,9,4,2,3',
+ '2014-01-03': '2014-01-03,12,5,4,3',
+ }
+ self.assertEqual(dict_1, expected)
+
+ def test_merge_sum_csv_data_dict_with_wrong_value(self):
+ dict_1 = {}
+ dict_2 = {
+ '2014-01-01': 'wrong value',
+ }
+ nose.tools.assert_raises(
+ ValueError,
+ aggregator.merge_sum_csv_data_dict,
+ dict_1,
+ dict_2)
+
+ def test_merge_sum_csv_data_dict_with_non_integers(self):
+ dict_1 = {}
+ dict_2 = {
+ '2014-01-01': '2014-01-01,not,an,integer,value',
+ }
+ nose.tools.assert_raises(
+ ValueError,
+ aggregator.merge_sum_csv_data_dict,
+ dict_1,
+ dict_2)
--
To view, visit https://gerrit.wikimedia.org/r/223031
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ieee012e60d2286e450779cabf253d6a1f11578b4
Gerrit-PatchSet: 2
Gerrit-Project: analytics/aggregator
Gerrit-Branch: master
Gerrit-Owner: Mforns <[email protected]>
Gerrit-Reviewer: Joal <[email protected]>
Gerrit-Reviewer: Mforns <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
