QChris has uploaded a new change for review.
https://gerrit.wikimedia.org/r/172198
Change subject: Add switch for automatic pushing of data repo for projectcounts
aggregation
......................................................................
Add switch for automatic pushing of data repo for projectcounts aggregation
Bug: 72740
Change-Id: I3fa7b025f1d5bad3e3fca53138c22aecb720f024
---
M bin/aggregate_projectcounts
1 file changed, 30 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/aggregator
refs/changes/98/172198/1
diff --git a/bin/aggregate_projectcounts b/bin/aggregate_projectcounts
index ae2bd3f..045b006 100755
--- a/bin/aggregate_projectcounts
+++ b/bin/aggregate_projectcounts
@@ -18,7 +18,7 @@
Usage: aggregate_projectcounts [--source SOURCE_DIR] [--target TARGET_DIR]
[--first-date FIRST_DATE] [--last-date LAST_DATE] [--date DATE]
- [--log LOG_FILE] [-v ...] [--help]
+ [--log LOG_FILE] [--push-target] [-v ...] [--help]
Options:
-h, --help Show this help message and exit.
@@ -34,6 +34,10 @@
[default: yesterday]
--date DATE Day to aggregate for (overrides --first-date, and
--last-date)
+ --push-target Assumes the target directory is a git repository,
+ and automatically hard reset it before the
+ aggregation, and commit and push after the
+ aggregation.
--log LOG_FILE In addition to stdout, also log to LOG_FILE
-v, --verbose Increase verbosity
@@ -48,11 +52,14 @@
from docopt import docopt
import logging
+import subprocess
import aggregator
LOG_MESSAGE_FORMAT = '%(asctime)s %(levelname)-6s %(message)s'
LOG_DATE_FORMAT = '%Y-%m-%dT%H:%M:%S'
+
+GIT_FILE_ABS = '/usr/bin/git'
def setup_logging(verbosity, log_file):
@@ -87,6 +94,15 @@
logging.basicConfig(level=log_level,
format=LOG_MESSAGE_FORMAT,
datefmt=LOG_DATE_FORMAT)
+
+
+def run_git(args):
+ # To avoid depending on not-per-default-included git module, we run git by
+ # hand.
+ git_command = [GIT_FILE_ABS]
+ git_command.extend(args)
+ logging.info("Spawning git with : %s", git_command)
+ subprocess.check_call(git_command)
if __name__ == '__main__':
@@ -142,9 +158,22 @@
logging.error("Parameters could not get parsed")
sys.exit(1)
+ if arguments["--push-target"]:
+ os.chdir(target_dir_abs)
+ run_git(['reset', '--quiet', '--hard'])
+ run_git(['checkout', '--quiet', 'master'])
+ run_git(['pull', '--quiet'])
+ run_git(['reset', '--quiet', '--hard', 'origin/master'])
+
aggregator.update_daily_per_project_csvs(
source_dir_abs,
target_dir_abs,
first_date,
last_date
)
+
+ if arguments["--push-target"]:
+ commit_message = "Automatic commit for dates %s until %s" % (
+ first_date.isoformat(), last_date.isoformat())
+ run_git(['commit', '--quiet', '*.csv', '-m', commit_message])
+ run_git(['push', '--quiet', 'origin', 'HEAD:refs/heads/master'])
--
To view, visit https://gerrit.wikimedia.org/r/172198
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I3fa7b025f1d5bad3e3fca53138c22aecb720f024
Gerrit-PatchSet: 1
Gerrit-Project: analytics/aggregator
Gerrit-Branch: master
Gerrit-Owner: QChris <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
