Milimetric has uploaded a new change for review.
https://gerrit.wikimedia.org/r/72847
Change subject: starting to port the cohort upload
......................................................................
starting to port the cohort upload
Change-Id: Ie18b420d5581139069b8650f61c2af40fbae3979
---
M tests/fixtures.py
M tests/test_controllers/test_cohorts.py
M wikimetrics/controllers/cohorts.py
M wikimetrics/metrics/bytes_added.py
M wikimetrics/models/cohort.py
M wikimetrics/models/mediawiki/user.py
A wikimetrics/static/js/cohortUpload.js
M wikimetrics/templates/csv_upload_form.html
A wikimetrics/utils.py
9 files changed, 376 insertions(+), 55 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/wikimetrics
refs/changes/47/72847/1
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 9232698..68d3d32 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -3,6 +3,7 @@
__all__ = [
'DatabaseTest',
+ 'DatabaseWithCohortTest',
'QueueTest',
'QueueDatabaseTest',
'WebTest',
@@ -98,6 +99,11 @@
cohort_id=private_cohort2.id,
role=CohortUserRole.OWNER,
)
+ web_user_owns_test = CohortUser(
+ user_id=web_test_user.id,
+ cohort_id=test_cohort.id,
+ role=CohortUserRole.OWNER,
+ )
web_user_owns_private = CohortUser(
user_id=web_test_user.id,
cohort_id=private_cohort.id,
@@ -117,6 +123,7 @@
dan_owns_test,
evan_owns_private,
evan_owns_private2,
+ web_user_owns_test,
web_user_owns_private,
web_user_owns_private2,
dan_views_private2
diff --git a/tests/test_controllers/test_cohorts.py
b/tests/test_controllers/test_cohorts.py
index a983f24..8973271 100644
--- a/tests/test_controllers/test_cohorts.py
+++ b/tests/test_controllers/test_cohorts.py
@@ -27,8 +27,20 @@
response = self.app.get('/cohorts/detail/1', follow_redirects=True)
parsed = json.loads(response.data)
assert_equal(
+ response.status_code,
+ 200,
+ )
+ assert_equal(
len(parsed['wikiusers']),
4,
'/cohorts/detail/1 should return JSON object with key `wikiusers`'
'for a list of length 4== `test`, but instead returned:
{0}'.format(parsed)
)
+
+ def test_not_found(self):
+ response =
self.app.get('/cohorts/detail/no_way_anybody_names_a_cohort_this_23982739873')
+
+ assert_equal(
+ response.status_code,
+ 404,
+ )
diff --git a/wikimetrics/controllers/cohorts.py
b/wikimetrics/controllers/cohorts.py
index 5d5a333..21eede3 100644
--- a/wikimetrics/controllers/cohorts.py
+++ b/wikimetrics/controllers/cohorts.py
@@ -1,7 +1,14 @@
-from flask import render_template, redirect, request, jsonify
+import json
+from flask import url_for, flash, render_template, redirect, request, jsonify
from flask.ext.login import current_user
+from sqlalchemy.sql import exists
+from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
+from ..utils import DateTimeCapableEncoder
from ..configurables import app, db
-from ..models import Cohort, CohortUser, CohortUserRole, User, WikiUser,
CohortWikiUser
+from ..models import (
+ Cohort, CohortUser, CohortUserRole,
+ User, WikiUser, CohortWikiUser, MediawikiUser
+)
import logging
logger = logging.getLogger(__name__)
@@ -29,8 +36,8 @@
return jsonify(cohorts=cohorts)
[email protected]('/cohorts/detail/<int:id>')
-def cohort_detail(id):
[email protected]('/cohorts/detail/<string:name_or_id>')
+def cohort_detail(name_or_id):
"""
Returns a JSON object of the form:
{id: 2, name: 'Berlin Beekeeping Society', description: '', wikiusers: [
@@ -40,18 +47,300 @@
{mediawiki_username: 'Gabriele', mediawiki_userid: 8, project:
'dewiki'},
]}
"""
+ d = db.get_session()
+ cohort = None
+ if str(name_or_id).isdigit():
+ cohort = get_cohort_by_id(int(name_or_id))
+ else:
+ cohort = get_cohort_by_name(name_or_id)
+
+ if cohort:
+ cohort_with_wikiusers = populate_cohort_wikiusers(cohort)
+ return json.dumps(cohort_with_wikiusers, cls=DateTimeCapableEncoder)
+
+ return '{}', 404
+
+
+def get_cohort_query():
db_session = db.get_session()
- cohort = db_session.query(Cohort)\
+ return db_session.query(Cohort)\
.join(CohortUser)\
.join(User)\
+ .filter(User.id == current_user.id)\
.filter(CohortUser.role.in_([CohortUserRole.OWNER,
CohortUserRole.VIEWER]))\
- .filter(Cohort.enabled)\
- .filter(Cohort.id == id)\
- .one()
+ .filter(Cohort.enabled)
+
+
+def get_cohort_by_id(id):
+ try:
+ return get_cohort_query().filter(Cohort.id == id).one()
+ # MultipleResultsFound NoResultFound
+ except:
+ return None
+
+
+def get_cohort_by_name(name):
+ try:
+ return get_cohort_query().filter(Cohort.name == name).one()
+ # MultipleResultsFound NoResultFound
+ except:
+ return None
+
+
+def populate_cohort_wikiusers(cohort):
+ db_session = db.get_session()
wikiusers = db_session.query(WikiUser)\
.join(CohortWikiUser)\
.filter(CohortWikiUser.cohort_id == cohort.id)\
.all()
cohort_dict = cohort._asdict()
cohort_dict['wikiusers'] = [wu._asdict() for wu in wikiusers]
- return jsonify(cohort_dict)
+ return cohort_dict
+
+
[email protected]('/cohorts/upload')
+def upload_csv_cohort():
+ """ View for uploading and validating a new cohort via CSV """
+ if request.method == 'GET':
+ return render_template('csv_upload.html')
+
+ elif request.method == 'POST':
+ try:
+ csv = request.files['csv']
+ name = request.form['name']
+ project = request.form['project']
+ if not csv or not name or len(name) is 0:
+ flash('The form was invalid, please select a file and name the
cohort.')
+ return redirect(url_for('upload_csv_cohort'))
+
+ if get_cohort_by_name(name):
+ flash('That Cohort name is already taken.')
+ return redirect(url_for('upload_csv_cohort'))
+
+ unparsed = csv.reader(normalize_newlines(csv.stream))
+ unvalidated = parse_records(unparsed, project)
+ (valid, invalid) = validate_records(unvalidated)
+
+ return render_template(
+ 'csv_upload_review.html',
+ valid=valid,
+ invalid=invalid,
+ valid_json=to_safe_json(valid),
+ invalid_json=to_safe_json(invalid),
+ name=name,
+ project=project,
+ )
+ except Exception, e:
+ logging.exception(str(e))
+ flash(
+ 'The file you uploaded was not in a valid format, could not be
validated,'
+ 'or the project you specified is not configured on this
instance of Wiki Metrics.'
+ )
+ return redirect('/uploads/cohort')
+
+
[email protected]('/cohorts/create', methods=['POST'])
+def upload_csv_cohort_finish():
+ try:
+ name = request.form.get('name')
+ project = request.form.get('project')
+ users_json = request.form.get('users')
+ users = json.loads(users_json)
+ # re-validate
+ if get_cohort_by_name(name):
+ raise Exception('Cohort name {0} is already used'.format(name))
+
+ # TODO: re-enable validation when either
+ # 1. the site is used by external, potentially untrusted users
+ # 2. the performance of validation is improved
+ #(valid, invalid) = validate_records(users)
+ #if invalid:
+ #raise Exception('Cohort changed since last validation')
+ # save the cohort
+ valid = users
+
+ if not project:
+ if all([user['project'] == users[0]['project'] for user in users]):
+ project = users[0]['project']
+ logging.debug('adding cohort: {0}, with project: {1}'.format(name,
project))
+ cohort = create_cohort(name, 'TODO: add description', project, valid)
+ return url_for('cohort_details', name_or_id=cohort.id)
+
+ except Exception, e:
+ logging.exception(str(e))
+ flash('There was a problem finishing the upload. The cohort was not
saved.')
+ return '<<error>>'
+
+
+def create_cohort(name, description, project, valid_users):
+ db_session = db.get_session()
+ cohort = Cohort(
+ name=name,
+ default_project=project,
+ description=description,
+ )
+ db_session.add(cohort)
+ db_session.commit()
+
+ cohort_owner = CohortUser(
+ cohort_id=cohort.id,
+ user_id=current_user.id,
+ )
+ db_session.add(cohort_owner)
+
+ wikiusers = []
+ for valid_user in valid_users:
+ wikiuser = WikiUser(
+ mediawiki_userid=valid_user['userid'],
+ mediawiki_username=valid_user['username'],
+ )
+ wikiusers.append(wikiuser)
+ db_session.add_all(wikiusers)
+ db_session.commit()
+
+ cohort_wikiusers = []
+ for wikiuser in wikiusers:
+ cohort_wikiuser = CohortWikiUser(
+ cohort_id=cohort.id,
+ wiki_user_id=wikiuser.id,
+ )
+ cohort_wikiusers.append(cohort_wikiuser)
+ db_session.add_all(cohort_wikiusers)
+ db_session.commit()
+
+
[email protected]('/cohorts/validate/name')
+def validate_cohort_name_allowed():
+ name = request.args.get('name')
+ return jsonify(get_cohort_by_name(name) is None)
+
+
+def normalize_newlines(stream):
+ for line in stream:
+ if '\r' in line:
+ for tok in line.split('\r'):
+ yield tok
+ else:
+ yield line
+
+
+def to_safe_json(s):
+ return json.dumps(s).replace("'", "\\'").replace('"', '\\"')
+
+
+def parse_records(records, default_project):
+ # NOTE: the reason for the crazy -1 and comma joins
+ # is that some users can have commas in their name
+ # TODO: This makes it impossible to add fields to the csv in the future,
+ # so maybe require the project to be the first field and the username to
be the last
+ # or maybe change to a tsv format
+ return [{
+ 'username': parse_username(",".join([str(p) for p in r[:-1]])),
+ 'project': r[-1].decode('utf8') if len(r) > 1 else default_project
+ } for r in records if r]
+
+
+def parse_username(raw_name):
+ stripped = str(raw_name).decode('utf8').strip()
+ # unfortunately .title() or .capitalize() don't work
+ # because 'miliMetric'.capitalize() == 'Milimetric'
+ return stripped[0].upper() + stripped[1:]
+
+
+def normalize_project(project):
+ project = project.strip().lower()
+ if project in db.project_host_map:
+ return project
+ else:
+ # try adding wiki to end
+ new_proj = project + 'wiki'
+ if new_proj not in db.project_host_map:
+ return None
+ else:
+ return new_proj
+
+
+def get_wikiuser_by_name(username, project):
+ # NOTE: Not needed right? username = username.encode('utf-8')
+ db_session = db.get_mw_session(project)
+ try:
+ return db_session.query(MediaWikiUser)\
+ .filter(MediaWikiUser.user_name == username)\
+ .one()
+ except:
+ return None
+
+
+def get_wikiuser_by_id(id, project):
+ db_session = db.get_mw_session(project)
+ try:
+ return db_session.query(MediaWikiUser)\
+ .filter(MediaWikiUser.user_id == id)\
+ .one()
+ except:
+ return None
+
+
+def normalize_user(user_str, project):
+ wikiuser = get_wikiuser_by_name(user_str, project)
+ if wikiuser is not None:
+ return (wikiuser.user_id, wikiuser.user_name)
+
+ if not user_str.isdigit():
+ return None
+
+ wikiuser = get_wikiuser_by_id(user_str, project)
+ if wikiuser is not None:
+ return (wikiuser.user_id, wikiuser.user_name)
+
+ return None
+
+
+def deduplicate(list_of_objects, key_function):
+ uniques = dict()
+ for o in list_of_objects:
+ key = key_function(o)
+ if not key in uniques:
+ uniques[key] = o
+
+ return uniques.values()
+
+
+def project_name_for_link(project):
+ if project.endswith('wiki'):
+ return project[:len(project) - 4]
+ return project
+
+
+def link_to_user_page(username, project):
+ project = project_name_for_link(project)
+ return 'https://%s.wikipedia.org/wiki/User:%s' % (project, username)
+
+
+def validate_records(records):
+ valid = []
+ invalid = []
+ for record in records:
+ record['user_str'] = record['username']
+ normalized_project = normalize_project(record['project'])
+ if normalized_project is None:
+ record['reason_invalid'] = 'invalid project: %s' %
record['project']
+ invalid.append(record)
+ continue
+ normalized_user = normalize_user(record['user_str'],
normalized_project)
+ # make a link to the potential user page even if user doesn't exist
+ # this gives a chance to see any misspelling etc.
+ record['link'] = link_to_user_page(record['username'],
normalized_project)
+ if normalized_user is None:
+ logging.debug('invalid: %s', record['user_str'])
+ record['reason_invalid'] = 'invalid user_name / user_id: %s' %
record['user_str']
+ invalid.append(record)
+ continue
+ # set the normalized values and append to valid
+ logging.debug('found a valid user_str: %s', record['user_str'])
+ record['project'] = normalized_project
+ record['user_id'], record['username'] = normalized_user
+ valid.append(record)
+
+ valid = deduplicate(valid, lambda record: record['username'])
+ return (valid, invalid)
diff --git a/wikimetrics/metrics/bytes_added.py
b/wikimetrics/metrics/bytes_added.py
index 21cc4fc..736ac1a 100644
--- a/wikimetrics/metrics/bytes_added.py
+++ b/wikimetrics/metrics/bytes_added.py
@@ -57,7 +57,6 @@
description = 'Compute different aggregations of the bytes contributed or
removed from a\
mediawiki project'
-
start_date = DateField()
end_date = DateField()
namespaces = CommaSeparatedIntegerListField(default=[0],
description='0, 2, 4, etc.')
diff --git a/wikimetrics/models/cohort.py b/wikimetrics/models/cohort.py
index c15a90b..37b1842 100644
--- a/wikimetrics/models/cohort.py
+++ b/wikimetrics/models/cohort.py
@@ -1,6 +1,6 @@
import itertools
from operator import itemgetter
-from sqlalchemy import Column, Integer, Boolean, DateTime, String
+from sqlalchemy import Column, Integer, Boolean, DateTime, String, func
from wikimetrics.configurables import db
from .wikiuser import WikiUser
from .cohort_wikiuser import CohortWikiUser
@@ -29,7 +29,7 @@
name = Column(String(50))
description = Column(String(254))
default_project = Column(String(50))
- created = Column(DateTime)
+ created = Column(DateTime, default=func.now())
changed = Column(DateTime)
enabled = Column(Boolean)
public = Column(Boolean, default=False)
diff --git a/wikimetrics/models/mediawiki/user.py
b/wikimetrics/models/mediawiki/user.py
index 54b2eda..2268c8d 100644
--- a/wikimetrics/models/mediawiki/user.py
+++ b/wikimetrics/models/mediawiki/user.py
@@ -12,14 +12,14 @@
user_id = Column(Integer, primary_key=True)
user_name = Column(String(255))
user_real_name = Column(String(255))
- user_password = None # TODO: Password? = Column(String(255))
- user_newpassword = None # TODO: Password? = Column(String(255))
+ # do not map: user_password
+ # do not map: user_newpassword
user_newpass_time = Column(DateTime)
user_email = Column(String(255))
user_touched = Column(DateTime)
- user_token = None # TODO: Token? binary(32) DEFAULT NULL,
+ # do not map: user_token
user_email_authenticated = Column(DateTime)
- user_email_token = None # TODO: Token? binary(32) DEFAULT NULL,
- user_email_token_expires = Column(DateTime)
+ # do not map: user_email_token
+ # do not map: user_email_token_expires = Column(DateTime)
user_registration = Column(DateTime)
user_editcount = Column(Integer)
diff --git a/wikimetrics/static/js/cohortUpload.js
b/wikimetrics/static/js/cohortUpload.js
new file mode 100644
index 0000000..d877a1b
--- /dev/null
+++ b/wikimetrics/static/js/cohortUpload.js
@@ -0,0 +1,24 @@
+$(document).ready(function(){
+
+ jQuery.validator.addMethod('cohortName', function(value, element) {
+ return /^[0-9_\-A-Za-z ]*$/.test(value);
+ }, 'Cohort names should only contain letters, numbers, spaces, dashes, and
underscores');
+
+ $('form.upload-cohort').validate({
+ messages: {
+ name: {
+ remote: 'This cohort name is taken.',
+ }
+ },
+ rules: {
+ name: {
+ required: true,
+ cohortName: true,
+ remote: '/validate/cohort/allowed'
+ },
+ csv: {
+ required: true
+ }
+ }
+ });
+});
diff --git a/wikimetrics/templates/csv_upload_form.html
b/wikimetrics/templates/csv_upload_form.html
index 0c44dfd..3a47773 100644
--- a/wikimetrics/templates/csv_upload_form.html
+++ b/wikimetrics/templates/csv_upload_form.html
@@ -1,55 +1,26 @@
<form enctype="multipart/form-data" action="/uploads/cohort" method="POST"
class="upload-cohort form-horizontal">
<div class="control-group">
- <label for="cohort_name" class="control-label">Cohort Name</label>
+ <label for="name" class="control-label">Cohort Name</label>
<div class="controls">
- <input type="text" name="cohort_name" id="cohort_name" value="{%
if cohort_name %}{{cohort_name}}{% endif %}"/>
- <label for="cohort_name">(automatically checks
availability)</label>
+ <input type="text" name="name" id="name" value="{% if name
%}{{name}}{% endif %}"/>
+ <label for="name">(automatically checks availability)</label>
</div>
</div>
- <div class="control-group"> <label for="cohort_project"
class="control-label">Wiki Project</label>
+ <div class="control-group"> <label for="project"
class="control-label">Wiki Project</label>
+ <label for="project" class="control-label">Default Cohort
Project</label>
<div class="controls">
- <select name="cohort_project" id="cohort_project">
- <option value="">(select one or specify in file)</option>
- {% for project in wiki_projects %}
- <option {% if project == cohort_project %}selected{% endif
%}>{{ project }}</option>
- {% endfor %}
- </select>
+ <input type="text" name="project" id="project" value="{% project
%}{{project}}{% endif %}"/>
</div>
</div>
<div class="control-group">
- <label for="csv_cohort" class="control-label">CSV File</label>
+ <label for="csv" class="control-label">CSV File</label>
<div class="controls">
- <input type="file" name="csv_cohort" id="csv_cohort"/>
+ <input type="file" name="csv" id="csv"/>
</div>
</div>
<div class="form-actions">
<input type="submit" class="btn btn-primary" value="Upload CSV"/>
</div>
</form>
+
<script
src="//ajax.aspnetcdn.com/ajax/jquery.validate/1.11.1/jquery.validate.min.js"></script>
-<script>
- $(document).ready(function(){
-
- jQuery.validator.addMethod('cohortName', function(value, element) {
- return /^[0-9_\-A-Za-z ]*$/.test(value);
- }, 'Cohort names should only contain letters, numbers, spaces, dashes,
and underscores');
-
- $('form.upload-cohort').validate({
- messages: {
- cohort_name: {
- remote: 'This cohort name is taken.',
- }
- },
- rules: {
- cohort_name: {
- required: true,
- cohortName: true,
- remote: '/validate/cohort/allowed'
- },
- csv_cohort: {
- required: true
- }
- }
- });
- });
-</script>
diff --git a/wikimetrics/utils.py b/wikimetrics/utils.py
new file mode 100644
index 0000000..8596d4b
--- /dev/null
+++ b/wikimetrics/utils.py
@@ -0,0 +1,19 @@
+import json
+import datetime
+from time import mktime
+
+class DateTimeCapableEncoder(json.JSONEncoder):
+ """
+ Date/Time objects are not serializable by the built-in
+ json library because there is no agreed upon standard of how to do so
+ This class can be used as follows to allow your json.dumps to serialize
dates properly.
+ You should make sure your client is happy with this serialization:
+ print json.dumps(obj, cls=DateTimeCapableEncoder)
+ """
+
+ def default(self, obj):
+ if isinstance(obj, datetime.datetime):
+ return int(mktime(obj.timetuple()))
+
+ return json.JSONEncoder.default(self, obj)
+
--
To view, visit https://gerrit.wikimedia.org/r/72847
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie18b420d5581139069b8650f61c2af40fbae3979
Gerrit-PatchSet: 1
Gerrit-Project: analytics/wikimetrics
Gerrit-Branch: master
Gerrit-Owner: Milimetric <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
