Milimetric has submitted this change and it was merged.

Change subject: [ready for review] A step towards 818
......................................................................


[ready for review] A step towards 818

Card: analytics 1242

Added changes to the models
Working on ValidateCohort
Added 3 tests for a small cohort

Change-Id: I848a11ec952c0ca865ee789366c31c7700f678e8
---
A .travis.yml
A scripts/console_celery.py
M scripts/debug.py
M scripts/test
M tests/fixtures.py
M tests/test_controllers/test_cohorts.py
A tests/test_models/test_cohort.py
M tests/test_models/test_run_report.py
A tests/test_queue/test_async_cohort_validation.py
M tests/test_queue/test_queue_setup_teardown.py
M wikimetrics/config/celery_config.yaml
M wikimetrics/config/db_config.yaml
M wikimetrics/controllers/cohorts.py
M wikimetrics/models/__init__.py
M wikimetrics/models/cohort.py
A wikimetrics/models/cohort_upload_node.py
M wikimetrics/models/wikiuser.py
M wikimetrics/static/js/reportCreate.js
18 files changed, 509 insertions(+), 24 deletions(-)

Approvals:
  Milimetric: Verified; Looks good to me, approved



diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..8104276
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,26 @@
+language: python
+python:
+  - "2.7"
+  #- "3.2"
+  #- "3.3"
+
+before_install:
+   - sudo apt-get update  -qq
+   - sudo apt-get install -qq mysql-server mysql-client redis-server mlocate
+   - echo "create database wikimetrics;" | mysql -uroot
+   - echo "create database enwiki;" | mysql -uroot
+   - cat seed.sql | mysql -uroot
+   - echo "CREATE USER 'wikimetrics'@'localhost' IDENTIFIED BY 'wikimetrics';" 
| mysql -uroot
+   - echo "GRANT ALL PRIVILEGES ON wikimetrics.* TO 
'wikimetrics'@'localhost';" | mysql -uroot
+   - echo "GRANT ALL PRIVILEGES ON enwiki.*      TO 
'wikimetrics'@'localhost';" | mysql -uroot
+   - pip install -r requirements.txt
+   - pip install -e .
+   #- git config --global user.email "[email protected]"
+   #- git config --global user.name "Mr Test"
+
+script: "nosetests -v .;"
+
+after_install: echo "finished install"
+
+notifications:
+    irc: "chat.freenode.net#wikimedia-analytics"
diff --git a/scripts/console_celery.py b/scripts/console_celery.py
new file mode 100755
index 0000000..c127425
--- /dev/null
+++ b/scripts/console_celery.py
@@ -0,0 +1,20 @@
+#!/usr/bin/python
+import celery
+from wikimetrics.models import add
+
+c = {
+    "BROKER_URL": 'redis://localhost:6379/0',
+    "CELERY_RESULT_BACKEND": 'redis://localhost:6379/0',
+    "CELERY_TASK_RESULT_EXPIRES": 3600,
+    "DEBUG": True,
+    "LOG_LEVEL": 'WARNING'
+}
+
+q = celery.Celery()
+q.config_from_object(c)
+r = add.delay(3, 4)
+
+while not r.ready():
+    pass
+
+print r.result
diff --git a/scripts/debug.py b/scripts/debug.py
index abe44a4..020412b 100755
--- a/scripts/debug.py
+++ b/scripts/debug.py
@@ -8,11 +8,11 @@
 #####################################
 
 from sqlalchemy import func
-from wikimetrics.models import Revision, Page, User, MediawikiUser
+from wikimetrics.models import Revision, Page, Cohort, CohortUser, User, 
MediawikiUser
 #from wikimetrics.models.mediawiki import
 from tests.fixtures import DatabaseTest, DatabaseWithSurvivorCohortTest
 from tests.test_metrics.test_survivors import *
-from wikimetrics.metrics import Survivors
+#from wikimetrics.metrics import Survivors
 from wikimetrics.configurables import db
 import calendar
 
@@ -23,7 +23,7 @@
 m = db.get_session()
 
 s = SurvivalTest()
-s.setUp()
+#s.setUp()
 
 # %load_ext autoreload
 # %autoreload 2
diff --git a/scripts/test b/scripts/test
index fc20168..a2015a0 100755
--- a/scripts/test
+++ b/scripts/test
@@ -1,4 +1,4 @@
 # for example:
 # scripts/test "tests/test_controllers/test_cohorts.py:CohortsControllerTest"
 # rm .coverage *.db
-find . -name *.pyc | xargs rm ; nosetests --cover-erase $1
+find . -name *.pyc | xargs rm ; nosetests -s --cover-erase $1
diff --git a/tests/fixtures.py b/tests/fixtures.py
index b15b3a1..50981a0 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -141,7 +141,12 @@
         
         self.project = 'enwiki'
         
-        self.cohort = Cohort(name='{0}-cohort'.format(name), enabled=True, 
public=False)
+        self.cohort = Cohort(
+            name='{0}-cohort'.format(name),
+            enabled=True,
+            public=False,
+            validated=True,
+        )
         self.session.add(self.cohort)
         self.session.commit()
         
@@ -169,6 +174,7 @@
                     'mediawiki_username'    : editor.user_name,
                     'mediawiki_userid'      : editor.user_id,
                     'project'               : self.project,
+                    'valid'                 : True,
                 }
                 for editor in self.editors
             ]
diff --git a/tests/test_controllers/test_cohorts.py 
b/tests/test_controllers/test_cohorts.py
index 04e2b45..2ab599d 100644
--- a/tests/test_controllers/test_cohorts.py
+++ b/tests/test_controllers/test_cohorts.py
@@ -15,7 +15,7 @@
     link_to_user_page,
     validate_records,
 )
-from wikimetrics.models import Cohort
+from wikimetrics.models import Cohort, CohortUser, CohortUserRole
 
 
 class CohortsControllerTest(WebTest):
@@ -29,6 +29,25 @@
         parsed = json.loads(response.data)
         assert_equal(parsed['cohorts'][0]['name'], self.cohort.name)
     
+    def test_list_includes_only_validated(self):
+        cohorts = [Cohort(name='c1', validated=False), Cohort(name='c2', 
validated=True)]
+        self.session.add_all(cohorts)
+        owners = [
+            CohortUser(
+                cohort_id=c.id,
+                user_id=self.owner_user_id,
+                role=CohortUserRole.OWNER
+            )
+            for c in cohorts
+        ]
+        self.session.add_all(owners)
+        
+        response = self.app.get('/cohorts/list', follow_redirects=True)
+        parsed = json.loads(response.data)
+        assert_equal(len(parsed['cohorts']), 2)
+        assert_equal(parsed['cohorts'][0].validated, True)
+        assert_equal(parsed['cohorts'][1].validated, True)
+    
     def test_detail(self):
         response = self.app.get('/cohorts/detail/{0}'.format(self.cohort.id))
         parsed = json.loads(response.data)
diff --git a/tests/test_models/test_cohort.py b/tests/test_models/test_cohort.py
new file mode 100644
index 0000000..bd94059
--- /dev/null
+++ b/tests/test_models/test_cohort.py
@@ -0,0 +1,34 @@
+from nose.tools import assert_equal
+from wikimetrics.models import Cohort
+from ..fixtures import DatabaseTest
+
+
+class CohortTest(DatabaseTest):
+    def setUp(self):
+        DatabaseTest.setUp(self)
+        self.common_cohort_1()
+    
+    def test_iter(self):
+        self.editors[0].valid = False
+        self.editors[1].valid = None
+        self.session.commit()
+        
+        user_ids = list(self.cohort)
+        assert_equal(self.editors[0].user_id in user_ids, False)
+        assert_equal(self.editors[1].user_id in user_ids, False)
+        assert_equal(self.editors[2].user_id in user_ids, True)
+        assert_equal(self.editors[3].user_id in user_ids, True)
+        assert_equal(len(user_ids), 2)
+    
+    def test_group_by_project(self):
+        self.editors[0].valid = False
+        self.editors[1].valid = None
+        self.session.commit()
+        
+        grouped = self.cohort.group_by_project()
+        user_ids = list(list(grouped)[0])
+        assert_equal(self.editors[0].user_id in user_ids, False)
+        assert_equal(self.editors[1].user_id in user_ids, False)
+        assert_equal(self.editors[2].user_id in user_ids, True)
+        assert_equal(self.editors[3].user_id in user_ids, True)
+        assert_equal(len(user_ids), 2)
diff --git a/tests/test_models/test_run_report.py 
b/tests/test_models/test_run_report.py
index c83aa50..eb6010a 100644
--- a/tests/test_models/test_run_report.py
+++ b/tests/test_models/test_run_report.py
@@ -4,7 +4,7 @@
 from wikimetrics.models import (
     RunReport, Aggregation, PersistentReport
 )
-from wikimetrics.metrics import TimeseriesChoices
+from wikimetrics.metrics import TimeseriesChoices, metric_classes
 
 
 class RunReportTest(QueueDatabaseTest):
@@ -50,6 +50,36 @@
             2,
         )
     
+    def test_does_not_run_invalid_cohort_for_any_metric(self):
+        self.cohort.validated = False
+        self.session.commit()
+        
+        for name, metric in metric_classes.iteritems():
+            desired_responses = [{
+                'name': '{0} - test'.format(name),
+                'cohort': {
+                    'id': self.cohort.id,
+                },
+                'metric': {
+                    'name': name,
+                    'namespaces': [0, 1, 2],
+                    'start_date': '2013-01-01 00:00:00',
+                    'end_date': '2013-01-02 00:00:00',
+                    'individualResults': True,
+                    'aggregateResults': False,
+                    'aggregateSum': False,
+                    'aggregateAverage': False,
+                    'aggregateStandardDeviation': False,
+                },
+            }]
+            jr = RunReport(desired_responses, user_id=self.owner_user_id)
+            celery_task = jr.task.delay(jr)
+            # return value is not used from this .get() call
+            celery_task.get()
+            error_message = '{0} ran with invalid cohort'.format(name)
+            assert_equals(celery_task.ready(), True, error_message)
+            assert_equals(celery_task.failed(), True, error_message)
+    
     def test_aggregated_response_namespace_edits(self):
         desired_responses = [{
             'name': 'Edits - test',
diff --git a/tests/test_queue/test_async_cohort_validation.py 
b/tests/test_queue/test_async_cohort_validation.py
new file mode 100644
index 0000000..d9581af
--- /dev/null
+++ b/tests/test_queue/test_async_cohort_validation.py
@@ -0,0 +1,59 @@
+from nose.tools import assert_not_equals, assert_equals
+from tests.fixtures import QueueTest, QueueDatabaseTest
+from wikimetrics.models import MetricReport
+from wikimetrics.configurables import db
+from wikimetrics.metrics import RandomMetric
+from wikimetrics.models.cohort_upload_node import ValidateCohort
+from wikimetrics.models import (
+    MediawikiUser, Cohort, WikiUser
+)
+from pprint import pprint
+import sys
+
+
+class ValidateCohortTest(QueueDatabaseTest):
+
+    def setUp(self):
+        QueueDatabaseTest.setUp(self)
+
+        db_session = self.mwSession
+        db_session.add(MediawikiUser(user_name="Editor test-specific-0"))
+        db_session.add(MediawikiUser(user_name="Editor test-specific-1"))
+        db_session.commit()
+        #self.common_cohort_1()
+        #session = db.get_session()
+        #session.
+        pass
+
+    def test_small_cohort(self):
+        records = [
+            # two existing users
+            {"raw_username": "Editor test-specific-0",
+             "project": "enwiki"},
+            {"raw_username": "Editor test-specific-1",
+             "project": "enwiki"},
+
+            # one invalid username
+            # NOTE: we're assuming the project will be valid since
+            # otherwise db.get_mw_session(project) will break inside 
get_wikiuser_by_name,
+            # get_wikiuser_by_id
+            # since the project is invalid. So we won't test for that
+            {"raw_username": "Nonexisting",
+             "project": "enwiki"},
+        ]
+
+        session = self.session
+        for r in records:
+            r["username"] = r["raw_username"]
+        v = ValidateCohort(records, "small_cohort", "cohort_desc", "enwiki")
+        async_result = v.task.delay(v)
+        sync_result = async_result.get()
+
+        pprint(sync_result)
+
+        assert_equals(session.query(WikiUser).filter(
+            WikiUser.mediawiki_username == 'Editor 
test-specific-0').one().valid, True)
+        assert_equals(session.query(WikiUser).filter(
+            WikiUser.mediawiki_username == 'Editor 
test-specific-1').one().valid, True)
+        assert_equals(session.query(WikiUser).filter(
+            WikiUser.mediawiki_username == 'Nonexisting').one().valid, False)
diff --git a/tests/test_queue/test_queue_setup_teardown.py 
b/tests/test_queue/test_queue_setup_teardown.py
index df21f74..486d3de 100644
--- a/tests/test_queue/test_queue_setup_teardown.py
+++ b/tests/test_queue/test_queue_setup_teardown.py
@@ -10,6 +10,8 @@
         metric = RandomMetric()
         report = MetricReport(metric, [1, 2], 'enwiki')
         async_result = report.task.delay(report)
+        
+        # .get() blocks until task finishes executing
         sync_result = async_result.get()
         assert_not_equals(
             sync_result[1],
diff --git a/wikimetrics/config/celery_config.yaml 
b/wikimetrics/config/celery_config.yaml
index df8d682..27d6f37 100644
--- a/wikimetrics/config/celery_config.yaml
+++ b/wikimetrics/config/celery_config.yaml
@@ -7,4 +7,4 @@
 CELERYD_TASK_TIME_LIMIT             : 60
 CELERYD_TASK_SOFT_TIME_LIMIT        : 30
 DEBUG                               : True
-LOG_LEVEL                           : 'WARNING'
+LOG_LEVEL                           : 'DEBUG'
diff --git a/wikimetrics/config/db_config.yaml 
b/wikimetrics/config/db_config.yaml
index 4a22149..2be115e 100644
--- a/wikimetrics/config/db_config.yaml
+++ b/wikimetrics/config/db_config.yaml
@@ -1,4 +1,4 @@
-SQL_ECHO                        : False
+SQL_ECHO                        : True
 #WIKIMETRICS_ENGINE_URL          : 'sqlite:///test.db'
 #MEDIAWIKI_ENGINE_URL_TEMPLATE   : 'sqlite:///{0}.db'
 # For testing with mysql locally (useful for manual connection survival tests)
diff --git a/wikimetrics/controllers/cohorts.py 
b/wikimetrics/controllers/cohorts.py
index d3c866b..83d1e29 100644
--- a/wikimetrics/controllers/cohorts.py
+++ b/wikimetrics/controllers/cohorts.py
@@ -7,7 +7,8 @@
 from ..configurables import app, db
 from ..models import (
     Cohort, CohortUser, CohortUserRole,
-    User, WikiUser, CohortWikiUser, MediawikiUser
+    User, WikiUser, CohortWikiUser, MediawikiUser,
+    ValidateCohort
 )
 
 
@@ -140,14 +141,20 @@
             
             unparsed = csv.reader(normalize_newlines(csv_file.stream))
             unvalidated = parse_records(unparsed, project)
-            (valid, invalid) = validate_records(unvalidated)
-            
+            #(valid, invalid) = validate_records(unvalidated)
+            vcohort = ValidateCohort(unvalidated, name, description, project)
+            vcohort.task.delay(vcohort)
+
             return render_template(
                 'csv_upload_review.html',
-                valid=valid,
-                invalid=invalid,
-                valid_json=to_safe_json(valid),
-                invalid_json=to_safe_json(invalid),
+                #valid=valid,
+                #invalid=invalid,
+                valid=[],
+                invalid=[],
+                #valid_json=to_safe_json(valid),
+                #invalid_json=to_safe_json(invalid),
+                valid_json=to_safe_json({}),
+                invalid_json=to_safe_json({}),
                 name=name,
                 project=project,
                 description=description,
diff --git a/wikimetrics/models/__init__.py b/wikimetrics/models/__init__.py
index a3b2f35..19ead4f 100644
--- a/wikimetrics/models/__init__.py
+++ b/wikimetrics/models/__init__.py
@@ -7,6 +7,7 @@
 from persistent_report import *
 from user import *
 from wikiuser import *
+from cohort_upload_node import *
 
 # ignore flake8 because of F403 violation
 # flake8: noqa
diff --git a/wikimetrics/models/cohort.py b/wikimetrics/models/cohort.py
index 5bb919e..86edd83 100644
--- a/wikimetrics/models/cohort.py
+++ b/wikimetrics/models/cohort.py
@@ -36,6 +36,7 @@
     changed = Column(DateTime)
     enabled = Column(Boolean)
     public = Column(Boolean, default=False)
+    validated = Column(Boolean, default=False)
     
     def __repr__(self):
         return '<Cohort("{0}")>'.format(self.id)
diff --git a/wikimetrics/models/cohort_upload_node.py 
b/wikimetrics/models/cohort_upload_node.py
new file mode 100644
index 0000000..46e8f17
--- /dev/null
+++ b/wikimetrics/models/cohort_upload_node.py
@@ -0,0 +1,275 @@
+import celery
+from celery.utils.log import get_task_logger
+from flask.ext.login import current_user
+from wikimetrics.configurables import app, db, queue
+from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
+from sqlalchemy.sql.expression import label, between, and_, or_
+from ..utils import deduplicate_by_key
+from wikimetrics.models import (
+    MediawikiUser, Cohort, WikiUser
+)
+from pprint import pprint
+
+__init__ = [
+    'get_wikiuser_by_name',
+    'get_wikiuser_by_id',
+    'update_wu_invalid',
+    'update_wu_valid',
+]
+
+
+# test
+task_logger = get_task_logger(__name__)
+
+
[email protected]()
+def async_validate(validate_cohort):
+    task_logger.info("Running Cohort Validation job")
+    validate_cohort.run()
+    return validate_cohort
+
+
+class ValidateCohort(object):
+    """
+    An instance of this class is responsible for:
+    * Creating a cohort, or loading an existing one
+    * Re-validating the cohort's existing users or validating a CSV record list
+    * Validating asynchronously and updating the database as it goes
+    * Updating the cohort to validated == True once all users have been 
validated
+    """
+    task = async_validate
+    
+    def __init__(self, records, name, description, project):
+        """
+        Initialize validation, set metadata
+        
+        Parameters:
+            cohort  : an existing cohort with validated == False
+            records : a list of records as parsed from a CSV upload
+        """
+        self.records = records
+        self.name = name
+        self.description = description
+        self.project = project
+        self.create_cohort_and_dummy_wikiusers()
+    
+    @classmethod
+    def fromupload(cls, cohort_upload):
+        """
+        Create a new cohort and validate a list of uploaded users for it
+        
+        Parameters:
+            cohort_upload   : the cohort upload form, parsed by WTForms
+        
+        Returns:
+            An instance of ValidateCohort
+        """
+        cohort = Cohort(
+            name=cohort_upload.name,
+            description=cohort_upload.description,
+            default_project=cohort_upload.default_project,
+            enabled=False,
+            public=False,
+            validated=False,
+        )
+        session = db.get_session()
+        session.add(cohort)
+        session.commit()
+        
+        #return cls(cohort, cohort_upload.records)
+        return None
+    
+    #@classmethod
+    #def fromcohort(cls, partial_cohort_upload):
+        #"""
+        #Using an existing cohort, reset all validation metadata and 
re-validate
+        
+        #Parameters:
+            #partial_cohort_upload   : the cohort_id and user records, parsed 
by WTForms
+        
+        #Returns:
+            #An instance of ValidateCohort
+        #"""
+        #session = db.get_session()
+        #cohort = session.query(Cohort).get(partial_cohort_upload.cohort_id)
+        
+        #return cls(cohort, partial_cohort_upload.records)
+
+    def create_cohort_and_dummy_wikiusers(self):
+        session = db.get_session()
+        new_cohort = Cohort(
+            name=self.name,
+            default_project=self.project,
+            description=self.description,
+            enabled=True,
+        )
+        self.cohort = new_cohort
+
+        wikiusers = []
+        session.bind.engine.execute(
+            WikiUser.__table__.insert(), [
+                {
+                    "mediawiki_username": record["raw_username"],
+                    "project": record["project"],
+                    "valid": None,
+                    "reason_invalid": "",
+                } for record in self.records
+            ]
+        )
+        session.add_all(wikiusers)
+        session.commit()
+    
+    def validate_records(self):
+        session = db.get_session()
+        valid = []
+        #invalid = []
+
+        for record in self.records:
+            normalized_project = normalize_project(record['project'])
+            link_project = normalized_project or record['project'] or 'invalid'
+            record['user_str'] = record['username']
+            record['link'] = link_to_user_page(record['username'], 
link_project)
+            if normalized_project is None:
+                record['reason_invalid'] = 'invalid project: %s' % 
record['project']
+                #invalid.append(record)
+                update_wu_invalid(record)
+                continue
+            normalized_user = normalize_user(record['raw_username'], 
normalized_project)
+            # make a link to the potential user page even if user doesn't exist
+            # this gives a chance to see any misspelling etc.
+            if normalized_user is None:
+                app.logger.info(
+                    'invalid user: {0} in project {1}'
+                    .format(record['raw_username'], normalized_project)
+                )
+                record['reason_invalid'] = 'invalid user_name / user_id: 
{0}'.format(
+                    record['raw_username']
+                )
+                #invalid.append(record)
+                update_wu_invalid(record)
+                continue
+            # set the normalized values and append to valid
+            record['project'] = normalized_project
+            record['user_id'], record['username'] = normalized_user
+            #valid.append(record)
+            update_wu_valid(record)
+        valid = deduplicate_by_key(valid, lambda r: (r['username'], 
r['project']))
+        session.query(Cohort).filter(Cohort.id == self.cohort.id) \
+               .update({"validated": True})
+        session.commit()
+    
+    def run(self):
+        self.validate_records()
+    
+    def __repr__(self):
+        return 'ValidateCohort'
+
+
+def update_wu_invalid(record):
+    session = db.get_session()
+    session.query(WikiUser) \
+        .filter(or_(
+            WikiUser.mediawiki_username == record['raw_username'],
+            WikiUser.mediawiki_userid   == record['raw_username'])) \
+        .update({
+            "reason_invalid": record['reason_invalid'],
+            "valid"         : False,
+        })
+    session.commit()
+    session.close()
+
+
+# TODO: naming change for get_wikiuser_by_name because it
+# returns a MediawikiUser object and not WikiUser
+def update_wu_valid(record):
+    wu_by_name = get_wikiuser_by_name(record['raw_username'], 
record['project'])
+    wu_by_uid  = get_wikiuser_by_id(record['raw_username'], record['project'])
+    #wu = wu_by_name if wu_by_name is not None else wu_by_uid
+    wu = wu_by_name or wu_by_uid
+    if wu:
+        session = db.get_session()
+        session.query(WikiUser) \
+            .filter(WikiUser.mediawiki_username == wu.user_name) \
+            .update({
+                "mediawiki_username": wu.user_name,
+                "mediawiki_userid": wu.user_id,
+                "valid"         : True,
+            })
+        session.commit()
+        session.close()
+    else:
+        update_wu_invalid(record)
+
+
+def project_name_for_link(project):
+    if project.endswith('wiki'):
+        return project[:len(project) - 4]
+    return project
+
+
+def get_wikiuser_by_name(username, project):
+    db_session = db.get_mw_session(project)
+    try:
+        wikiuser = db_session.query(MediawikiUser)\
+            .filter(MediawikiUser.user_name == username)\
+            .one()
+        db_session.close()
+        return wikiuser
+    except (MultipleResultsFound, NoResultFound):
+        db_session.close()
+        return None
+
+
+def link_to_user_page(username, project):
+    project = project_name_for_link(project)
+    user_link = 'https://{0}.wikipedia.org/wiki/User:{1}'
+    user_not_found_link = 
'https://{0}.wikipedia.org/wiki/Username_could_not_be_parsed'
+    # TODO: python 2 has insane unicode handling, switch to python 3
+    try:
+        return user_link.format(project, username)
+    except UnicodeEncodeError:
+        try:
+            return user_link.format(project, username.encode('utf8'))
+        except:
+            return user_not_found_link.format(project)
+
+
+def get_wikiuser_by_id(id, project):
+    db_session = db.get_mw_session(project)
+    try:
+        wikiuser = db_session.query(MediawikiUser)\
+            .filter(MediawikiUser.user_id == id)\
+            .one()
+        db_session.close()
+        return wikiuser
+    except (MultipleResultsFound, NoResultFound):
+        db_session.close()
+        return None
+
+
+def normalize_project(project):
+    project = project.strip().lower()
+    if project in db.project_host_map:
+        return project
+    else:
+        # try adding wiki to end
+        new_proj = project + 'wiki'
+        if new_proj not in db.project_host_map:
+            return None
+        else:
+            return new_proj
+
+
+def normalize_user(user_str, project):
+    wikiuser = get_wikiuser_by_name(user_str, project)
+    if wikiuser is not None:
+        return (wikiuser.user_id, wikiuser.user_name)
+    
+    if not user_str.isdigit():
+        return None
+    
+    wikiuser = get_wikiuser_by_id(user_str, project)
+    if wikiuser is not None:
+        return (wikiuser.user_id, wikiuser.user_name)
+    
+    return None
diff --git a/wikimetrics/models/wikiuser.py b/wikimetrics/models/wikiuser.py
index 12115c8..78d8488 100644
--- a/wikimetrics/models/wikiuser.py
+++ b/wikimetrics/models/wikiuser.py
@@ -1,4 +1,4 @@
-from sqlalchemy import Column, Integer, String
+from sqlalchemy import Column, Integer, String, Boolean
 from wikimetrics.configurables import db
 
 __all__ = [
@@ -21,6 +21,11 @@
     mediawiki_username = Column(String(50))
     mediawiki_userid = Column(Integer(50))
     project = Column(String(45))
+    # valid = None means it's not been validated yet
+    # valid = True means it's valid
+    # valid = False means it's invalid
+    valid = Column(Boolean, default=None)
+    reason_invalid = Column(String(200))
     
     def __repr__(self):
         return '<WikiUser("{0}")>'.format(self.id)
diff --git a/wikimetrics/static/js/reportCreate.js 
b/wikimetrics/static/js/reportCreate.js
index a130087..b197742 100644
--- a/wikimetrics/static/js/reportCreate.js
+++ b/wikimetrics/static/js/reportCreate.js
@@ -152,7 +152,7 @@
         var ret = [];
         ko.utils.arrayForEach(request.metrics(), function(metric){
             ko.utils.arrayForEach(request.cohorts(), function(cohort){
-                response = {
+                var response = {
                     name: metric.label + ' - ' + cohort.name,
                     cohort: cohort,
                     metric: metric,
@@ -168,7 +168,7 @@
     
     viewModel.filteredCohorts = ko.computed(function(){
         if (this.cohorts().length && this.filter().length) {
-            filter = this.filter().toLowerCase();
+            var filter = this.filter().toLowerCase();
             return this.cohorts().filter(function(it){
                 var name = it.name.toLowerCase();
                 return name.indexOf(filter) >= 0;
@@ -179,21 +179,21 @@
     }, viewModel);
     
     function setSelected(list){
-        bareList = ko.utils.unwrapObservable(list);
+        var bareList = ko.utils.unwrapObservable(list);
         ko.utils.arrayForEach(bareList, function(item){
             item.selected = ko.observable(false);
         });
     }
     
     function setConfigure(list){
-        bareList = ko.utils.unwrapObservable(list);
+        var bareList = ko.utils.unwrapObservable(list);
         ko.utils.arrayForEach(bareList, function(item){
             item.configure = ko.observable('');
         });
     }
     
     function setAggregationOptions(list){
-        bareList = ko.utils.unwrapObservable(list);
+        var bareList = ko.utils.unwrapObservable(list);
         ko.utils.arrayForEach(bareList, function(item){
             item.individualResults = ko.observable(false);
             item.aggregateResults = ko.observable(true);
@@ -218,7 +218,7 @@
         if (!prefix) {
             prefix = 'should-be-unique';
         }
-        bareList = ko.utils.unwrapObservable(list);
+        var bareList = ko.utils.unwrapObservable(list);
         ko.utils.arrayForEach(bareList, function(item){
             
             item.tabId = ko.computed(function(){
@@ -236,7 +236,7 @@
         var controls = $('#' + parentId + ' div.datetimepicker');
         controls.datetimepicker({language: 'en'});
         // TODO: this might be cleaner if it metric[name] was an observable
-        controls.on('changeDate', function(e){
+        controls.on('changeDate', function(){
             var input = $(this).find('input');
             var name = input.attr('name');
             metric[name] = input.val();

-- 
To view, visit https://gerrit.wikimedia.org/r/91207
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I848a11ec952c0ca865ee789366c31c7700f678e8
Gerrit-PatchSet: 20
Gerrit-Project: analytics/wikimetrics
Gerrit-Branch: master
Gerrit-Owner: Stefan.petrea <[email protected]>
Gerrit-Reviewer: Milimetric <[email protected]>
Gerrit-Reviewer: Stefan.petrea <[email protected]>

_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to