Milimetric has submitted this change and it was merged.
Change subject: db design update, forms renamed, tests fixed
......................................................................
db design update, forms renamed, tests fixed
Change-Id: I8664f0f1b8a3eadeb1f110c4be5209aef393807b
---
M .gitignore
D design/User Metrics Database Model - User Tagging.mwb
A design/Wiki Metrics Database Model.mwb
M tests/fixtures.py
M tests/test_controllers/test_cohorts.py
M tests/test_metrics/test_bytes_added.py
M tests/test_models/test_mappings.py
M wikimetrics/config/db_config.yaml
M wikimetrics/controllers/metrics.py
M wikimetrics/controllers/reports.py
M wikimetrics/static/js/reportCreate.js
M wikimetrics/templates/csv_upload.html
M wikimetrics/templates/csv_upload_review.html
R wikimetrics/templates/forms/csv_upload.html
R wikimetrics/templates/forms/metric_configuration.html
A wikimetrics/templates/forms/output_configuration.html
R wikimetrics/templates/report.html
17 files changed, 170 insertions(+), 117 deletions(-)
Approvals:
Milimetric: Verified; Looks good to me, approved
diff --git a/.gitignore b/.gitignore
index 4a28ba2..00c7c63 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,7 @@
*.csv
*.json
*.db
+*.bak
*egg-info
diff --git a/design/User Metrics Database Model - User Tagging.mwb
b/design/User Metrics Database Model - User Tagging.mwb
deleted file mode 100644
index 6628c28..0000000
--- a/design/User Metrics Database Model - User Tagging.mwb
+++ /dev/null
Binary files differ
diff --git a/design/Wiki Metrics Database Model.mwb b/design/Wiki Metrics
Database Model.mwb
new file mode 100644
index 0000000..73ed219
--- /dev/null
+++ b/design/Wiki Metrics Database Model.mwb
Binary files differ
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 689a6e6..282c55a 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -34,24 +34,122 @@
def setUp(self):
- # create basic test records for non-mediawiki tests
- self.session = db.get_session()
-
+ #****************************************************************
+ # set up for every test - delete and re-create all needed records
+ #****************************************************************
project = 'enwiki'
+ self.session = db.get_session()
engine = db.get_mw_engine(project)
db.MediawikiBase.metadata.create_all(engine, checkfirst=True)
self.mwSession = db.get_mw_session(project)
DatabaseTest.tearDown(self)
+ #****************************************************************
+ # create records for enwiki tests
+ #****************************************************************
+ mw_user_dan = MediawikiUser(user_name='Dan')
+ mw_user_evan = MediawikiUser(user_name='Evan')
+ mw_user_andrew = MediawikiUser(user_name='Andrew')
+ mw_user_diederik = MediawikiUser(user_name='Diederik')
+ mw_logging = Logging(log_user_text='Reedy')
+ mw_page = Page(page_namespace=0, page_title='Main_Page')
+ self.mwSession.add_all([
+ mw_user_dan,
+ mw_user_evan,
+ mw_user_andrew,
+ mw_logging,
+ mw_page,
+ ])
+ self.mwSession.commit()
+
+ # edits in between Dan and Evan edits
+ rev_before_1 = Revision(
+ rev_page=mw_page.page_id, rev_user=mw_user_diederik.user_id,
+ rev_comment='before Dan edit 1',
+ rev_len=4, rev_timestamp=datetime(2013, 05, 30),
+ )
+ rev_before_2 = Revision(
+ rev_page=mw_page.page_id, rev_user=mw_user_diederik.user_id,
+ rev_comment='before Dan edit 2',
+ rev_len=0, rev_timestamp=datetime(2013, 06, 30),
+ )
+ rev_before_3 = Revision(
+ rev_page=mw_page.page_id, rev_user=mw_user_diederik.user_id,
+ rev_comment='before Evan edit 1',
+ rev_len=0, rev_timestamp=datetime(2013, 05, 30),
+ )
+ rev_before_4 = Revision(
+ rev_page=mw_page.page_id, rev_user=mw_user_diederik.user_id,
+ rev_comment='before Evan edit 2',
+ rev_len=100, rev_timestamp=datetime(2013, 06, 30),
+ )
+ rev_before_5 = Revision(
+ rev_page=mw_page.page_id, rev_user=mw_user_diederik.user_id,
+ rev_comment='before Evan edit 3',
+ rev_len=140, rev_timestamp=datetime(2013, 07, 23),
+ )
+ self.mwSession.add_all([
+ rev_before_1,
+ rev_before_2,
+ rev_before_3,
+ rev_before_4,
+ rev_before_5,
+ ])
+ self.mwSession.commit()
+
+ # Dan edits
+ rev1 = Revision(
+ rev_page=mw_page.page_id, rev_user=mw_user_dan.user_id,
rev_comment='Dan edit 1',
+ rev_parent_id=rev_before_1.rev_id, rev_len=0,
rev_timestamp=datetime(2013, 06, 01),
+ )
+ rev2 = Revision(
+ rev_page=mw_page.page_id, rev_user=mw_user_dan.user_id,
rev_comment='Dan edit 2',
+ rev_parent_id=rev_before_2.rev_id, rev_len=10,
rev_timestamp=datetime(2013, 07, 01),
+ )
+ # Evan edits
+ rev3 = Revision(
+ rev_page=mw_page.page_id, rev_user=mw_user_evan.user_id,
rev_comment='Evan edit 1',
+ rev_parent_id=rev_before_3.rev_id, rev_len=100,
rev_timestamp=datetime(2013, 06, 01),
+ )
+ rev4 = Revision(
+ rev_page=mw_page.page_id, rev_user=mw_user_evan.user_id,
rev_comment='Evan edit 2',
+ rev_parent_id=rev_before_4.rev_id, rev_len=140,
rev_timestamp=datetime(2013, 07, 01),
+ )
+ rev5 = Revision(
+ rev_page=mw_page.page_id, rev_user=mw_user_evan.user_id,
rev_comment='Evan edit 3',
+ rev_parent_id=rev_before_5.rev_id, rev_len=136,
rev_timestamp=datetime(2013, 07, 24),
+ )
+ self.mwSession.add_all([rev1, rev2, rev3, rev4, rev5])
+ self.mwSession.commit()
+
+ #****************************************************************
+ # create basic test records for non-mediawiki tests
+ #****************************************************************
dan_user = User(username='Dan')
evan_user = User(username='Evan')
web_test_user = User(email='[email protected]')
# create a test cohort
- dan = WikiUser(mediawiki_username='Dan', mediawiki_userid=1,
project='enwiki')
- evan = WikiUser(mediawiki_username='Evan', mediawiki_userid=2,
project='enwiki')
- andrew = WikiUser(mediawiki_username='Andrew', mediawiki_userid=3,
project='enwiki')
- diederik = WikiUser(mediawiki_username='Diederik', mediawiki_userid=4,
project='enwiki')
+ dan = WikiUser(
+ mediawiki_username=mw_user_dan.user_name,
+ mediawiki_userid=mw_user_dan.user_id,
+ project=project
+ )
+ evan = WikiUser(
+ mediawiki_username=mw_user_evan.user_name,
+ mediawiki_userid=mw_user_evan.user_id,
+ project=project
+ )
+ andrew = WikiUser(
+ mediawiki_username=mw_user_andrew.user_name,
+ mediawiki_userid=mw_user_andrew.user_id,
+ project=project
+ )
+ diederik = WikiUser(
+ mediawiki_username=mw_user_diederik.user_name,
+ mediawiki_userid=mw_user_diederik.user_id,
+ project=project
+ )
# create cohorts
test_cohort = Cohort(name='test', enabled=True, public=True)
@@ -166,57 +264,19 @@
])
self.session.commit()
- # create records for enwiki tests
- # TODO: make this safe to execute in any environment
- self.mwSession.add(MediawikiUser(user_id=1, user_name='Dan'))
- self.mwSession.add(MediawikiUser(user_id=2, user_name='Evan'))
- self.mwSession.add(MediawikiUser(user_id=3, user_name='Andrew'))
- self.mwSession.add(Logging(log_id=1, log_user_text='Reedy'))
- self.mwSession.add(Page(page_id=1, page_namespace=0,
page_title='Main_Page'))
- # edits in between Dan and Evan edits
- self.mwSession.add(Revision(
- rev_id=1, rev_page=1, rev_user=4, rev_comment='before Dan edit 1',
- rev_len=4, rev_timestamp=datetime(2013, 05, 30),
- ))
- self.mwSession.add(Revision(
- rev_id=3, rev_page=1, rev_user=4, rev_comment='before Dan edit 2',
- rev_len=0, rev_timestamp=datetime(2013, 06, 30),
- ))
- self.mwSession.add(Revision(
- rev_id=5, rev_page=1, rev_user=4, rev_comment='before Evan edit 1',
- rev_len=0, rev_timestamp=datetime(2013, 05, 30),
- ))
- self.mwSession.add(Revision(
- rev_id=7, rev_page=1, rev_user=4, rev_comment='before Evan edit 2',
- rev_len=100, rev_timestamp=datetime(2013, 06, 30),
- ))
- self.mwSession.add(Revision(
- rev_id=9, rev_page=1, rev_user=4, rev_comment='before Evan edit 3',
- rev_len=140, rev_timestamp=datetime(2013, 07, 23),
- ))
- # Dan edits
- self.mwSession.add(Revision(
- rev_id=2, rev_page=1, rev_user=1, rev_comment='Dan edit 1',
- rev_parent_id=1, rev_len=0, rev_timestamp=datetime(2013, 06, 01),
- ))
- self.mwSession.add(Revision(
- rev_id=4, rev_page=1, rev_user=1, rev_comment='Dan edit 2',
- rev_parent_id=3, rev_len=10, rev_timestamp=datetime(2013, 07, 01),
- ))
- # Evan edits
- self.mwSession.add(Revision(
- rev_id=6, rev_page=1, rev_user=2, rev_comment='Evan edit 1',
- rev_parent_id=5, rev_len=100, rev_timestamp=datetime(2013, 06, 01),
- ))
- self.mwSession.add(Revision(
- rev_id=8, rev_page=1, rev_user=2, rev_comment='Evan edit 2',
- rev_parent_id=7, rev_len=140, rev_timestamp=datetime(2013, 07, 01),
- ))
- self.mwSession.add(Revision(
- rev_id=10, rev_page=1, rev_user=2, rev_comment='Evan edit 3',
- rev_parent_id=9, rev_len=136, rev_timestamp=datetime(2013, 07, 24),
- ))
- self.mwSession.commit()
+ #****************************************************************
+ # keep the test ids around so subclasses can use them
+ #****************************************************************
+ self.test_report_id = report_created.id
+ self.test_user_id = dan_user.id
+ self.test_cohort_id = test_cohort.id
+ self.test_cohort_user_id = dan_owns_test.id
+ self.test_wiki_user_id = dan.id
+ self.test_cohort_wiki_user_id = dan_in_test.id
+ self.test_logging_id = mw_logging.log_id
+ self.test_mediawiki_user_id = mw_user_dan.user_id
+ self.test_page_id = mw_page.page_id
+ self.test_revision_id = rev1.rev_id
def tearDown(self):
@@ -291,7 +351,15 @@
.join(CohortWikiUser)\
.filter(CohortWikiUser.cohort_id == self.cohort.id)
- self.dan_id = wikiusers.filter(WikiUser.mediawiki_username ==
'Dan').one().id
- self.evan_id = wikiusers.filter(WikiUser.mediawiki_username ==
'Evan').one().id
- self.andrew_id = wikiusers.filter(WikiUser.mediawiki_username ==
'Andrew').one().id
- self.diederik_id = wikiusers.filter(WikiUser.mediawiki_username ==
'Diederik').one().id
+ self.dan_id = wikiusers\
+ .filter(WikiUser.mediawiki_username == 'Dan')\
+ .one().mediawiki_userid
+ self.evan_id = wikiusers\
+ .filter(WikiUser.mediawiki_username == 'Evan')\
+ .one().mediawiki_userid
+ self.andrew_id = wikiusers\
+ .filter(WikiUser.mediawiki_username == 'Andrew')\
+ .one().mediawiki_userid
+ self.diederik_id = wikiusers\
+ .filter(WikiUser.mediawiki_username == 'Diederik')\
+ .one().mediawiki_userid
diff --git a/tests/test_controllers/test_cohorts.py
b/tests/test_controllers/test_cohorts.py
index 7700f30..943f215 100644
--- a/tests/test_controllers/test_cohorts.py
+++ b/tests/test_controllers/test_cohorts.py
@@ -3,17 +3,14 @@
import json
from nose.tools import assert_equal, assert_not_equal
from tests.fixtures import WebTest
-from wikimetrics.controllers.cohorts import *
+from wikimetrics.controllers.cohorts import parse_username, normalize_user
class TestCohortsController(WebTest):
def test_index(self):
response = self.app.get('/cohorts/', follow_redirects=True)
- assert_equal(
- response.status_code, 200,
- '/cohorts should get the list of cohorts'
- )
+ assert_equal(response.status_code, 200)
def test_list(self):
response = self.app.get('/cohorts/list', follow_redirects=True)
@@ -26,40 +23,21 @@
)
def test_detail(self):
- response = self.app.get('/cohorts/detail/1', follow_redirects=True)
+ response =
self.app.get('/cohorts/detail/{0}'.format(self.test_cohort_id))
parsed = json.loads(response.data)
- assert_equal(
- response.status_code,
- 200,
- )
- assert_equal(
- len(parsed['wikiusers']),
- 3,
- '/cohorts/detail/1 should return JSON object with key `wikiusers`'
- 'for a list of length 3== `test`, but instead returned:
{0}'.format(parsed)
- )
+ assert_equal(response.status_code, 200)
+ assert_equal(len(parsed['wikiusers']), 3)
def test_full_detail(self):
- response = self.app.get('/cohorts/detail/1?full_detail=true',
follow_redirects=True)
+ response =
self.app.get('/cohorts/detail/{0}?full_detail=true'.format(self.test_cohort_id))
parsed = json.loads(response.data)
- assert_equal(
- response.status_code,
- 200,
- )
- assert_equal(
- len(parsed['wikiusers']),
- 4,
- '/cohorts/detail/1 should return JSON object with key `wikiusers`'
- 'for a list of length 4== `test`, but instead returned:
{0}'.format(parsed)
- )
+ assert_equal(response.status_code, 200)
+ assert_equal(len(parsed['wikiusers']), 4)
def test_not_found(self):
response =
self.app.get('/cohorts/detail/no_way_anybody_names_a_cohort_this_23982739873')
- assert_equal(
- response.status_code,
- 404,
- )
+ assert_equal(response.status_code, 404)
def test_validate_username(self):
# this username has a few problems that the normalize call should
handle
diff --git a/tests/test_metrics/test_bytes_added.py
b/tests/test_metrics/test_bytes_added.py
index 995dd29..0eef4f8 100644
--- a/tests/test_metrics/test_bytes_added.py
+++ b/tests/test_metrics/test_bytes_added.py
@@ -17,6 +17,7 @@
)
results = metric(list(self.cohort), self.mwSession)
+ print results
dan_expected = {
'net_sum': 6,
'absolute_sum': 14,
@@ -48,6 +49,7 @@
assert_true(metric.validate())
results = metric(list(self.cohort), self.mwSession)
+ print results
dan_expected = {
'net_sum': 10,
'absolute_sum': 10,
@@ -72,6 +74,7 @@
assert_true(metric.validate())
results = metric(list(self.cohort), self.mwSession)
+ print results
dan_expected = {
'net_sum': 6,
'absolute_sum': 14,
diff --git a/tests/test_models/test_mappings.py
b/tests/test_models/test_mappings.py
index d026f7a..07a7b8d 100644
--- a/tests/test_models/test_mappings.py
+++ b/tests/test_models/test_mappings.py
@@ -23,50 +23,50 @@
#***********
def test_report(self):
- j = self.session.query(PersistentReport).get(1)
+ j = self.session.query(PersistentReport).get(self.test_report_id)
assert_true(j.status == celery.states.PENDING)
def test_user(self):
- u = self.session.query(User).get(1)
+ u = self.session.query(User).get(self.test_user_id)
assert_true(u.username == 'Dan')
assert_true(u.role == UserRole.GUEST)
def test_wikiuser(self):
- wu = self.session.query(WikiUser).get(1)
+ wu = self.session.query(WikiUser).get(self.test_wiki_user_id)
assert_equals(wu.mediawiki_username, 'Dan')
def test_cohort(self):
- c = self.session.query(Cohort).get(1)
+ c = self.session.query(Cohort).get(self.test_cohort_id)
assert_equals(c.name, 'test')
assert_true(c.public is True)
def test_cohort_user(self):
- cu = self.session.query(CohortUser).get(1)
- assert_equals(cu.user_id, 1)
- assert_equals(cu.cohort_id, 1)
+ cu = self.session.query(CohortUser).get(self.test_cohort_user_id)
+ assert_equals(cu.user_id, self.test_user_id)
+ assert_equals(cu.cohort_id, self.test_cohort_id)
def test_cohort_wikiuser(self):
- cwu = self.session.query(CohortWikiUser).get(1)
- assert_equals(cwu.wiki_user_id, 1)
- assert_equals(cwu.cohort_id, 1)
+ cwu =
self.session.query(CohortWikiUser).get(self.test_cohort_wiki_user_id)
+ assert_equals(cwu.wiki_user_id, self.test_wiki_user_id)
+ assert_equals(cwu.cohort_id, self.test_cohort_id)
#***********
# Mapping tests for mediawiki tables
#***********
def test_mediawiki_logging(self):
- row = self.mwSession.query(Logging).get(1)
+ row = self.mwSession.query(Logging).get(self.test_logging_id)
assert_equals(row.log_user_text, 'Reedy')
def test_mediawiki_user(self):
- row = self.mwSession.query(MediawikiUser).get(2)
- assert_equals(row.user_name, 'Evan')
+ row =
self.mwSession.query(MediawikiUser).get(self.test_mediawiki_user_id)
+ assert_equals(row.user_name, 'Dan')
def test_mediawiki_page(self):
- row = self.mwSession.query(Page).get(1)
+ row = self.mwSession.query(Page).get(self.test_page_id)
assert_equals(row.page_title, 'Main_Page')
def test_mediawiki_revision(self):
- row = self.mwSession.query(Revision).get(2)
+ row = self.mwSession.query(Revision).get(self.test_revision_id)
assert_equals(row.rev_comment, 'Dan edit 1')
#***********
@@ -76,10 +76,10 @@
user_ids = self.session\
.query(WikiUser.mediawiki_userid)\
.join(CohortWikiUser)\
- .filter(CohortWikiUser.cohort_id == 1)\
+ .filter(CohortWikiUser.cohort_id == self.test_cohort_id)\
.all()
- assert_true(len(user_ids) == 4, "Cohort 1 should have 4 wiki users in
it")
+ assert_equal(len(user_ids), 4)
def test_cohort_ownership(self):
cohorts = self.session\
diff --git a/wikimetrics/config/db_config.yaml
b/wikimetrics/config/db_config.yaml
index dfd0624..611b6d1 100644
--- a/wikimetrics/config/db_config.yaml
+++ b/wikimetrics/config/db_config.yaml
@@ -1,5 +1,5 @@
SQL_ECHO : False
-WIKIMETRICS_ENGINE_URL : 'sqlite:///test.db'
+#WIKIMETRICS_ENGINE_URL : 'sqlite:///test.db'
# For testing with mysql locally (useful for manual connection survival tests)
-#WIKIMETRICS_ENGINE_URL :
'mysql://wikimetrics:wikimetrics@localhost/wikimetrics'
+WIKIMETRICS_ENGINE_URL :
'mysql://wikimetrics:wikimetrics@localhost/wikimetrics'
MEDIAWIKI_ENGINE_URL_TEMPLATE : 'sqlite:///{0}.db'
diff --git a/wikimetrics/controllers/metrics.py
b/wikimetrics/controllers/metrics.py
index 95fbc7f..62574c1 100644
--- a/wikimetrics/controllers/metrics.py
+++ b/wikimetrics/controllers/metrics.py
@@ -67,7 +67,7 @@
metric_form = metric_classes[name]()
return render_template(
- 'form_for_metrics.html',
+ 'forms/metric_configuration.html',
form=metric_form,
action=request.url,
)
diff --git a/wikimetrics/controllers/reports.py
b/wikimetrics/controllers/reports.py
index 664fcf1..fbd792c 100644
--- a/wikimetrics/controllers/reports.py
+++ b/wikimetrics/controllers/reports.py
@@ -30,7 +30,7 @@
"""
if request.method == 'GET':
- return render_template('request.html')
+ return render_template('report.html')
else:
parsed = json.loads(request.form['responses'])
metric_reports = []
@@ -100,7 +100,7 @@
@app.route('/reports/status/<task_id>')
def report_status(task_id):
- celery_task = Report.task.AsyncResult(db_report.result_key)
+ celery_task = Report.task.AsyncResult(task_id)
return json_response(status=celery_task.status)
diff --git a/wikimetrics/static/js/reportCreate.js
b/wikimetrics/static/js/reportCreate.js
index 82d4b7c..8c502eb 100644
--- a/wikimetrics/static/js/reportCreate.js
+++ b/wikimetrics/static/js/reportCreate.js
@@ -106,11 +106,14 @@
return cohort.selected();
});
}, viewModel).extend({ throttle: 1 }),
+
metrics: ko.computed(function(){
return this.metrics().filter(function(metric){
return metric.selected();
});
}, viewModel).extend({ throttle: 1 }),
+
+ aggregators: ko.observable([]),
});
// second level computed pieces of the viewModel
diff --git a/wikimetrics/templates/csv_upload.html
b/wikimetrics/templates/csv_upload.html
index d9e7b02..f458c51 100644
--- a/wikimetrics/templates/csv_upload.html
+++ b/wikimetrics/templates/csv_upload.html
@@ -1,6 +1,6 @@
{% extends "layout.html" %}
{% block body %}
-{% include "csv_upload_form.html" %}
+{% include "forms/csv_upload.html" %}
<div>
<h6>File Format</h6>
<p>User Metrics expects a CSV file with the following format (the
username, project header row is not required)</p>
diff --git a/wikimetrics/templates/csv_upload_review.html
b/wikimetrics/templates/csv_upload_review.html
index baa91a0..71bb548 100644
--- a/wikimetrics/templates/csv_upload_review.html
+++ b/wikimetrics/templates/csv_upload_review.html
@@ -50,7 +50,7 @@
</tbody>
</table>
- {% include "csv_upload_form.html" %}
+ {% include "forms/csv_upload.html" %}
</div>
{% endif %}
{% if valid %}
diff --git a/wikimetrics/templates/csv_upload_form.html
b/wikimetrics/templates/forms/csv_upload.html
similarity index 100%
rename from wikimetrics/templates/csv_upload_form.html
rename to wikimetrics/templates/forms/csv_upload.html
diff --git a/wikimetrics/templates/form_for_metrics.html
b/wikimetrics/templates/forms/metric_configuration.html
similarity index 100%
rename from wikimetrics/templates/form_for_metrics.html
rename to wikimetrics/templates/forms/metric_configuration.html
diff --git a/wikimetrics/templates/forms/output_configuration.html
b/wikimetrics/templates/forms/output_configuration.html
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/wikimetrics/templates/forms/output_configuration.html
diff --git a/wikimetrics/templates/request.html
b/wikimetrics/templates/report.html
similarity index 100%
rename from wikimetrics/templates/request.html
rename to wikimetrics/templates/report.html
--
To view, visit https://gerrit.wikimedia.org/r/76514
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I8664f0f1b8a3eadeb1f110c4be5209aef393807b
Gerrit-PatchSet: 1
Gerrit-Project: analytics/wikimetrics
Gerrit-Branch: master
Gerrit-Owner: Milimetric <[email protected]>
Gerrit-Reviewer: Milimetric <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
