jenkins-bot has submitted this change and it was merged.
Change subject: Move package into extendable types structure
......................................................................
Move package into extendable types structure
This also utilizes flask-restplus for handling swagger
and scaling the app.
Bug: T144421
Change-Id: I0619639a11ddda8925dac11e337e6358932586a3
---
M recommendation/api/api.py
A recommendation/api/helper.py
D recommendation/api/specification.py
D recommendation/api/swagger.yml
D recommendation/api/test/test_api.py
A recommendation/api/types/__init__.py
A recommendation/api/types/translation/__init__.py
R recommendation/api/types/translation/candidate_finders.py
R recommendation/api/types/translation/data_fetcher.py
R recommendation/api/types/translation/filters.py
R recommendation/api/types/translation/pageviews.py
R recommendation/api/types/translation/test/conftest.py
A recommendation/api/types/translation/test/test_api.py
R recommendation/api/types/translation/test/test_candidate_finders.py
R recommendation/api/types/translation/test/test_filters.py
R recommendation/api/types/translation/test/test_pageviews.py
R recommendation/api/types/translation/test/test_recommendation.ini
A recommendation/api/types/translation/translation.py
R recommendation/api/types/translation/utils.py
M recommendation/data/recommendation.wsgi
M recommendation/test/conftest.py
M recommendation/web/static/gf-input.tag
M recommendation/web/templates/index.html
M setup.py
24 files changed, 373 insertions(+), 372 deletions(-)
Approvals:
jenkins-bot: Verified
Nschaaf: Looks good to me, approved
diff --git a/recommendation/api/api.py b/recommendation/api/api.py
index 77330b2..7576cdc 100644
--- a/recommendation/api/api.py
+++ b/recommendation/api/api.py
@@ -1,84 +1,33 @@
-import logging
-import time
-from flask import Blueprint, request, jsonify
+import collections
-from recommendation.api import filters
-from recommendation.api import candidate_finders
-from recommendation.api import pageviews
-from recommendation.api import specification
-from recommendation.utils import event_logger
-from recommendation.utils import language_pairs
+import flask
+import flask_restplus
+from flask_restplus import fields
-api = Blueprint('api', __name__)
-log = logging.getLogger(__name__)
+from recommendation.api import helper
+from recommendation.web import gapfinder
+
+api = helper.build_api('api', __name__)
+
+TypeSpec = collections.namedtuple('Type', ['name', 'spec_path'])
+
+type_model = api.model(TypeSpec.__name__, TypeSpec(
+ name=fields.String(description='Name of the subpart'),
+ spec_path=fields.String(description='Path to spec')
+)._asdict())
[email protected]('/')
-def get_recommendations():
- t1 = time.time()
- try:
- args = parse_and_validate_args(request.args)
- except ValueError as e:
- return jsonify(error=str(e))
[email protected]('/types')
+class Type(flask_restplus.Resource):
- event_logger.log_api_request(**args)
- recs = recommend(**args)
-
- if len(recs) == 0:
- return jsonify(error='Sorry, failed to get recommendations')
-
- t2 = time.time()
- log.info('Request processed in %f seconds', t2 - t1)
-
- return jsonify(specification.marshal_response(recs))
-
-
-def parse_and_validate_args(args):
- clean_args = specification.parse_and_validate_parameters(args)
-
- if not language_pairs.is_valid_language_pair(clean_args['source'],
clean_args['target']):
- raise ValueError('Invalid or duplicate source and/or target language')
-
- return clean_args
-
-
[email protected]_request
-def after_request(response):
- response.headers.add('Access-Control-Allow-Origin', '*')
- response.headers.add('Access-Control-Allow-Headers',
'Content-Type,Authorization')
- response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE')
- return response
-
-
-finder_map = {
- 'morelike': candidate_finders.MorelikeCandidateFinder(),
- 'wiki': candidate_finders.MorelikeCandidateFinder(),
- 'mostpopular': candidate_finders.PageviewCandidateFinder(),
-}
-
-
-def recommend(source, target, search, seed, count, include_pageviews,
max_candidates=500):
- """
- 1. Use finder to select a set of candidate articles
- 2. Filter out candidates that are not missing, are disambiguation pages,
etc
- 3. get pageview info for each passing candidate if desired
- """
-
- recs = []
-
- if seed:
- finder = finder_map[search]
- for seed in seed.split('|'):
- recs.extend(finder.get_candidates(source, seed, max_candidates))
- else:
- recs.extend(finder_map['mostpopular'].get_candidates(source, seed,
max_candidates))
-
- recs = sorted(recs, key=lambda x: x.rank)
-
- recs = filters.apply_filters(source, target, recs, count)
-
- if recs and include_pageviews:
- recs = pageviews.set_pageview_data(source, recs)
-
- recs = sorted(recs, key=lambda x: x.rank)
- return [{'title': r.title, 'pageviews': r.pageviews, 'wikidata_id':
r.wikidata_id} for r in recs]
+ @api.marshal_with(type_model, as_list=True)
+ @api.doc(description='This returns the available subparts and the paths to
their specs')
+ def get(self):
+ types = []
+ for blue in flask.current_app.iter_blueprints():
+ if type(blue) is flask.Blueprint and blue not in (api.blueprint,
gapfinder.gapfinder):
+ types.append(TypeSpec(
+ name=blue.name,
+ spec_path=flask.url_for(blue.name + '.spec')
+ )._asdict())
+ return types
diff --git a/recommendation/api/helper.py b/recommendation/api/helper.py
new file mode 100644
index 0000000..281bc83
--- /dev/null
+++ b/recommendation/api/helper.py
@@ -0,0 +1,25 @@
+import flask
+import flask_restplus as restplus
+
+
+def build_api(name, import_name, url_prefix=None):
+ blueprint = flask.Blueprint(name, import_name, url_prefix=url_prefix)
+ api = restplus.Api(blueprint, validate=True)
+
+ class Spec(restplus.Resource):
+ def get(self):
+ return flask.jsonify(api.__schema__)
+
+ @blueprint.after_request
+ def after_request(response):
+ response.headers.add('Access-Control-Allow-Origin', '*')
+ response.headers.add('Access-Control-Allow-Headers',
'Content-Type,Authorization')
+ response.headers.add('Access-Control-Allow-Methods', 'GET')
+ return response
+
+ api.add_resource(Spec, '/spec')
+ return api
+
+
+def build_namespace(api, name, description):
+ return api.namespace(name, description=description)
diff --git a/recommendation/api/specification.py
b/recommendation/api/specification.py
deleted file mode 100644
index dc37646..0000000
--- a/recommendation/api/specification.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import yaml
-import pkg_resources
-from bravado_core import spec
-from bravado_core import marshal
-from bravado_core import param
-from bravado_core import validate
-
-from recommendation import api
-
-_parsed_spec = None
-
-
-def initialize_specification():
- global _parsed_spec
- if _parsed_spec is None:
- spec_dict =
yaml.load(open(pkg_resources.resource_filename(api.__name__,
'swagger.yml')).read())
- _parsed_spec = spec.Spec.from_dict(spec_dict)
-
-
-def parse_and_validate_parameters(raw_params):
- initialize_specification()
-
- clean_params = {}
- for parameter in _parsed_spec.spec_dict['parameters']:
- parameter_spec = _parsed_spec.spec_dict['parameters'][parameter]
- parameter_type = parameter_spec['type']
- parameter_name = parameter_spec['name']
- try:
- value = param.cast_request_param(parameter_type, parameter,
raw_params.get(parameter_name))
- validate.validate_schema_object(_parsed_spec, parameter_spec,
value)
- clean_params[parameter] =
marshal.marshal_schema_object(_parsed_spec, parameter_spec, value)
- except Exception as e:
- raise ValueError(e)
-
- return clean_params
-
-
-def marshal_response(recommendations):
- initialize_specification()
-
- articles_spec = _parsed_spec.spec_dict['definitions']['Articles']
- articles = marshal.marshal_schema_object(_parsed_spec, articles_spec,
{'articles': recommendations})
- return articles
diff --git a/recommendation/api/swagger.yml b/recommendation/api/swagger.yml
deleted file mode 100644
index 4f90111..0000000
--- a/recommendation/api/swagger.yml
+++ /dev/null
@@ -1,101 +0,0 @@
-swagger: '2.0'
-
-info:
- version: "1"
- title: Wikimedia Recommendation API
- description: |
- This API provides personalized recommendations for a variety
- of use cases
- contact:
- name: '#wikimedia-research'
- url: http://freenode.net
-
-definitions:
- Articles:
- type: object
- required:
- - articles
- properties:
- articles:
- type: array
- items:
- $ref: '#/definitions/Article'
- Article:
- type: object
- properties:
- pageviews:
- type: integer
- title:
- type: string
- wikidata_id:
- type: string
-
-parameters:
- source:
- name: s
- in: query
- description: Source wiki project language code
- required: true
- type: string
- target:
- name: t
- in: query
- description: Target wiki project language code
- required: true
- type: string
- count:
- name: n
- in: query
- description: Number of recommendations to fetch
- required: false
- type: integer
- format: int32
- maximum: 24
- minimum: 0
- default: 12
- seed:
- name: article
- in: query
- description: |
- Seed article for personalized recommendations that
- can also be a list separated by "|"'
- required: false
- type: string
- pattern: '^([^|]+(\|[^|]+)*)?$'
- default: ''
- include_pageviews:
- name: pageviews
- in: query
- description: Whether to include pageview counts
- required: false
- type: boolean
- default: true
- search:
- name: search
- in: query
- description: Which search algorithm to use if a seed is specified
- required: false
- type: string
- enum:
- - morelike
- - wiki
- default: morelike
-
-paths:
- /api:
- get:
- description: |
- Gets `Article` objects of source articles that
- are missing in the target
- parameters:
- - $ref: '#/parameters/source'
- - $ref: '#/parameters/target'
- - $ref: '#/parameters/count'
- - $ref: '#/parameters/seed'
- - $ref: '#/parameters/include_pageviews'
- - $ref: '#/parameters/search'
- responses:
- '200':
- description: Successful response
- schema:
- $ref: '#/definitions/Articles'
diff --git a/recommendation/api/test/test_api.py
b/recommendation/api/test/test_api.py
deleted file mode 100644
index d380365..0000000
--- a/recommendation/api/test/test_api.py
+++ /dev/null
@@ -1,131 +0,0 @@
-import flask
-import pytest
-import json
-import urllib.parse
-
-from recommendation.api import api
-from recommendation.api import utils
-from recommendation.api import filters
-
-GOOD_RESPONSE = {'articles': [
- {'title': 'A', 'pageviews': 10, 'wikidata_id': 123},
- {'title': 'B', 'pageviews': 11, 'wikidata_id': 122},
- {'title': 'C', 'pageviews': 12, 'wikidata_id': 121},
- {'title': 'D', 'pageviews': 13, 'wikidata_id': 120},
- {'title': 'E', 'pageviews': 14, 'wikidata_id': 119},
- {'title': 'F', 'pageviews': 15, 'wikidata_id': 118},
- {'title': 'G', 'pageviews': 16, 'wikidata_id': 117},
- {'title': 'H', 'pageviews': 17, 'wikidata_id': 116},
-]}
-
-
-def get_query_string(input_dict):
- return '/?' + urllib.parse.urlencode(input_dict)
-
-
[email protected]
-def recommend_response(monkeypatch):
- monkeypatch.setattr(api, 'recommend', lambda *args, **kwargs:
GOOD_RESPONSE['articles'])
-
-
[email protected]
-def client():
- app_instance = flask.Flask(__name__)
- app_instance.register_blueprint(api.api)
- return app_instance.test_client()
-
-
[email protected]('url', [
- get_query_string(dict(s='xx', t='yy')),
- get_query_string(dict(s='xx', t='yy', n=13)),
- get_query_string(dict(s='xx', t='yy', article='separated|list|of|titles')),
- get_query_string(dict(s='xx', t='yy', article='Some Article')),
- get_query_string(dict(s='xx', t='yy', article='')),
- get_query_string(dict(s='xx', t='yy', pageviews='false')),
- get_query_string(dict(s='xx', t='yy', search='morelike')),
- get_query_string(dict(s='xx', t='yy', search='wiki')),
-])
[email protected]('recommend_response')
-def test_good_arg_parsing(client, url):
- result = client.get(url)
- assert 200 == result.status_code
- assert GOOD_RESPONSE == json.loads(result.data.decode('utf-8'))
-
-
[email protected]('url', [
- get_query_string(dict(s='xx')),
- get_query_string(dict(t='xx')),
- '/',
- get_query_string(dict(s='xx', t='xx')),
- get_query_string(dict(s='xx', t='yy', n=-1)),
- get_query_string(dict(s='xx', t='yy', n=25)),
- get_query_string(dict(s='xx', t='yy', n='not a number')),
- get_query_string(dict(s='xx', t='yy', article='||||||||||||')),
- get_query_string(dict(s='xx', t='yy', pageviews='not a boolean')),
- get_query_string(dict(s='xx', t='yy', search='not a valid search')),
-])
[email protected]('recommend_response')
-def test_bad_args(client, url):
- result = client.get(url)
- assert 'error' in json.loads(result.data.decode('utf-8'))
-
-
[email protected]('params', [
- dict(s='xx', t='yy'),
-])
-def test_default_params(params):
- args = api.parse_and_validate_args(params)
- assert 12 == args['count']
- assert '' is args['seed']
- assert True is args['include_pageviews']
- assert 'morelike' == args['search']
-
-
-def test_recommend(monkeypatch):
- class MockFinder:
- @classmethod
- def get_candidates(cls, s, seed, n):
- return []
-
- monkeypatch.setattr(api, 'finder_map', {'customsearch': MockFinder})
- args = api.parse_and_validate_args(dict(s='xx', t='yy',
article='Something'))
- args['search'] = 'customsearch'
- result = api.recommend(**args)
- assert [] == result
-
-
-def test_recommend_uses_mostpopular_if_no_seed_is_specified(monkeypatch):
- class MockFinder:
- @classmethod
- def get_candidates(cls, s, seed, n):
- return []
-
- monkeypatch.setattr(api, 'finder_map', {'mostpopular': MockFinder})
- args = api.parse_and_validate_args(dict(s='xx', t='yy'))
- args['search'] = 'customsearch'
- result = api.recommend(**args)
- assert [] == result
-
-
-def test_generated_recommend_response_is_marshalled(client, monkeypatch):
- class MockFinder:
- @classmethod
- def get_candidates(cls, s, seed, n):
- articles = []
- for item in GOOD_RESPONSE['articles']:
- article = utils.Article(item['title'])
- article.pageviews = item['pageviews']
- article.wikidata_id = item['wikidata_id']
- article.rank = article.pageviews
- articles.append(article)
- return articles
- monkeypatch.setattr(api, 'finder_map', {'mostpopular': MockFinder})
- monkeypatch.setattr(filters, 'apply_filters', lambda source, target, recs,
count: recs)
- result = client.get(get_query_string(dict(s='xx', t='yy',
pageviews=False)))
- assert GOOD_RESPONSE == json.loads(result.data.decode('utf-8'))
-
-
[email protected]('recommend_response')
-def test_cors_is_present(client):
- result = client.get(get_query_string(dict(s='xx', t='yy')))
- assert '*' == result.headers.get('Access-Control-Allow-Origin')
diff --git a/recommendation/api/types/__init__.py
b/recommendation/api/types/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/recommendation/api/types/__init__.py
diff --git a/recommendation/api/types/translation/__init__.py
b/recommendation/api/types/translation/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/recommendation/api/types/translation/__init__.py
diff --git a/recommendation/api/candidate_finders.py
b/recommendation/api/types/translation/candidate_finders.py
similarity index 96%
rename from recommendation/api/candidate_finders.py
rename to recommendation/api/types/translation/candidate_finders.py
index 8095588..4f4560b 100644
--- a/recommendation/api/candidate_finders.py
+++ b/recommendation/api/types/translation/candidate_finders.py
@@ -2,9 +2,9 @@
import datetime
import logging
-from recommendation.api.utils import Article
+from recommendation.api.types.translation.utils import Article
from recommendation.utils import configuration
-from recommendation.api import data_fetcher
+from recommendation.api.types.translation import data_fetcher
log = logging.getLogger(__name__)
diff --git a/recommendation/api/data_fetcher.py
b/recommendation/api/types/translation/data_fetcher.py
similarity index 100%
rename from recommendation/api/data_fetcher.py
rename to recommendation/api/types/translation/data_fetcher.py
diff --git a/recommendation/api/filters.py
b/recommendation/api/types/translation/filters.py
similarity index 96%
rename from recommendation/api/filters.py
rename to recommendation/api/types/translation/filters.py
index 0daf039..3261192 100644
--- a/recommendation/api/filters.py
+++ b/recommendation/api/types/translation/filters.py
@@ -1,6 +1,6 @@
import logging
-from recommendation.api import data_fetcher
+from recommendation.api.types.translation import data_fetcher
log = logging.getLogger(__name__)
diff --git a/recommendation/api/pageviews.py
b/recommendation/api/types/translation/pageviews.py
similarity index 89%
rename from recommendation/api/pageviews.py
rename to recommendation/api/types/translation/pageviews.py
index 9f04503..7b923f9 100644
--- a/recommendation/api/pageviews.py
+++ b/recommendation/api/types/translation/pageviews.py
@@ -1,6 +1,6 @@
import concurrent.futures
-from recommendation.api import data_fetcher
+from recommendation.api.types.translation import data_fetcher
def set_pageview_data(source, articles):
diff --git a/recommendation/api/test/conftest.py
b/recommendation/api/types/translation/test/conftest.py
similarity index 100%
rename from recommendation/api/test/conftest.py
rename to recommendation/api/types/translation/test/conftest.py
diff --git a/recommendation/api/types/translation/test/test_api.py
b/recommendation/api/types/translation/test/test_api.py
new file mode 100644
index 0000000..be1af9c
--- /dev/null
+++ b/recommendation/api/types/translation/test/test_api.py
@@ -0,0 +1,140 @@
+import flask
+import pytest
+import json
+import urllib.parse
+
+from recommendation.api.types.translation import translation
+from recommendation.api.types.translation import utils
+from recommendation.api.types.translation import filters
+
+GOOD_RESPONSE = [
+ {'title': 'A', 'pageviews': 10, 'wikidata_id': '123'},
+ {'title': 'B', 'pageviews': 11, 'wikidata_id': '122'},
+ {'title': 'C', 'pageviews': 12, 'wikidata_id': '121'},
+ {'title': 'D', 'pageviews': 13, 'wikidata_id': '120'},
+ {'title': 'E', 'pageviews': 14, 'wikidata_id': '119'},
+ {'title': 'F', 'pageviews': 15, 'wikidata_id': '118'},
+ {'title': 'G', 'pageviews': 16, 'wikidata_id': '117'},
+ {'title': 'H', 'pageviews': 17, 'wikidata_id': '116'},
+]
+
+
[email protected](params=[
+ '/types/translation/v1/articles',
+ '/api/'
+])
+def get_url(request):
+ return lambda input_dict: request.param + '?' +
urllib.parse.urlencode(input_dict)
+
+
[email protected]
+def recommend_response(monkeypatch):
+ monkeypatch.setattr(translation, 'recommend', lambda *args, **kwargs:
GOOD_RESPONSE)
+
+
[email protected]
+def client():
+ app_instance = flask.Flask(__name__)
+ app_instance.register_blueprint(translation.api.blueprint)
+ app_instance.register_blueprint(translation.legacy.blueprint)
+ app_instance.testing = True
+ return app_instance.test_client()
+
+
[email protected]('params', [
+ dict(s='xx', t='yy'),
+ dict(s='xx', t='yy', n=13),
+ dict(s='xx', t='yy', article='separated|list|of|titles'),
+ dict(s='xx', t='yy', article='Some Article'),
+ dict(s='xx', t='yy', article=''),
+ dict(s='xx', t='yy', pageviews='false'),
+ dict(s='xx', t='yy', search='morelike'),
+ dict(s='xx', t='yy', search='wiki'),
+])
[email protected]('recommend_response')
+def test_good_arg_parsing(client, get_url, params):
+ result = client.get(get_url(params))
+ assert 200 == result.status_code
+ assert GOOD_RESPONSE == json.loads(result.data.decode('utf-8'))
+
+
[email protected]('value,expected', [
+ ('false', False),
+ ('true', True),
+ ('False', False),
+ ('True', True),
+ (1, True),
+ (0, False)
+])
+def test_boolean_arg_parsing(client, get_url, value, expected):
+ with client as c:
+ c.get(get_url(dict(s='xx', t='yy', pageviews=value)))
+ args = translation.legacy_params.parse_args()
+ assert expected is args['include_pageviews']
+
+
[email protected]('params', [
+ dict(s='xx'),
+ dict(t='xx'),
+ {},
+ dict(s='xx', t='xx'),
+ dict(s='xx', t='yy', n=-1),
+ dict(s='xx', t='yy', n=25),
+ dict(s='xx', t='yy', n='not a number'),
+ dict(s='xx', t='yy', article='||||||||||||'),
+ dict(s='xx', t='yy', pageviews='not a boolean'),
+ dict(s='xx', t='yy', search='not a valid search'),
+])
[email protected]('recommend_response')
+def test_bad_args(client, get_url, params):
+ result = client.get(get_url(params))
+ assert 'errors' in json.loads(result.data.decode('utf-8'))
+
+
[email protected]('params', [
+ dict(s='xx', t='yy'),
+])
+def test_default_params(client, get_url, params):
+ with client as c:
+ c.get(get_url(params))
+ args = translation.legacy_params.parse_args()
+ assert 12 == args['count']
+ assert None is args['seed']
+ assert True is args['include_pageviews']
+ assert 'morelike' == args['search']
+
+
+def test_recommend_uses_mostpopular_if_no_seed_is_specified(monkeypatch):
+ class MockFinder:
+ @classmethod
+ def get_candidates(cls, s, seed, n):
+ return []
+
+ monkeypatch.setattr(translation, 'finder_map', {'mostpopular': MockFinder})
+ result = translation.recommend(source='xx', target='yy',
search='customsearch', seed=None, count=12,
+ include_pageviews=True)
+ assert [] == result
+
+
+def test_generated_recommend_response_is_marshalled(client, get_url,
monkeypatch):
+ class MockFinder:
+ @classmethod
+ def get_candidates(cls, s, seed, n):
+ articles = []
+ for item in GOOD_RESPONSE:
+ article = utils.Article(item['title'])
+ article.pageviews = item['pageviews']
+ article.wikidata_id = item['wikidata_id']
+ article.rank = article.pageviews
+ articles.append(article)
+ return articles
+ monkeypatch.setattr(translation, 'finder_map', {'mostpopular': MockFinder})
+ monkeypatch.setattr(filters, 'apply_filters', lambda source, target, recs,
count: recs)
+ result = client.get(get_url(dict(s='xx', t='yy', pageviews=False)))
+ assert GOOD_RESPONSE == json.loads(result.data.decode('utf-8'))
+
+
[email protected]('recommend_response')
+def test_cors_is_present(client, get_url):
+ result = client.get(get_url(dict(s='xx', t='yy')))
+ assert '*' == result.headers.get('Access-Control-Allow-Origin')
diff --git a/recommendation/api/test/test_candidate_finders.py
b/recommendation/api/types/translation/test/test_candidate_finders.py
similarity index 98%
rename from recommendation/api/test/test_candidate_finders.py
rename to recommendation/api/types/translation/test/test_candidate_finders.py
index a349054..cfa9492 100644
--- a/recommendation/api/test/test_candidate_finders.py
+++ b/recommendation/api/types/translation/test/test_candidate_finders.py
@@ -5,7 +5,7 @@
import json
import re
-from recommendation.api import candidate_finders
+from recommendation.api.types.translation import candidate_finders
PAGEVIEW_RESPONSE = {
'items': [
diff --git a/recommendation/api/test/test_filters.py
b/recommendation/api/types/translation/test/test_filters.py
similarity index 94%
rename from recommendation/api/test/test_filters.py
rename to recommendation/api/types/translation/test/test_filters.py
index d422dce..4a28ba7 100644
--- a/recommendation/api/test/test_filters.py
+++ b/recommendation/api/types/translation/test/test_filters.py
@@ -2,9 +2,9 @@
import responses
import re
-from recommendation.api import filters
+from recommendation.api.types.translation import filters
from recommendation.utils import configuration
-from recommendation.api import utils
+from recommendation.api.types.translation import utils
SOURCE = 'xx'
diff --git a/recommendation/api/test/test_pageviews.py
b/recommendation/api/types/translation/test/test_pageviews.py
similarity index 92%
rename from recommendation/api/test/test_pageviews.py
rename to recommendation/api/types/translation/test/test_pageviews.py
index 9fe0a0e..b9e9055 100644
--- a/recommendation/api/test/test_pageviews.py
+++ b/recommendation/api/types/translation/test/test_pageviews.py
@@ -3,9 +3,9 @@
import datetime
import re
-from recommendation.api import pageviews
-from recommendation.api import data_fetcher
-from recommendation.api import utils
+from recommendation.api.types.translation import pageviews
+from recommendation.api.types.translation import data_fetcher
+from recommendation.api.types.translation import utils
from recommendation.utils import configuration
TITLE = 'Sample_Title'
diff --git a/recommendation/api/test/test_recommendation.ini
b/recommendation/api/types/translation/test/test_recommendation.ini
similarity index 100%
rename from recommendation/api/test/test_recommendation.ini
rename to recommendation/api/types/translation/test/test_recommendation.ini
diff --git a/recommendation/api/types/translation/translation.py
b/recommendation/api/types/translation/translation.py
new file mode 100644
index 0000000..a58ab0e
--- /dev/null
+++ b/recommendation/api/types/translation/translation.py
@@ -0,0 +1,156 @@
+import collections
+import logging
+import time
+
+import flask_restplus
+from flask_restplus import fields
+from flask_restplus import reqparse
+from flask_restplus import abort
+from flask_restplus import inputs
+
+from recommendation.utils import event_logger
+from recommendation.utils import language_pairs
+from recommendation.api import helper
+from recommendation.api.types.translation import filters
+from recommendation.api.types.translation import candidate_finders
+from recommendation.api.types.translation import pageviews
+
+log = logging.getLogger(__name__)
+
+legacy = helper.build_api('legacy', __name__, url_prefix='/api')
+
+ArticleSpec = collections.namedtuple('Article', ['pageviews', 'title',
'wikidata_id'])
+
+legacy_params = reqparse.RequestParser()
+
+legacy_params.add_argument(
+ 's',
+ type=str,
+ dest='source',
+ required=True)
+legacy_params.add_argument(
+ 't',
+ type=str,
+ dest='target',
+ required=True)
+legacy_params.add_argument(
+ 'n',
+ type=inputs.int_range(low=0, high=24),
+ dest='count',
+ required=False,
+ default=12)
+legacy_params.add_argument(
+ 'article',
+ type=inputs.regex(r'^([^|]+(\|[^|]+)*)?$'),
+ dest='seed',
+ required=False)
+legacy_params.add_argument(
+ 'pageviews',
+ type=inputs.boolean,
+ dest='include_pageviews',
+ required=False,
+ default=True)
+legacy_params.add_argument(
+ 'search',
+ type=str,
+ required=False,
+ default='morelike',
+ choices=['morelike', 'wiki'])
+
+legacy_model = legacy.model(ArticleSpec.__name__, ArticleSpec(
+ pageviews=fields.Integer(description='pageviews', required=False),
+ title=fields.String(description='title', required=True),
+ wikidata_id=fields.String(description='wikidata_id', required=True)
+)._asdict())
+
+legacy_doc = dict(description='Gets recommendations of source articles that
are missing in the target',
+ params=dict(s='Source wiki project language code',
+ t='Target wiki project language code',
+ n='Number of recommendations to fetch',
+ article='Seed article for personalized
recommendations '
+ 'that can also be a list separated by
"|"',
+ pageviews='Whether to include pageview counts',
+ search='Which search algorithm to use if a seed
is specified')
+ )
+
+
[email protected]
[email protected]('/')
+class LegacyArticle(flask_restplus.Resource):
+ @legacy.expect(legacy_params)
+ @legacy.marshal_with(legacy_model, as_list=True)
+ @legacy.doc(**legacy_doc)
+ def get(self):
+ args = legacy_params.parse_args()
+ return process_request(args)
+
+
+api = helper.build_api('translation', __name__,
url_prefix='/types/translation')
+v1 = helper.build_namespace(api, 'v1', description='')
+v1_params = legacy_params.copy()
+v1_model = legacy.clone(ArticleSpec.__name__, legacy_model)
+v1_doc = legacy_doc.copy()
+
+
[email protected]('/articles')
+class Article(flask_restplus.Resource):
+ @v1.expect(v1_params)
+ @v1.marshal_with(v1_model, as_list=True)
+ @v1.doc(**v1_doc)
+ def get(self):
+ args = v1_params.parse_args()
+ return process_request(args)
+
+
+article_model = v1.model(ArticleSpec.__name__, ArticleSpec(
+ pageviews=fields.Integer(description='pageviews', required=False),
+ title=fields.String(description='title', required=True),
+ wikidata_id=fields.String(description='wikidata_id', required=True)
+)._asdict())
+
+
+def process_request(args):
+ t1 = time.time()
+
+ if not language_pairs.is_valid_language_pair(args['source'],
args['target']):
+ abort(400, errors='Invalid or duplicate source and/or target language')
+
+ event_logger.log_api_request(**args)
+ recs = recommend(**args)
+ t2 = time.time()
+ log.info('Request processed in %f seconds', t2 - t1)
+ return recs
+
+
+finder_map = {
+ 'morelike': candidate_finders.MorelikeCandidateFinder(),
+ 'wiki': candidate_finders.MorelikeCandidateFinder(),
+ 'mostpopular': candidate_finders.PageviewCandidateFinder(),
+}
+
+
+def recommend(source, target, search, seed, count, include_pageviews,
max_candidates=500):
+ """
+ 1. Use finder to select a set of candidate articles
+ 2. Filter out candidates that are not missing, are disambiguation pages,
etc
+ 3. get pageview info for each passing candidate if desired
+ """
+
+ recs = []
+
+ if seed:
+ finder = finder_map[search]
+ for seed in seed.split('|'):
+ recs.extend(finder.get_candidates(source, seed, max_candidates))
+ else:
+ recs.extend(finder_map['mostpopular'].get_candidates(source, seed,
max_candidates))
+
+ recs = sorted(recs, key=lambda x: x.rank)
+
+ recs = filters.apply_filters(source, target, recs, count)
+
+ if recs and include_pageviews:
+ recs = pageviews.set_pageview_data(source, recs)
+
+ recs = sorted(recs, key=lambda x: x.rank)
+ return [{'title': r.title, 'pageviews': r.pageviews, 'wikidata_id':
r.wikidata_id} for r in recs]
diff --git a/recommendation/api/utils.py
b/recommendation/api/types/translation/utils.py
similarity index 100%
rename from recommendation/api/utils.py
rename to recommendation/api/types/translation/utils.py
diff --git a/recommendation/data/recommendation.wsgi
b/recommendation/data/recommendation.wsgi
index 88d2455..f7a6579 100644
--- a/recommendation/data/recommendation.wsgi
+++ b/recommendation/data/recommendation.wsgi
@@ -1,5 +1,6 @@
from flask import Flask
+from recommendation.api.types.translation import translation
from recommendation.api import api
from recommendation.web import gapfinder
from recommendation.utils import logger
@@ -7,8 +8,12 @@
logger.initialize_logging()
app = Flask(__name__)
-app.register_blueprint(api.api, url_prefix='/api')
-app.register_blueprint(gapfinder.gapfinder)
+app.register_blueprint(api.api.blueprint)
+app.register_blueprint(translation.api.blueprint)
+app.register_blueprint(translation.legacy.blueprint)
+app.register_blueprint(gapfinder.gapfinder, url_prefix='/tool')
+app.config['RESTPLUS_VALIDATE'] = True
+app.config['RESTPLUS_MASK_SWAGGER'] = False
application = app
if __name__ == '__main__':
diff --git a/recommendation/test/conftest.py b/recommendation/test/conftest.py
index ed46945..2666af4 100644
--- a/recommendation/test/conftest.py
+++ b/recommendation/test/conftest.py
@@ -14,7 +14,7 @@
to apply a decorator to every test function
"""
configuration._config = configuration.get_configuration('',
recommendation.__name__,
-
'api/test/test_recommendation.ini')
+
'api/types/translation/test/test_recommendation.ini')
logger.initialize_logging()
responses._default_mock.__enter__()
diff --git a/recommendation/web/static/gf-input.tag
b/recommendation/web/static/gf-input.tag
index 04c574e..3bee49d 100644
--- a/recommendation/web/static/gf-input.tag
+++ b/recommendation/web/static/gf-input.tag
@@ -66,7 +66,7 @@
var mappedSource = self.mapLanguageToDomainCode(self.source);
var mappedTarget = self.mapLanguageToDomainCode(self.target);
- var url = '/api/?s=' + mappedSource + '&t=' + mappedTarget;
+ var url = translationAppGlobals.translationPath + '?s=' +
mappedSource + '&t=' + mappedTarget;
var seed;
if (this.seedArticle.value) {
@@ -88,7 +88,7 @@
return;
}
- var articles = self.filter(data.articles);
+ var articles = self.filter(data);
if (!articles || !articles.length) {
self.error_msg = articles['error'];
self.error = true;
diff --git a/recommendation/web/templates/index.html
b/recommendation/web/templates/index.html
index cd1c0bd..60d550f 100644
--- a/recommendation/web/templates/index.html
+++ b/recommendation/web/templates/index.html
@@ -69,7 +69,9 @@
eventLoggerUrl: '{{ event_logger_url }}',
i18nGapFinderPath: '{{ url_for('.static', filename='i18n/')
}}',
- i18nUlsPath: '{{ url_for('.resource',
filename='bower_components/uls/i18n/') }}'
+ i18nUlsPath: '{{ url_for('.resource',
filename='bower_components/uls/i18n/') }}',
+
+ translationPath: '{{ url_for('translation.v1_article') }}'
};
riot.mount('gf-disclaimer, gf-title, gf-input');
diff --git a/setup.py b/setup.py
index 68a5edc..de732fa 100644
--- a/setup.py
+++ b/setup.py
@@ -12,8 +12,7 @@
long_description='',
packages=find_packages(exclude=['test', 'test.*', '*.test']),
install_requires=['flask',
- 'bravado-core',
- 'PyYAML',
+ 'flask-restplus',
'requests',
'numpy'],
package_data={'recommendation.web': ['static/*.*',
--
To view, visit https://gerrit.wikimedia.org/r/316557
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I0619639a11ddda8925dac11e337e6358932586a3
Gerrit-PatchSet: 3
Gerrit-Project: research/recommendation-api
Gerrit-Branch: master
Gerrit-Owner: Nschaaf <[email protected]>
Gerrit-Reviewer: Nschaaf <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
