jenkins-bot has submitted this change and it was merged.

Change subject: Move package into extendable types structure
......................................................................


Move package into extendable types structure

This also utilizes flask-restplus for handling swagger
and scaling the app.

Bug: T144421
Change-Id: I0619639a11ddda8925dac11e337e6358932586a3
---
M recommendation/api/api.py
A recommendation/api/helper.py
D recommendation/api/specification.py
D recommendation/api/swagger.yml
D recommendation/api/test/test_api.py
A recommendation/api/types/__init__.py
A recommendation/api/types/translation/__init__.py
R recommendation/api/types/translation/candidate_finders.py
R recommendation/api/types/translation/data_fetcher.py
R recommendation/api/types/translation/filters.py
R recommendation/api/types/translation/pageviews.py
R recommendation/api/types/translation/test/conftest.py
A recommendation/api/types/translation/test/test_api.py
R recommendation/api/types/translation/test/test_candidate_finders.py
R recommendation/api/types/translation/test/test_filters.py
R recommendation/api/types/translation/test/test_pageviews.py
R recommendation/api/types/translation/test/test_recommendation.ini
A recommendation/api/types/translation/translation.py
R recommendation/api/types/translation/utils.py
M recommendation/data/recommendation.wsgi
M recommendation/test/conftest.py
M recommendation/web/static/gf-input.tag
M recommendation/web/templates/index.html
M setup.py
24 files changed, 373 insertions(+), 372 deletions(-)

Approvals:
  jenkins-bot: Verified
  Nschaaf: Looks good to me, approved



diff --git a/recommendation/api/api.py b/recommendation/api/api.py
index 77330b2..7576cdc 100644
--- a/recommendation/api/api.py
+++ b/recommendation/api/api.py
@@ -1,84 +1,33 @@
-import logging
-import time
-from flask import Blueprint, request, jsonify
+import collections
 
-from recommendation.api import filters
-from recommendation.api import candidate_finders
-from recommendation.api import pageviews
-from recommendation.api import specification
-from recommendation.utils import event_logger
-from recommendation.utils import language_pairs
+import flask
+import flask_restplus
+from flask_restplus import fields
 
-api = Blueprint('api', __name__)
-log = logging.getLogger(__name__)
+from recommendation.api import helper
+from recommendation.web import gapfinder
+
+api = helper.build_api('api', __name__)
+
+TypeSpec = collections.namedtuple('Type', ['name', 'spec_path'])
+
+type_model = api.model(TypeSpec.__name__, TypeSpec(
+    name=fields.String(description='Name of the subpart'),
+    spec_path=fields.String(description='Path to spec')
+)._asdict())
 
 
-@api.route('/')
-def get_recommendations():
-    t1 = time.time()
-    try:
-        args = parse_and_validate_args(request.args)
-    except ValueError as e:
-        return jsonify(error=str(e))
+@api.route('/types')
+class Type(flask_restplus.Resource):
 
-    event_logger.log_api_request(**args)
-    recs = recommend(**args)
-
-    if len(recs) == 0:
-        return jsonify(error='Sorry, failed to get recommendations')
-
-    t2 = time.time()
-    log.info('Request processed in %f seconds', t2 - t1)
-
-    return jsonify(specification.marshal_response(recs))
-
-
-def parse_and_validate_args(args):
-    clean_args = specification.parse_and_validate_parameters(args)
-
-    if not language_pairs.is_valid_language_pair(clean_args['source'], 
clean_args['target']):
-        raise ValueError('Invalid or duplicate source and/or target language')
-
-    return clean_args
-
-
-@api.after_request
-def after_request(response):
-    response.headers.add('Access-Control-Allow-Origin', '*')
-    response.headers.add('Access-Control-Allow-Headers', 
'Content-Type,Authorization')
-    response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE')
-    return response
-
-
-finder_map = {
-    'morelike': candidate_finders.MorelikeCandidateFinder(),
-    'wiki': candidate_finders.MorelikeCandidateFinder(),
-    'mostpopular': candidate_finders.PageviewCandidateFinder(),
-}
-
-
-def recommend(source, target, search, seed, count, include_pageviews, 
max_candidates=500):
-    """
-    1. Use finder to select a set of candidate articles
-    2. Filter out candidates that are not missing, are disambiguation pages, 
etc
-    3. get pageview info for each passing candidate if desired
-    """
-
-    recs = []
-
-    if seed:
-        finder = finder_map[search]
-        for seed in seed.split('|'):
-            recs.extend(finder.get_candidates(source, seed, max_candidates))
-    else:
-        recs.extend(finder_map['mostpopular'].get_candidates(source, seed, 
max_candidates))
-
-    recs = sorted(recs, key=lambda x: x.rank)
-
-    recs = filters.apply_filters(source, target, recs, count)
-
-    if recs and include_pageviews:
-        recs = pageviews.set_pageview_data(source, recs)
-
-    recs = sorted(recs, key=lambda x: x.rank)
-    return [{'title': r.title, 'pageviews': r.pageviews, 'wikidata_id': 
r.wikidata_id} for r in recs]
+    @api.marshal_with(type_model, as_list=True)
+    @api.doc(description='This returns the available subparts and the paths to 
their specs')
+    def get(self):
+        types = []
+        for blue in flask.current_app.iter_blueprints():
+            if type(blue) is flask.Blueprint and blue not in (api.blueprint, 
gapfinder.gapfinder):
+                types.append(TypeSpec(
+                    name=blue.name,
+                    spec_path=flask.url_for(blue.name + '.spec')
+                )._asdict())
+        return types
diff --git a/recommendation/api/helper.py b/recommendation/api/helper.py
new file mode 100644
index 0000000..281bc83
--- /dev/null
+++ b/recommendation/api/helper.py
@@ -0,0 +1,25 @@
+import flask
+import flask_restplus as restplus
+
+
+def build_api(name, import_name, url_prefix=None):
+    blueprint = flask.Blueprint(name, import_name, url_prefix=url_prefix)
+    api = restplus.Api(blueprint, validate=True)
+
+    class Spec(restplus.Resource):
+        def get(self):
+            return flask.jsonify(api.__schema__)
+
+    @blueprint.after_request
+    def after_request(response):
+        response.headers.add('Access-Control-Allow-Origin', '*')
+        response.headers.add('Access-Control-Allow-Headers', 
'Content-Type,Authorization')
+        response.headers.add('Access-Control-Allow-Methods', 'GET')
+        return response
+
+    api.add_resource(Spec, '/spec')
+    return api
+
+
+def build_namespace(api, name, description):
+    return api.namespace(name, description=description)
diff --git a/recommendation/api/specification.py 
b/recommendation/api/specification.py
deleted file mode 100644
index dc37646..0000000
--- a/recommendation/api/specification.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import yaml
-import pkg_resources
-from bravado_core import spec
-from bravado_core import marshal
-from bravado_core import param
-from bravado_core import validate
-
-from recommendation import api
-
-_parsed_spec = None
-
-
-def initialize_specification():
-    global _parsed_spec
-    if _parsed_spec is None:
-        spec_dict = 
yaml.load(open(pkg_resources.resource_filename(api.__name__, 
'swagger.yml')).read())
-        _parsed_spec = spec.Spec.from_dict(spec_dict)
-
-
-def parse_and_validate_parameters(raw_params):
-    initialize_specification()
-
-    clean_params = {}
-    for parameter in _parsed_spec.spec_dict['parameters']:
-        parameter_spec = _parsed_spec.spec_dict['parameters'][parameter]
-        parameter_type = parameter_spec['type']
-        parameter_name = parameter_spec['name']
-        try:
-            value = param.cast_request_param(parameter_type, parameter, 
raw_params.get(parameter_name))
-            validate.validate_schema_object(_parsed_spec, parameter_spec, 
value)
-            clean_params[parameter] = 
marshal.marshal_schema_object(_parsed_spec, parameter_spec, value)
-        except Exception as e:
-            raise ValueError(e)
-
-    return clean_params
-
-
-def marshal_response(recommendations):
-    initialize_specification()
-
-    articles_spec = _parsed_spec.spec_dict['definitions']['Articles']
-    articles = marshal.marshal_schema_object(_parsed_spec, articles_spec, 
{'articles': recommendations})
-    return articles
diff --git a/recommendation/api/swagger.yml b/recommendation/api/swagger.yml
deleted file mode 100644
index 4f90111..0000000
--- a/recommendation/api/swagger.yml
+++ /dev/null
@@ -1,101 +0,0 @@
-swagger: '2.0'
-
-info:
-  version: "1"
-  title: Wikimedia Recommendation API
-  description: |
-    This API provides personalized recommendations for a variety
-    of use cases
-  contact:
-    name: '#wikimedia-research'
-    url: http://freenode.net
-
-definitions:
-  Articles:
-    type: object
-    required:
-      - articles
-    properties:
-      articles:
-        type: array
-        items:
-          $ref: '#/definitions/Article'
-  Article:
-    type: object
-    properties:
-      pageviews:
-        type: integer
-      title:
-        type: string
-      wikidata_id:
-        type: string
-
-parameters:
-  source:
-    name: s
-    in: query
-    description: Source wiki project language code
-    required: true
-    type: string
-  target:
-    name: t
-    in: query
-    description: Target wiki project language code
-    required: true
-    type: string
-  count:
-    name: n
-    in: query
-    description: Number of recommendations to fetch
-    required: false
-    type: integer
-    format: int32
-    maximum: 24
-    minimum: 0
-    default: 12
-  seed:
-    name: article
-    in: query
-    description: |
-      Seed article for personalized recommendations that
-      can also be a list separated by "|"'
-    required: false
-    type: string
-    pattern: '^([^|]+(\|[^|]+)*)?$'
-    default: ''
-  include_pageviews:
-    name: pageviews
-    in: query
-    description: Whether to include pageview counts
-    required: false
-    type: boolean
-    default: true
-  search:
-    name: search
-    in: query
-    description: Which search algorithm to use if a seed is specified
-    required: false
-    type: string
-    enum:
-      - morelike
-      - wiki
-    default: morelike
-
-paths:
-  /api:
-    get:
-      description: |
-        Gets `Article` objects of source articles that
-        are missing in the target
-      parameters:
-        - $ref: '#/parameters/source'
-        - $ref: '#/parameters/target'
-        - $ref: '#/parameters/count'
-        - $ref: '#/parameters/seed'
-        - $ref: '#/parameters/include_pageviews'
-        - $ref: '#/parameters/search'
-      responses:
-        '200':
-          description: Successful response
-          schema:
-            $ref: '#/definitions/Articles'
diff --git a/recommendation/api/test/test_api.py 
b/recommendation/api/test/test_api.py
deleted file mode 100644
index d380365..0000000
--- a/recommendation/api/test/test_api.py
+++ /dev/null
@@ -1,131 +0,0 @@
-import flask
-import pytest
-import json
-import urllib.parse
-
-from recommendation.api import api
-from recommendation.api import utils
-from recommendation.api import filters
-
-GOOD_RESPONSE = {'articles': [
-    {'title': 'A', 'pageviews': 10, 'wikidata_id': 123},
-    {'title': 'B', 'pageviews': 11, 'wikidata_id': 122},
-    {'title': 'C', 'pageviews': 12, 'wikidata_id': 121},
-    {'title': 'D', 'pageviews': 13, 'wikidata_id': 120},
-    {'title': 'E', 'pageviews': 14, 'wikidata_id': 119},
-    {'title': 'F', 'pageviews': 15, 'wikidata_id': 118},
-    {'title': 'G', 'pageviews': 16, 'wikidata_id': 117},
-    {'title': 'H', 'pageviews': 17, 'wikidata_id': 116},
-]}
-
-
-def get_query_string(input_dict):
-    return '/?' + urllib.parse.urlencode(input_dict)
-
-
-@pytest.fixture
-def recommend_response(monkeypatch):
-    monkeypatch.setattr(api, 'recommend', lambda *args, **kwargs: 
GOOD_RESPONSE['articles'])
-
-
-@pytest.fixture
-def client():
-    app_instance = flask.Flask(__name__)
-    app_instance.register_blueprint(api.api)
-    return app_instance.test_client()
-
-
-@pytest.mark.parametrize('url', [
-    get_query_string(dict(s='xx', t='yy')),
-    get_query_string(dict(s='xx', t='yy', n=13)),
-    get_query_string(dict(s='xx', t='yy', article='separated|list|of|titles')),
-    get_query_string(dict(s='xx', t='yy', article='Some Article')),
-    get_query_string(dict(s='xx', t='yy', article='')),
-    get_query_string(dict(s='xx', t='yy', pageviews='false')),
-    get_query_string(dict(s='xx', t='yy', search='morelike')),
-    get_query_string(dict(s='xx', t='yy', search='wiki')),
-])
-@pytest.mark.usefixtures('recommend_response')
-def test_good_arg_parsing(client, url):
-    result = client.get(url)
-    assert 200 == result.status_code
-    assert GOOD_RESPONSE == json.loads(result.data.decode('utf-8'))
-
-
-@pytest.mark.parametrize('url', [
-    get_query_string(dict(s='xx')),
-    get_query_string(dict(t='xx')),
-    '/',
-    get_query_string(dict(s='xx', t='xx')),
-    get_query_string(dict(s='xx', t='yy', n=-1)),
-    get_query_string(dict(s='xx', t='yy', n=25)),
-    get_query_string(dict(s='xx', t='yy', n='not a number')),
-    get_query_string(dict(s='xx', t='yy', article='||||||||||||')),
-    get_query_string(dict(s='xx', t='yy', pageviews='not a boolean')),
-    get_query_string(dict(s='xx', t='yy', search='not a valid search')),
-])
-@pytest.mark.usefixtures('recommend_response')
-def test_bad_args(client, url):
-    result = client.get(url)
-    assert 'error' in json.loads(result.data.decode('utf-8'))
-
-
-@pytest.mark.parametrize('params', [
-    dict(s='xx', t='yy'),
-])
-def test_default_params(params):
-    args = api.parse_and_validate_args(params)
-    assert 12 == args['count']
-    assert '' is args['seed']
-    assert True is args['include_pageviews']
-    assert 'morelike' == args['search']
-
-
-def test_recommend(monkeypatch):
-    class MockFinder:
-        @classmethod
-        def get_candidates(cls, s, seed, n):
-            return []
-
-    monkeypatch.setattr(api, 'finder_map', {'customsearch': MockFinder})
-    args = api.parse_and_validate_args(dict(s='xx', t='yy', 
article='Something'))
-    args['search'] = 'customsearch'
-    result = api.recommend(**args)
-    assert [] == result
-
-
-def test_recommend_uses_mostpopular_if_no_seed_is_specified(monkeypatch):
-    class MockFinder:
-        @classmethod
-        def get_candidates(cls, s, seed, n):
-            return []
-
-    monkeypatch.setattr(api, 'finder_map', {'mostpopular': MockFinder})
-    args = api.parse_and_validate_args(dict(s='xx', t='yy'))
-    args['search'] = 'customsearch'
-    result = api.recommend(**args)
-    assert [] == result
-
-
-def test_generated_recommend_response_is_marshalled(client, monkeypatch):
-    class MockFinder:
-        @classmethod
-        def get_candidates(cls, s, seed, n):
-            articles = []
-            for item in GOOD_RESPONSE['articles']:
-                article = utils.Article(item['title'])
-                article.pageviews = item['pageviews']
-                article.wikidata_id = item['wikidata_id']
-                article.rank = article.pageviews
-                articles.append(article)
-            return articles
-    monkeypatch.setattr(api, 'finder_map', {'mostpopular': MockFinder})
-    monkeypatch.setattr(filters, 'apply_filters', lambda source, target, recs, 
count: recs)
-    result = client.get(get_query_string(dict(s='xx', t='yy', 
pageviews=False)))
-    assert GOOD_RESPONSE == json.loads(result.data.decode('utf-8'))
-
-
-@pytest.mark.usefixtures('recommend_response')
-def test_cors_is_present(client):
-    result = client.get(get_query_string(dict(s='xx', t='yy')))
-    assert '*' == result.headers.get('Access-Control-Allow-Origin')
diff --git a/recommendation/api/types/__init__.py 
b/recommendation/api/types/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/recommendation/api/types/__init__.py
diff --git a/recommendation/api/types/translation/__init__.py 
b/recommendation/api/types/translation/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/recommendation/api/types/translation/__init__.py
diff --git a/recommendation/api/candidate_finders.py 
b/recommendation/api/types/translation/candidate_finders.py
similarity index 96%
rename from recommendation/api/candidate_finders.py
rename to recommendation/api/types/translation/candidate_finders.py
index 8095588..4f4560b 100644
--- a/recommendation/api/candidate_finders.py
+++ b/recommendation/api/types/translation/candidate_finders.py
@@ -2,9 +2,9 @@
 import datetime
 import logging
 
-from recommendation.api.utils import Article
+from recommendation.api.types.translation.utils import Article
 from recommendation.utils import configuration
-from recommendation.api import data_fetcher
+from recommendation.api.types.translation import data_fetcher
 
 log = logging.getLogger(__name__)
 
diff --git a/recommendation/api/data_fetcher.py 
b/recommendation/api/types/translation/data_fetcher.py
similarity index 100%
rename from recommendation/api/data_fetcher.py
rename to recommendation/api/types/translation/data_fetcher.py
diff --git a/recommendation/api/filters.py 
b/recommendation/api/types/translation/filters.py
similarity index 96%
rename from recommendation/api/filters.py
rename to recommendation/api/types/translation/filters.py
index 0daf039..3261192 100644
--- a/recommendation/api/filters.py
+++ b/recommendation/api/types/translation/filters.py
@@ -1,6 +1,6 @@
 import logging
 
-from recommendation.api import data_fetcher
+from recommendation.api.types.translation import data_fetcher
 
 log = logging.getLogger(__name__)
 
diff --git a/recommendation/api/pageviews.py 
b/recommendation/api/types/translation/pageviews.py
similarity index 89%
rename from recommendation/api/pageviews.py
rename to recommendation/api/types/translation/pageviews.py
index 9f04503..7b923f9 100644
--- a/recommendation/api/pageviews.py
+++ b/recommendation/api/types/translation/pageviews.py
@@ -1,6 +1,6 @@
 import concurrent.futures
 
-from recommendation.api import data_fetcher
+from recommendation.api.types.translation import data_fetcher
 
 
 def set_pageview_data(source, articles):
diff --git a/recommendation/api/test/conftest.py 
b/recommendation/api/types/translation/test/conftest.py
similarity index 100%
rename from recommendation/api/test/conftest.py
rename to recommendation/api/types/translation/test/conftest.py
diff --git a/recommendation/api/types/translation/test/test_api.py 
b/recommendation/api/types/translation/test/test_api.py
new file mode 100644
index 0000000..be1af9c
--- /dev/null
+++ b/recommendation/api/types/translation/test/test_api.py
@@ -0,0 +1,140 @@
+import flask
+import pytest
+import json
+import urllib.parse
+
+from recommendation.api.types.translation import translation
+from recommendation.api.types.translation import utils
+from recommendation.api.types.translation import filters
+
+GOOD_RESPONSE = [
+    {'title': 'A', 'pageviews': 10, 'wikidata_id': '123'},
+    {'title': 'B', 'pageviews': 11, 'wikidata_id': '122'},
+    {'title': 'C', 'pageviews': 12, 'wikidata_id': '121'},
+    {'title': 'D', 'pageviews': 13, 'wikidata_id': '120'},
+    {'title': 'E', 'pageviews': 14, 'wikidata_id': '119'},
+    {'title': 'F', 'pageviews': 15, 'wikidata_id': '118'},
+    {'title': 'G', 'pageviews': 16, 'wikidata_id': '117'},
+    {'title': 'H', 'pageviews': 17, 'wikidata_id': '116'},
+]
+
+
+@pytest.fixture(params=[
+    '/types/translation/v1/articles',
+    '/api/'
+])
+def get_url(request):
+    return lambda input_dict: request.param + '?' + 
urllib.parse.urlencode(input_dict)
+
+
+@pytest.fixture
+def recommend_response(monkeypatch):
+    monkeypatch.setattr(translation, 'recommend', lambda *args, **kwargs: 
GOOD_RESPONSE)
+
+
+@pytest.fixture
+def client():
+    app_instance = flask.Flask(__name__)
+    app_instance.register_blueprint(translation.api.blueprint)
+    app_instance.register_blueprint(translation.legacy.blueprint)
+    app_instance.testing = True
+    return app_instance.test_client()
+
+
+@pytest.mark.parametrize('params', [
+    dict(s='xx', t='yy'),
+    dict(s='xx', t='yy', n=13),
+    dict(s='xx', t='yy', article='separated|list|of|titles'),
+    dict(s='xx', t='yy', article='Some Article'),
+    dict(s='xx', t='yy', article=''),
+    dict(s='xx', t='yy', pageviews='false'),
+    dict(s='xx', t='yy', search='morelike'),
+    dict(s='xx', t='yy', search='wiki'),
+])
+@pytest.mark.usefixtures('recommend_response')
+def test_good_arg_parsing(client, get_url, params):
+    result = client.get(get_url(params))
+    assert 200 == result.status_code
+    assert GOOD_RESPONSE == json.loads(result.data.decode('utf-8'))
+
+
+@pytest.mark.parametrize('value,expected', [
+    ('false', False),
+    ('true', True),
+    ('False', False),
+    ('True', True),
+    (1, True),
+    (0, False)
+])
+def test_boolean_arg_parsing(client, get_url, value, expected):
+    with client as c:
+        c.get(get_url(dict(s='xx', t='yy', pageviews=value)))
+        args = translation.legacy_params.parse_args()
+    assert expected is args['include_pageviews']
+
+
+@pytest.mark.parametrize('params', [
+    dict(s='xx'),
+    dict(t='xx'),
+    {},
+    dict(s='xx', t='xx'),
+    dict(s='xx', t='yy', n=-1),
+    dict(s='xx', t='yy', n=25),
+    dict(s='xx', t='yy', n='not a number'),
+    dict(s='xx', t='yy', article='||||||||||||'),
+    dict(s='xx', t='yy', pageviews='not a boolean'),
+    dict(s='xx', t='yy', search='not a valid search'),
+])
+@pytest.mark.usefixtures('recommend_response')
+def test_bad_args(client, get_url, params):
+    result = client.get(get_url(params))
+    assert 'errors' in json.loads(result.data.decode('utf-8'))
+
+
+@pytest.mark.parametrize('params', [
+    dict(s='xx', t='yy'),
+])
+def test_default_params(client, get_url, params):
+    with client as c:
+        c.get(get_url(params))
+        args = translation.legacy_params.parse_args()
+    assert 12 == args['count']
+    assert None is args['seed']
+    assert True is args['include_pageviews']
+    assert 'morelike' == args['search']
+
+
+def test_recommend_uses_mostpopular_if_no_seed_is_specified(monkeypatch):
+    class MockFinder:
+        @classmethod
+        def get_candidates(cls, s, seed, n):
+            return []
+
+    monkeypatch.setattr(translation, 'finder_map', {'mostpopular': MockFinder})
+    result = translation.recommend(source='xx', target='yy', 
search='customsearch', seed=None, count=12,
+                                   include_pageviews=True)
+    assert [] == result
+
+
+def test_generated_recommend_response_is_marshalled(client, get_url, 
monkeypatch):
+    class MockFinder:
+        @classmethod
+        def get_candidates(cls, s, seed, n):
+            articles = []
+            for item in GOOD_RESPONSE:
+                article = utils.Article(item['title'])
+                article.pageviews = item['pageviews']
+                article.wikidata_id = item['wikidata_id']
+                article.rank = article.pageviews
+                articles.append(article)
+            return articles
+    monkeypatch.setattr(translation, 'finder_map', {'mostpopular': MockFinder})
+    monkeypatch.setattr(filters, 'apply_filters', lambda source, target, recs, 
count: recs)
+    result = client.get(get_url(dict(s='xx', t='yy', pageviews=False)))
+    assert GOOD_RESPONSE == json.loads(result.data.decode('utf-8'))
+
+
+@pytest.mark.usefixtures('recommend_response')
+def test_cors_is_present(client, get_url):
+    result = client.get(get_url(dict(s='xx', t='yy')))
+    assert '*' == result.headers.get('Access-Control-Allow-Origin')
diff --git a/recommendation/api/test/test_candidate_finders.py 
b/recommendation/api/types/translation/test/test_candidate_finders.py
similarity index 98%
rename from recommendation/api/test/test_candidate_finders.py
rename to recommendation/api/types/translation/test/test_candidate_finders.py
index a349054..cfa9492 100644
--- a/recommendation/api/test/test_candidate_finders.py
+++ b/recommendation/api/types/translation/test/test_candidate_finders.py
@@ -5,7 +5,7 @@
 import json
 import re
 
-from recommendation.api import candidate_finders
+from recommendation.api.types.translation import candidate_finders
 
 PAGEVIEW_RESPONSE = {
     'items': [
diff --git a/recommendation/api/test/test_filters.py 
b/recommendation/api/types/translation/test/test_filters.py
similarity index 94%
rename from recommendation/api/test/test_filters.py
rename to recommendation/api/types/translation/test/test_filters.py
index d422dce..4a28ba7 100644
--- a/recommendation/api/test/test_filters.py
+++ b/recommendation/api/types/translation/test/test_filters.py
@@ -2,9 +2,9 @@
 import responses
 import re
 
-from recommendation.api import filters
+from recommendation.api.types.translation import filters
 from recommendation.utils import configuration
-from recommendation.api import utils
+from recommendation.api.types.translation import utils
 
 SOURCE = 'xx'
 
diff --git a/recommendation/api/test/test_pageviews.py 
b/recommendation/api/types/translation/test/test_pageviews.py
similarity index 92%
rename from recommendation/api/test/test_pageviews.py
rename to recommendation/api/types/translation/test/test_pageviews.py
index 9fe0a0e..b9e9055 100644
--- a/recommendation/api/test/test_pageviews.py
+++ b/recommendation/api/types/translation/test/test_pageviews.py
@@ -3,9 +3,9 @@
 import datetime
 import re
 
-from recommendation.api import pageviews
-from recommendation.api import data_fetcher
-from recommendation.api import utils
+from recommendation.api.types.translation import pageviews
+from recommendation.api.types.translation import data_fetcher
+from recommendation.api.types.translation import utils
 from recommendation.utils import configuration
 
 TITLE = 'Sample_Title'
diff --git a/recommendation/api/test/test_recommendation.ini 
b/recommendation/api/types/translation/test/test_recommendation.ini
similarity index 100%
rename from recommendation/api/test/test_recommendation.ini
rename to recommendation/api/types/translation/test/test_recommendation.ini
diff --git a/recommendation/api/types/translation/translation.py 
b/recommendation/api/types/translation/translation.py
new file mode 100644
index 0000000..a58ab0e
--- /dev/null
+++ b/recommendation/api/types/translation/translation.py
@@ -0,0 +1,156 @@
+import collections
+import logging
+import time
+
+import flask_restplus
+from flask_restplus import fields
+from flask_restplus import reqparse
+from flask_restplus import abort
+from flask_restplus import inputs
+
+from recommendation.utils import event_logger
+from recommendation.utils import language_pairs
+from recommendation.api import helper
+from recommendation.api.types.translation import filters
+from recommendation.api.types.translation import candidate_finders
+from recommendation.api.types.translation import pageviews
+
+log = logging.getLogger(__name__)
+
+legacy = helper.build_api('legacy', __name__, url_prefix='/api')
+
+ArticleSpec = collections.namedtuple('Article', ['pageviews', 'title', 
'wikidata_id'])
+
+legacy_params = reqparse.RequestParser()
+
+legacy_params.add_argument(
+    's',
+    type=str,
+    dest='source',
+    required=True)
+legacy_params.add_argument(
+    't',
+    type=str,
+    dest='target',
+    required=True)
+legacy_params.add_argument(
+    'n',
+    type=inputs.int_range(low=0, high=24),
+    dest='count',
+    required=False,
+    default=12)
+legacy_params.add_argument(
+    'article',
+    type=inputs.regex(r'^([^|]+(\|[^|]+)*)?$'),
+    dest='seed',
+    required=False)
+legacy_params.add_argument(
+    'pageviews',
+    type=inputs.boolean,
+    dest='include_pageviews',
+    required=False,
+    default=True)
+legacy_params.add_argument(
+    'search',
+    type=str,
+    required=False,
+    default='morelike',
+    choices=['morelike', 'wiki'])
+
+legacy_model = legacy.model(ArticleSpec.__name__, ArticleSpec(
+    pageviews=fields.Integer(description='pageviews', required=False),
+    title=fields.String(description='title', required=True),
+    wikidata_id=fields.String(description='wikidata_id', required=True)
+)._asdict())
+
+legacy_doc = dict(description='Gets recommendations of source articles that 
are missing in the target',
+                  params=dict(s='Source wiki project language code',
+                              t='Target wiki project language code',
+                              n='Number of recommendations to fetch',
+                              article='Seed article for personalized 
recommendations '
+                                      'that can also be a list separated by 
"|"',
+                              pageviews='Whether to include pageview counts',
+                              search='Which search algorithm to use if a seed 
is specified')
+                  )
+
+
+@legacy.deprecated
+@legacy.route('/')
+class LegacyArticle(flask_restplus.Resource):
+    @legacy.expect(legacy_params)
+    @legacy.marshal_with(legacy_model, as_list=True)
+    @legacy.doc(**legacy_doc)
+    def get(self):
+        args = legacy_params.parse_args()
+        return process_request(args)
+
+
+api = helper.build_api('translation', __name__, 
url_prefix='/types/translation')
+v1 = helper.build_namespace(api, 'v1', description='')
+v1_params = legacy_params.copy()
+v1_model = legacy.clone(ArticleSpec.__name__, legacy_model)
+v1_doc = legacy_doc.copy()
+
+
+@v1.route('/articles')
+class Article(flask_restplus.Resource):
+    @v1.expect(v1_params)
+    @v1.marshal_with(v1_model, as_list=True)
+    @v1.doc(**v1_doc)
+    def get(self):
+        args = v1_params.parse_args()
+        return process_request(args)
+
+
+article_model = v1.model(ArticleSpec.__name__, ArticleSpec(
+    pageviews=fields.Integer(description='pageviews', required=False),
+    title=fields.String(description='title', required=True),
+    wikidata_id=fields.String(description='wikidata_id', required=True)
+)._asdict())
+
+
+def process_request(args):
+    t1 = time.time()
+
+    if not language_pairs.is_valid_language_pair(args['source'], 
args['target']):
+        abort(400, errors='Invalid or duplicate source and/or target language')
+
+    event_logger.log_api_request(**args)
+    recs = recommend(**args)
+    t2 = time.time()
+    log.info('Request processed in %f seconds', t2 - t1)
+    return recs
+
+
+finder_map = {
+    'morelike': candidate_finders.MorelikeCandidateFinder(),
+    'wiki': candidate_finders.MorelikeCandidateFinder(),
+    'mostpopular': candidate_finders.PageviewCandidateFinder(),
+}
+
+
+def recommend(source, target, search, seed, count, include_pageviews, 
max_candidates=500):
+    """
+    1. Use finder to select a set of candidate articles
+    2. Filter out candidates that are not missing, are disambiguation pages, 
etc
+    3. get pageview info for each passing candidate if desired
+    """
+
+    recs = []
+
+    if seed:
+        finder = finder_map[search]
+        for seed in seed.split('|'):
+            recs.extend(finder.get_candidates(source, seed, max_candidates))
+    else:
+        recs.extend(finder_map['mostpopular'].get_candidates(source, seed, 
max_candidates))
+
+    recs = sorted(recs, key=lambda x: x.rank)
+
+    recs = filters.apply_filters(source, target, recs, count)
+
+    if recs and include_pageviews:
+        recs = pageviews.set_pageview_data(source, recs)
+
+    recs = sorted(recs, key=lambda x: x.rank)
+    return [{'title': r.title, 'pageviews': r.pageviews, 'wikidata_id': 
r.wikidata_id} for r in recs]
diff --git a/recommendation/api/utils.py 
b/recommendation/api/types/translation/utils.py
similarity index 100%
rename from recommendation/api/utils.py
rename to recommendation/api/types/translation/utils.py
diff --git a/recommendation/data/recommendation.wsgi 
b/recommendation/data/recommendation.wsgi
index 88d2455..f7a6579 100644
--- a/recommendation/data/recommendation.wsgi
+++ b/recommendation/data/recommendation.wsgi
@@ -1,5 +1,6 @@
 from flask import Flask
 
+from recommendation.api.types.translation import translation
 from recommendation.api import api
 from recommendation.web import gapfinder
 from recommendation.utils import logger
@@ -7,8 +8,12 @@
 logger.initialize_logging()
 
 app = Flask(__name__)
-app.register_blueprint(api.api, url_prefix='/api')
-app.register_blueprint(gapfinder.gapfinder)
+app.register_blueprint(api.api.blueprint)
+app.register_blueprint(translation.api.blueprint)
+app.register_blueprint(translation.legacy.blueprint)
+app.register_blueprint(gapfinder.gapfinder, url_prefix='/tool')
+app.config['RESTPLUS_VALIDATE'] = True
+app.config['RESTPLUS_MASK_SWAGGER'] = False
 application = app
 
 if __name__ == '__main__':
diff --git a/recommendation/test/conftest.py b/recommendation/test/conftest.py
index ed46945..2666af4 100644
--- a/recommendation/test/conftest.py
+++ b/recommendation/test/conftest.py
@@ -14,7 +14,7 @@
      to apply a decorator to every test function
     """
     configuration._config = configuration.get_configuration('', 
recommendation.__name__,
-                                                            
'api/test/test_recommendation.ini')
+                                                            
'api/types/translation/test/test_recommendation.ini')
     logger.initialize_logging()
     responses._default_mock.__enter__()
 
diff --git a/recommendation/web/static/gf-input.tag 
b/recommendation/web/static/gf-input.tag
index 04c574e..3bee49d 100644
--- a/recommendation/web/static/gf-input.tag
+++ b/recommendation/web/static/gf-input.tag
@@ -66,7 +66,7 @@
 
             var mappedSource = self.mapLanguageToDomainCode(self.source);
             var mappedTarget = self.mapLanguageToDomainCode(self.target);
-            var url = '/api/?s=' + mappedSource + '&t=' + mappedTarget;
+            var url = translationAppGlobals.translationPath + '?s=' + 
mappedSource + '&t=' + mappedTarget;
 
             var seed;
             if (this.seedArticle.value) {
@@ -88,7 +88,7 @@
                     return;
                 }
 
-                var articles = self.filter(data.articles);
+                var articles = self.filter(data);
                 if (!articles || !articles.length) {
                     self.error_msg = articles['error'];
                     self.error = true;
diff --git a/recommendation/web/templates/index.html 
b/recommendation/web/templates/index.html
index cd1c0bd..60d550f 100644
--- a/recommendation/web/templates/index.html
+++ b/recommendation/web/templates/index.html
@@ -69,7 +69,9 @@
                 eventLoggerUrl: '{{ event_logger_url }}',
 
                 i18nGapFinderPath: '{{ url_for('.static', filename='i18n/') 
}}',
-                i18nUlsPath: '{{ url_for('.resource', 
filename='bower_components/uls/i18n/') }}'
+                i18nUlsPath: '{{ url_for('.resource', 
filename='bower_components/uls/i18n/') }}',
+
+                translationPath: '{{ url_for('translation.v1_article') }}'
             };
 
             riot.mount('gf-disclaimer, gf-title, gf-input');
diff --git a/setup.py b/setup.py
index 68a5edc..de732fa 100644
--- a/setup.py
+++ b/setup.py
@@ -12,8 +12,7 @@
     long_description='',
     packages=find_packages(exclude=['test', 'test.*', '*.test']),
     install_requires=['flask',
-                      'bravado-core',
-                      'PyYAML',
+                      'flask-restplus',
                       'requests',
                       'numpy'],
     package_data={'recommendation.web': ['static/*.*',

-- 
To view, visit https://gerrit.wikimedia.org/r/316557
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I0619639a11ddda8925dac11e337e6358932586a3
Gerrit-PatchSet: 3
Gerrit-Project: research/recommendation-api
Gerrit-Branch: master
Gerrit-Owner: Nschaaf <nsch...@wikimedia.org>
Gerrit-Reviewer: Nschaaf <nsch...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to