Colin Watson has proposed merging ~cjwatson/lp-archive:translate-paths into lp-archive:main.
Commit message: Add archive endpoints Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~cjwatson/lp-archive/+git/lp-archive/+merge/431569 If configured with a suitable Launchpad XML-RPC endpoint, this can serve apt archives based on publishing records and the librarian without needing access to the published archive on a local file system. -- Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/lp-archive:translate-paths into lp-archive:main.
diff --git a/.gitignore b/.gitignore index af6421f..2c2f17b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /.python-version /.tox /build +/config.toml /env /htmlcov /tmp diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 0000000..07a8171 --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,5 @@ +[mypy] +python_version = 3.10 + +[mypy-tomllib.*] +ignore_missing_imports = true diff --git a/lp_archive/__init__.py b/lp_archive/__init__.py index 63a22b1..83c12ec 100644 --- a/lp_archive/__init__.py +++ b/lp_archive/__init__.py @@ -3,16 +3,25 @@ """The Launchpad archive service.""" +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib # type: ignore from typing import Any from flask import Flask -from lp_archive import root +from lp_archive import archive, root, routing def create_app(test_config: dict[str, Any] | None = None) -> Flask: app = Flask(__name__) - if test_config is not None: + if test_config is None: + with open("config.toml", "rb") as f: + app.config.from_mapping(tomllib.load(f)) + else: app.config.from_mapping(test_config) + app.url_map.converters["archive"] = routing.ArchiveConverter app.register_blueprint(root.bp) + app.register_blueprint(archive.bp) return app diff --git a/lp_archive/archive.py b/lp_archive/archive.py new file mode 100644 index 0000000..5b2a683 --- /dev/null +++ b/lp_archive/archive.py @@ -0,0 +1,79 @@ +# Copyright 2022 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""The main archive view.""" + +from xmlrpc.client import Fault, ServerProxy + +from flask import Blueprint, current_app, g, request +from werkzeug.datastructures import WWWAuthenticate +from werkzeug.exceptions import Unauthorized + +bp = Blueprint("archive", __name__) + + +def get_archive_proxy() -> ServerProxy: + archive_proxy = getattr(g, "archive_proxy", None) + if archive_proxy is None: + archive_proxy = ServerProxy( + current_app.config["ARCHIVE_ENDPOINT"], allow_none=True + ) + g.archive_proxy = archive_proxy + return archive_proxy + + +def check_auth(archive: str) -> None: + """Check whether the current request may access an archive. + + If unauthorized, raises a suitable exception. + """ + # Ideally we'd use a Flask extension for this rather than rolling most + # of the HTTP Basic Authentication logic for ourselves, but nothing + # seems to be quite suitable. In particular, `flask-httpauth` doesn't + # currently support passing additional data from the route through to + # `verify_password`. + if request.authorization is None: + username = None + password = None + log_prefix = f"<anonymous>@{archive}" + else: + username = request.authorization.username + password = request.authorization.password + log_prefix = f"{username}@{archive}" + try: + # XXX cjwatson 2022-10-14: We should cache positive responses (maybe + # using `flask-caching`) for a while to reduce database load. + get_archive_proxy().checkArchiveAuthToken(archive, username, password) + except Fault as e: + if e.faultCode == 410: # Unauthorized + current_app.logger.info("%s: Password does not match.", log_prefix) + else: + # Interpret any other fault as NotFound (320). + current_app.logger.info("%s: %s", log_prefix, e.faultString) + basic = WWWAuthenticate() + basic.set_basic() + raise Unauthorized(www_authenticate=basic) + else: + current_app.logger.info("%s: Authorized.", log_prefix) + + +# The exact details of the URLs used here should be regarded as a proof of +# concept for now. +@bp.route("/<archive:archive>/<path:path>") +def translate(archive: str, path: str) -> tuple[str, int, dict[str, str]]: + check_auth(archive) + try: + url = get_archive_proxy().translatePath(archive, path) + except Fault as f: + if f.faultCode == 320: # NotFound + return "Not found", 404, {"Content-Type": "text/plain"} + else: + return "Internal server error", 500, {"Content-Type": "text/plain"} + assert isinstance(url, str) + return "", 307, {"Location": url} + + +@bp.after_request +def add_headers(response): + response.headers["Vary"] = "Authorization" + return response diff --git a/lp_archive/routing.py b/lp_archive/routing.py new file mode 100644 index 0000000..a3be94d --- /dev/null +++ b/lp_archive/routing.py @@ -0,0 +1,23 @@ +# Copyright 2022 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""Routing helpers.""" + +from werkzeug.routing import BaseConverter + + +class ArchiveConverter(BaseConverter): + """Match an archive reference. + + See `lp.soyuz.model.archive.Archive.reference` in Launchpad. + """ + + # This doesn't currently support the partner/copy archive reference + # syntax (distribution/archive), since it's hard to avoid that being + # ambiguous when parsing URLs (compare with the primary archive + # reference syntax). + # + # PPA: ~[^/]+/[^/]+/[^/]+ (~owner/distribution/archive) + # Primary: [^~+][^/]* (distribution) + regex = r"~[^/]+/[^/]+/[^/]+|[^~+][^/]*" + part_isolating = False diff --git a/requirements.in b/requirements.in index e3e9a71..da656c4 100644 --- a/requirements.in +++ b/requirements.in @@ -1 +1,2 @@ Flask +tomli; python_version < "3.11" diff --git a/requirements.txt b/requirements.txt index 27acd36..73d76c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,5 +16,7 @@ markupsafe==2.1.1 # via # jinja2 # werkzeug +tomli==2.0.1 ; python_version < "3.11" + # via -r requirements.in werkzeug==2.2.2 # via flask diff --git a/setup.cfg b/setup.cfg index 8410614..f54019b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,6 +18,7 @@ classifiers = packages = find: install_requires = Flask + tomli;python_version < "3.11" python_requires = >=3.10 [options.extras_require] diff --git a/tests/test_archive.py b/tests/test_archive.py new file mode 100644 index 0000000..22bd8e1 --- /dev/null +++ b/tests/test_archive.py @@ -0,0 +1,121 @@ +# Copyright 2022 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +from threading import Thread +from typing import Any +from xmlrpc.client import Fault +from xmlrpc.server import SimpleXMLRPCServer + +import pytest + + +class ArchiveXMLRPCServer(SimpleXMLRPCServer): + + path_map = {"dists/focal/InRelease": "http://librarian.example.org/1"} + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.call_log = [] + self.register_function( + self.check_archive_auth_token, name="checkArchiveAuthToken" + ) + self.register_function(self.translate_path, name="translatePath") + + def check_archive_auth_token(self, archive, username, password): + self.call_log.append( + ("checkArchiveAuthToken", archive, username, password) + ) + if archive.endswith("/private"): + raise Fault(410, "Authorization required") + elif archive.endswith("/nonexistent"): + raise Fault(320, "Not found") + else: + return True + + def translate_path(self, archive, path): + # See `lp.xmlrpc.faults` in Launchpad for fault codes. + self.call_log.append(("translatePath", archive, path)) + if path == "oops": + raise Fault(380, "Oops") + elif path in self.path_map: + return self.path_map[path] + else: + raise Fault(320, "Not found") + + +@pytest.fixture +def archive_proxy(app): + with ArchiveXMLRPCServer(("127.0.0.1", 0)) as server: + host, port = server.server_address + app.config.update({"ARCHIVE_ENDPOINT": f"http://{host}:{port}"}) + thread = Thread(target=server.serve_forever) + thread.start() + yield server + server.shutdown() + thread.join() + + +def test_auth_failed(client, archive_proxy): + response = client.get( + "/~user/ubuntu/private/dists/focal/InRelease", auth=("user", "secret") + ) + assert response.status_code == 401 + assert ( + response.headers["WWW-Authenticate"] + == 'Basic realm="authentication required"' + ) + assert response.headers["Vary"] == "Authorization" + assert archive_proxy.call_log == [ + ("checkArchiveAuthToken", "~user/ubuntu/private", "user", "secret") + ] + + +def test_auth_not_found(client, archive_proxy): + response = client.get( + "/~user/ubuntu/nonexistent/dists/focal/InRelease", + auth=("user", "secret"), + ) + assert response.status_code == 401 + assert ( + response.headers["WWW-Authenticate"] + == 'Basic realm="authentication required"' + ) + assert response.headers["Vary"] == "Authorization" + assert archive_proxy.call_log == [ + ("checkArchiveAuthToken", "~user/ubuntu/nonexistent", "user", "secret") + ] + + +def test_translate(client, archive_proxy): + response = client.get("/ubuntu/dists/focal/InRelease") + assert response.status_code == 307 + assert response.headers["Location"] == "http://librarian.example.org/1" + assert response.headers["Vary"] == "Authorization" + assert archive_proxy.call_log == [ + ("checkArchiveAuthToken", "ubuntu", None, None), + ("translatePath", "ubuntu", "dists/focal/InRelease"), + ] + + +def test_translate_not_found(client, archive_proxy): + response = client.get("/ubuntu/nonexistent") + assert response.status_code == 404 + assert response.headers["Content-Type"] == "text/plain" + assert response.headers["Vary"] == "Authorization" + assert response.data == b"Not found" + assert archive_proxy.call_log == [ + ("checkArchiveAuthToken", "ubuntu", None, None), + ("translatePath", "ubuntu", "nonexistent"), + ] + + +def test_translate_oops(client, archive_proxy): + response = client.get("/ubuntu/oops") + assert response.status_code == 500 + assert response.headers["Content-Type"] == "text/plain" + assert response.headers["Vary"] == "Authorization" + assert response.data == b"Internal server error" + assert archive_proxy.call_log == [ + ("checkArchiveAuthToken", "ubuntu", None, None), + ("translatePath", "ubuntu", "oops"), + ] diff --git a/tests/test_routing.py b/tests/test_routing.py new file mode 100644 index 0000000..e11ae4e --- /dev/null +++ b/tests/test_routing.py @@ -0,0 +1,41 @@ +# Copyright 2022 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +from flask import url_for + + +def test_primary(app, client): + @app.route("/+test/<archive:archive>") + def index(archive): + return archive + + response = client.get("/+test/ubuntu") + assert response.status_code == 200 + assert response.data == b"ubuntu" + + with app.test_request_context(): + assert url_for("index", archive="ubuntu") == "/+test/ubuntu" + + +def test_ppa(app, client): + @app.route("/+test/<archive:archive>") + def index(archive): + return archive + + response = client.get("/+test/~owner/ubuntu/ppa") + assert response.status_code == 200 + assert response.data == b"~owner/ubuntu/ppa" + + with app.test_request_context(): + assert ( + url_for("index", archive="~owner/ubuntu/ppa") + == "/+test/~owner/ubuntu/ppa" + ) + + +def test_invalid_archive(app, client): + @app.route("/+test/<archive:archive>") + def index(archive): # pragma: no cover + return archive + + assert client.get("/+test/~owner/ubuntu").status_code == 404
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : launchpad-reviewers@lists.launchpad.net Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp