Alvaro Crespo Serrano has proposed merging
~alvarocs/launchpad:fetch-service-policy-charms into launchpad:master.
Commit message:
Add support for fetch service policy configuration for Charm builds
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~alvarocs/launchpad/+git/launchpad/+merge/485980
This also includes changes in
https://code.launchpad.net/~alvarocs/launchpad/+git/launchpad/+merge/485867
which adds 'use_fetch_service' flag, as I rebased on it to add the
'fetch_service_policy'
Changes:
- Add fetch_service_policy to the CharmRecipe model and interface classes.
- Add fetch_service_policy to the proxy session in charmrecipebuildbehaviour.py
- Update tests with fetch_service_policy, add new test
test_requestFetchServiceSession_permissive.
--
Your team Launchpad code reviewers is requested to review the proposed merge of
~alvarocs/launchpad:fetch-service-policy-charms into launchpad:master.
diff --git a/lib/lp/charms/interfaces/charmrecipe.py b/lib/lp/charms/interfaces/charmrecipe.py
index 9e5c06e..77dcf02 100644
--- a/lib/lp/charms/interfaces/charmrecipe.py
+++ b/lib/lp/charms/interfaces/charmrecipe.py
@@ -74,6 +74,7 @@ from lp.app.errors import NameLookupFailed
from lp.app.interfaces.informationtype import IInformationType
from lp.app.validators.name import name_validator
from lp.app.validators.path import path_does_not_escape
+from lp.buildmaster.builderproxy import FetchServicePolicy
from lp.code.interfaces.gitref import IGitRef
from lp.code.interfaces.gitrepository import IGitRepository
from lp.registry.interfaces.person import IPerson
@@ -771,6 +772,32 @@ class ICharmRecipeAdminAttributes(Interface):
)
)
+ use_fetch_service = exported(
+ Bool(
+ title=_("Use fetch service"),
+ required=True,
+ readonly=False,
+ description=_(
+ "If set, Charm builds will use the fetch-service instead "
+ "of the builder-proxy to access external resources."
+ ),
+ )
+ )
+
+ fetch_service_policy = exported(
+ Choice(
+ title=_("Fetch service policy"),
+ vocabulary=FetchServicePolicy,
+ required=False,
+ readonly=False,
+ default=FetchServicePolicy.STRICT,
+ description=_(
+ "Which policy to use when using the fetch service. Ignored if "
+ "`use_fetch_service` flag is False."
+ ),
+ )
+ )
+
# XXX cjwatson 2021-09-15 bug=760849: "beta" is a lie to get WADL
# generation working. Individual attributes must set their version to
@@ -830,6 +857,8 @@ class ICharmRecipeSet(Interface):
store_secrets=None,
store_channels=None,
date_created=None,
+ use_fetch_service=False,
+ fetch_service_policy=FetchServicePolicy.STRICT,
):
"""Create an `ICharmRecipe`."""
diff --git a/lib/lp/charms/model/charmrecipe.py b/lib/lp/charms/model/charmrecipe.py
index 965206e..82ea9c3 100644
--- a/lib/lp/charms/model/charmrecipe.py
+++ b/lib/lp/charms/model/charmrecipe.py
@@ -44,6 +44,7 @@ from lp.app.enums import (
InformationType,
)
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
+from lp.buildmaster.builderproxy import FetchServicePolicy
from lp.buildmaster.enums import BuildStatus
from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet
from lp.buildmaster.model.builder import Builder
@@ -318,6 +319,15 @@ class CharmRecipe(StormBase, WebhookTargetMixin):
_store_channels = JSON("store_channels", allow_none=True)
+ use_fetch_service = Bool(name="use_fetch_service", allow_none=False)
+
+ fetch_service_policy = DBEnum(
+ enum=FetchServicePolicy,
+ default=FetchServicePolicy.STRICT,
+ name="fetch_service_policy",
+ allow_none=True,
+ )
+
def __init__(
self,
registrant,
@@ -336,6 +346,8 @@ class CharmRecipe(StormBase, WebhookTargetMixin):
store_secrets=None,
store_channels=None,
date_created=DEFAULT,
+ use_fetch_service=False,
+ fetch_service_policy=FetchServicePolicy.STRICT,
):
"""Construct a `CharmRecipe`."""
if not getFeatureFlag(CHARM_RECIPE_ALLOW_CREATE):
@@ -361,6 +373,8 @@ class CharmRecipe(StormBase, WebhookTargetMixin):
self.store_name = store_name
self.store_secrets = store_secrets
self.store_channels = store_channels
+ self.use_fetch_service = use_fetch_service
+ self.fetch_service_policy = fetch_service_policy
def __repr__(self):
return "<CharmRecipe ~%s/%s/+charm/%s>" % (
@@ -957,6 +971,8 @@ class CharmRecipeSet:
store_secrets=None,
store_channels=None,
date_created=DEFAULT,
+ use_fetch_service=False,
+ fetch_service_policy=FetchServicePolicy.STRICT,
):
"""See `ICharmRecipeSet`."""
if not registrant.inTeam(owner):
@@ -1002,6 +1018,8 @@ class CharmRecipeSet:
store_secrets=store_secrets,
store_channels=store_channels,
date_created=date_created,
+ use_fetch_service=use_fetch_service,
+ fetch_service_policy=fetch_service_policy,
)
store.add(recipe)
diff --git a/lib/lp/charms/model/charmrecipebuildbehaviour.py b/lib/lp/charms/model/charmrecipebuildbehaviour.py
index 4405065..1386da3 100644
--- a/lib/lp/charms/model/charmrecipebuildbehaviour.py
+++ b/lib/lp/charms/model/charmrecipebuildbehaviour.py
@@ -79,7 +79,11 @@ class CharmRecipeBuildBehaviour(BuilderProxyMixin, BuildFarmJobBehaviourBase):
"""
build = self.build
args: BuildArgs = yield super().extraBuildArgs(logger=logger)
- yield self.startProxySession(args)
+ yield self.startProxySession(
+ args,
+ use_fetch_service=build.recipe.use_fetch_service,
+ fetch_service_policy=build.recipe.fetch_service_policy,
+ )
args["name"] = build.recipe.store_name or build.recipe.name
channels = build.channels or {}
# We have to remove the security proxy that Zope applies to this
diff --git a/lib/lp/charms/tests/test_charmrecipe.py b/lib/lp/charms/tests/test_charmrecipe.py
index f6bde77..e837e4b 100644
--- a/lib/lp/charms/tests/test_charmrecipe.py
+++ b/lib/lp/charms/tests/test_charmrecipe.py
@@ -38,6 +38,7 @@ from zope.security.proxy import removeSecurityProxy
from lp.app.enums import InformationType
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
+from lp.buildmaster.builderproxy import FetchServicePolicy
from lp.buildmaster.enums import (
BuildBaseImageType,
BuildQueueStatus,
@@ -467,6 +468,21 @@ class TestCharmRecipe(TestCaseWithFactory):
build = recipe.requestBuild(build_request, das)
self.assertEqual(build_virt, build.virtualized)
+ def test_requestBuild_fetch_service(self):
+ # Activate fetch service for a charm recipe.
+ recipe = self.factory.makeCharmRecipe(use_fetch_service=True)
+ self.assertEqual(True, recipe.use_fetch_service)
+ distro_series = self.factory.makeDistroSeries()
+ das = self.makeBuildableDistroArchSeries(
+ distroseries=distro_series,
+ )
+ build_request = self.factory.makeCharmRecipeBuildRequest(recipe=recipe)
+ build = recipe.requestBuild(build_request, das)
+ self.assertEqual(True, build.recipe.use_fetch_service)
+ self.assertEqual(
+ FetchServicePolicy.STRICT, recipe.fetch_service_policy
+ )
+
def test_requestBuild_nonvirtualized(self):
# A non-virtualized processor can build a charm recipe iff the
# recipe has require_virtualized set to False.
@@ -1685,6 +1701,10 @@ class TestCharmRecipeSet(TestCaseWithFactory):
self.assertIsNone(recipe.store_name)
self.assertIsNone(recipe.store_secrets)
self.assertEqual([], recipe.store_channels)
+ self.assertFalse(recipe.use_fetch_service)
+ self.assertEqual(
+ FetchServicePolicy.STRICT, recipe.fetch_service_policy
+ )
def test_creation_no_source(self):
# Attempting to create a charm recipe without a Git repository
@@ -2128,6 +2148,50 @@ class TestCharmRecipeSet(TestCaseWithFactory):
recipe, "date_last_modified", UTC_NOW
)
+ def test_admins_can_update_admin_only_fields(self):
+ # Test the admin fields can be updated by an admin
+
+ [ref] = self.factory.makeGitRefs()
+ charm = self.factory.makeCharmRecipe(
+ git_ref=ref, use_fetch_service=True
+ )
+
+ admin_fields = {
+ "require_virtualized": True,
+ "use_fetch_service": True,
+ "fetch_service_policy": FetchServicePolicy.PERMISSIVE,
+ }
+
+ for field_name, field_value in admin_fields.items():
+ # exception is not raised when an admin does the same
+ with admin_logged_in():
+ setattr(charm, field_name, field_value)
+
+ def test_non_admins_cannot_update_admin_only_fields(self):
+ # The admin fields cannot be updated by a non admin
+
+ [ref] = self.factory.makeGitRefs()
+ charm = self.factory.makeCharmRecipe(
+ git_ref=ref, use_fetch_service=True
+ )
+ person = self.factory.makePerson()
+ admin_fields = {
+ "require_virtualized": True,
+ "use_fetch_service": True,
+ "fetch_service_policy": FetchServicePolicy.PERMISSIVE,
+ }
+
+ for field_name, field_value in admin_fields.items():
+ # exception is raised when a non admin updates the fields
+ with person_logged_in(person):
+ self.assertRaises(
+ Unauthorized,
+ setattr,
+ charm,
+ field_name,
+ field_value,
+ )
+
class TestCharmRecipeWebservice(TestCaseWithFactory):
layer = LaunchpadFunctionalLayer
diff --git a/lib/lp/charms/tests/test_charmrecipebuildbehaviour.py b/lib/lp/charms/tests/test_charmrecipebuildbehaviour.py
index 37b4a8b..3e72fd0 100644
--- a/lib/lp/charms/tests/test_charmrecipebuildbehaviour.py
+++ b/lib/lp/charms/tests/test_charmrecipebuildbehaviour.py
@@ -4,13 +4,15 @@
"""Test charm recipe build behaviour."""
import base64
+import json
import os.path
import time
import uuid
from datetime import datetime
+from unittest.mock import MagicMock
from urllib.parse import urlsplit
-from fixtures import MockPatch
+from fixtures import MockPatch, TempDir
from pymacaroons import Macaroon
from testtools import ExpectedException
from testtools.matchers import (
@@ -34,6 +36,7 @@ from lp.app.enums import InformationType
from lp.archivepublisher.interfaces.archivegpgsigningkey import (
IArchiveGPGSigningKey,
)
+from lp.buildmaster.builderproxy import FetchServicePolicy
from lp.buildmaster.enums import BuildBaseImageType, BuildStatus
from lp.buildmaster.interactor import shut_down_default_process_pool
from lp.buildmaster.interfaces.builder import CannotBuild
@@ -46,6 +49,9 @@ from lp.buildmaster.tests.builderproxy import (
ProxyURLMatcher,
RevocationEndpointMatcher,
)
+from lp.buildmaster.tests.fetchservice import (
+ InProcessFetchServiceAuthAPIFixture,
+)
from lp.buildmaster.tests.mock_workers import (
MockBuilder,
OkWorker,
@@ -561,6 +567,328 @@ class TestAsyncCharmRecipeBuildBehaviour(
)
+class TestAsyncCharmRecipeBuildBehaviourFetchService(
+ StatsMixin,
+ TestCharmRecipeBuildBehaviourBase,
+):
+ run_tests_with = AsynchronousDeferredRunTestForBrokenTwisted.make_factory(
+ timeout=30
+ )
+
+ @defer.inlineCallbacks
+ def setUp(self):
+ super().setUp()
+ self.session = {
+ "id": "1",
+ "token": uuid.uuid4().hex,
+ }
+ self.fetch_service_url = (
+ "http://{session_id}:{token}@{host}:{port}".format(
+ session_id=self.session["id"],
+ token=self.session["token"],
+ host=config.builddmaster.fetch_service_host,
+ port=config.builddmaster.fetch_service_port,
+ )
+ )
+ self.fetch_service_api = self.useFixture(
+ InProcessFetchServiceAuthAPIFixture()
+ )
+ yield self.fetch_service_api.start()
+ self.now = time.time()
+ self.useFixture(MockPatch("time.time", return_value=self.now))
+ self.addCleanup(shut_down_default_process_pool)
+ self.setUpStats()
+
+ def makeJob(self, **kwargs):
+ # We need a builder worker in these tests, in order that requesting
+ # a proxy token can piggyback on its reactor and pool.
+ job = super().makeJob(**kwargs)
+ builder = MockBuilder()
+ builder.processor = job.build.processor
+ worker = self.useFixture(WorkerTestHelpers()).getClientWorker()
+ job.setBuilder(builder, worker)
+ self.addCleanup(worker.pool.closeCachedConnections)
+ return job
+
+ @defer.inlineCallbacks
+ def test_requestFetchServiceSession_unconfigured(self):
+ """Create a charm recipe build request with an incomplete fetch service
+ configuration.
+
+ If `fetch_service_host` is not provided the function will return
+ without populating `proxy_url` and `revocation_endpoint`.
+ """
+ self.pushConfig("builddmaster", fetch_service_host=None)
+ job = self.makeJob(use_fetch_service=True)
+ with dbuser(config.builddmaster.dbuser):
+ args = yield job.extraBuildArgs()
+ self.assertEqual([], self.fetch_service_api.sessions.requests)
+ self.assertNotIn("proxy_url", args)
+ self.assertNotIn("revocation_endpoint", args)
+
+ @defer.inlineCallbacks
+ def test_requestFetchServiceSession_no_certificate(self):
+ """Create a charm recipe build request with an incomplete fetch service
+ configuration.
+
+ If `fetch_service_mitm_certificate` is not provided
+ the function raises a `CannotBuild` error.
+ """
+ self.pushConfig("builddmaster", fetch_service_mitm_certificate=None)
+ job = self.makeJob(use_fetch_service=True)
+ expected_exception_msg = (
+ "fetch_service_mitm_certificate is not configured."
+ )
+ with ExpectedException(CannotBuild, expected_exception_msg):
+ yield job.extraBuildArgs()
+
+ @defer.inlineCallbacks
+ def test_requestFetchServiceSession_no_secret(self):
+ """Create a charm recipe build request with an incomplete fetch service
+ configuration.
+
+ If `fetch_service_control_admin_secret` is not provided
+ the function raises a `CannotBuild` error.
+ """
+ self.pushConfig(
+ "builddmaster", fetch_service_control_admin_secret=None
+ )
+ job = self.makeJob(use_fetch_service=True)
+ expected_exception_msg = (
+ "fetch_service_control_admin_secret is not configured."
+ )
+ with ExpectedException(CannotBuild, expected_exception_msg):
+ yield job.extraBuildArgs()
+
+ @defer.inlineCallbacks
+ def test_requestFetchServiceSession(self):
+ """Create a charm recipe build request with a successful fetch service
+ configuration.
+
+ `proxy_url` and `revocation_endpoint` are correctly populated.
+ """
+ job = self.makeJob(use_fetch_service=True)
+ args = yield job.extraBuildArgs()
+ request_matcher = MatchesDict(
+ {
+ "method": Equals(b"POST"),
+ "uri": Equals(b"/session"),
+ "headers": ContainsDict(
+ {
+ b"Authorization": MatchesListwise(
+ [
+ Equals(
+ b"Basic "
+ + base64.b64encode(
+ b"admin-launchpad.test:admin-secret"
+ )
+ )
+ ]
+ ),
+ b"Content-Type": MatchesListwise(
+ [
+ Equals(b"application/json"),
+ ]
+ ),
+ }
+ ),
+ "json": MatchesDict(
+ {
+ "policy": Equals("strict"),
+ }
+ ),
+ }
+ )
+ self.assertThat(
+ self.fetch_service_api.sessions.requests,
+ MatchesListwise([request_matcher]),
+ )
+ self.assertIn("proxy_url", args)
+ self.assertIn("revocation_endpoint", args)
+ self.assertTrue(args["use_fetch_service"])
+ self.assertIn("secrets", args)
+ self.assertIn("fetch_service_mitm_certificate", args["secrets"])
+ self.assertIn(
+ "fake-cert", args["secrets"]["fetch_service_mitm_certificate"]
+ )
+
+ @defer.inlineCallbacks
+ def test_requestFetchServiceSession_permissive(self):
+ """Create a rock recipe build request with a successful fetch service
+ configuration.
+
+ `proxy_url` and `revocation_endpoint` are correctly populated.
+ """
+ job = self.makeJob(
+ use_fetch_service=True,
+ fetch_service_policy=FetchServicePolicy.PERMISSIVE,
+ )
+ args = yield job.extraBuildArgs()
+ request_matcher = MatchesDict(
+ {
+ "method": Equals(b"POST"),
+ "uri": Equals(b"/session"),
+ "headers": ContainsDict(
+ {
+ b"Authorization": MatchesListwise(
+ [
+ Equals(
+ b"Basic "
+ + base64.b64encode(
+ b"admin-launchpad.test:admin-secret"
+ )
+ )
+ ]
+ ),
+ b"Content-Type": MatchesListwise(
+ [
+ Equals(b"application/json"),
+ ]
+ ),
+ }
+ ),
+ "json": MatchesDict(
+ {
+ "policy": Equals("permissive"),
+ }
+ ),
+ }
+ )
+ self.assertThat(
+ self.fetch_service_api.sessions.requests,
+ MatchesListwise([request_matcher]),
+ )
+ self.assertIn("proxy_url", args)
+ self.assertIn("revocation_endpoint", args)
+ self.assertTrue(args["use_fetch_service"])
+ self.assertIn("secrets", args)
+ self.assertIn("fetch_service_mitm_certificate", args["secrets"])
+ self.assertIn(
+ "fake-cert", args["secrets"]["fetch_service_mitm_certificate"]
+ )
+
+ @defer.inlineCallbacks
+ def test_requestFetchServiceSession_mitm_certficate_redacted(self):
+ """The `fetch_service_mitm_certificate` field in the build arguments
+ is redacted in the build logs."""
+
+ job = self.makeJob(use_fetch_service=True)
+ args = yield job.extraBuildArgs()
+
+ chroot_lfa = self.factory.makeLibraryFileAlias(db_only=True)
+ job.build.distro_arch_series.addOrUpdateChroot(
+ chroot_lfa, image_type=BuildBaseImageType.CHROOT
+ )
+ lxd_lfa = self.factory.makeLibraryFileAlias(db_only=True)
+ job.build.distro_arch_series.addOrUpdateChroot(
+ lxd_lfa, image_type=BuildBaseImageType.LXD
+ )
+ deferred = defer.Deferred()
+ deferred.callback(None)
+ job._worker.sendFileToWorker = MagicMock(return_value=deferred)
+ job._worker.build = MagicMock(return_value=(None, None))
+
+ logger = BufferLogger()
+ yield job.dispatchBuildToWorker(logger)
+
+ # Secrets exist within the arguments
+ self.assertIn(
+ "fake-cert", args["secrets"]["fetch_service_mitm_certificate"]
+ )
+ # But are redacted in the log output
+ self.assertIn(
+ "'fetch_service_mitm_certificate': '<redacted>'",
+ logger.getLogBuffer(),
+ )
+
+ @defer.inlineCallbacks
+ def test_endProxySession(self):
+ """By ending a fetch service session, metadata is retrieved from the
+ fetch service and saved to a file; and call to end the session and
+ removing resources are made.
+ """
+ tem_upload_path = self.useFixture(TempDir()).path
+
+ job = self.makeJob(use_fetch_service=True)
+
+ host = config.builddmaster.fetch_service_host
+ port = config.builddmaster.fetch_service_port
+ session_id = self.fetch_service_api.sessions.session_id
+ revocation_endpoint = (
+ f"http://{host}:{port}/session/{session_id}/token";
+ )
+
+ job._worker.proxy_info = MagicMock(
+ return_value={
+ "revocation_endpoint": revocation_endpoint,
+ "use_fetch_service": True,
+ }
+ )
+ yield job.extraBuildArgs()
+
+ # End the session
+ yield job.endProxySession(upload_path=tem_upload_path)
+
+ # We expect 4 calls made to the fetch service API, in this order
+ self.assertEqual(4, len(self.fetch_service_api.sessions.requests))
+
+ # Request start a session
+ start_session_request = self.fetch_service_api.sessions.requests[0]
+ self.assertEqual(b"POST", start_session_request["method"])
+ self.assertEqual(b"/session", start_session_request["uri"])
+
+ # Request retrieve metadata
+ retrieve_metadata_request = self.fetch_service_api.sessions.requests[1]
+ self.assertEqual(b"GET", retrieve_metadata_request["method"])
+ self.assertEqual(
+ f"/session/{session_id}".encode(), retrieve_metadata_request["uri"]
+ )
+
+ # Request end session
+ end_session_request = self.fetch_service_api.sessions.requests[2]
+ self.assertEqual(b"DELETE", end_session_request["method"])
+ self.assertEqual(
+ f"/session/{session_id}".encode(), end_session_request["uri"]
+ )
+
+ # Request removal of resources
+ remove_resources_request = self.fetch_service_api.sessions.requests[3]
+ self.assertEqual(b"DELETE", remove_resources_request["method"])
+ self.assertEqual(
+ f"/resources/{session_id}".encode(),
+ remove_resources_request["uri"],
+ )
+
+ # The expected file is created in the `tem_upload_path`
+ expected_filename = f"{job.build.build_cookie}_metadata.json"
+ expected_file_path = os.path.join(tem_upload_path, expected_filename)
+ with open(expected_file_path) as f:
+ self.assertEqual(
+ json.dumps(self.fetch_service_api.sessions.responses[1]),
+ f.read(),
+ )
+
+ @defer.inlineCallbacks
+ def test_endProxySession_fetch_Service_false(self):
+ """When `use_fetch_service` is False, we don't make any calls to the
+ fetch service API."""
+
+ job = self.makeJob(use_fetch_service=False)
+
+ job._worker.proxy_info = MagicMock(
+ return_value={
+ "revocation_endpoint": "https://builder-proxy.test/revoke";,
+ "use_fetch_service": False,
+ }
+ )
+
+ yield job.extraBuildArgs()
+ yield job.endProxySession(upload_path="test_path")
+
+ # No calls go through to the fetch service
+ self.assertEqual(0, len(self.fetch_service_api.sessions.requests))
+
+
class MakeCharmRecipeBuildMixin:
"""Provide the common makeBuild method returning a queued build."""
diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py
index 02e0e80..1574c4b 100644
--- a/lib/lp/testing/factory.py
+++ b/lib/lp/testing/factory.py
@@ -6751,6 +6751,8 @@ class LaunchpadObjectFactory(ObjectFactory):
store_secrets=None,
store_channels=None,
date_created=DEFAULT,
+ use_fetch_service=False,
+ fetch_service_policy=FetchServicePolicy.STRICT,
):
"""Make a new charm recipe."""
if registrant is None:
@@ -6799,6 +6801,8 @@ class LaunchpadObjectFactory(ObjectFactory):
store_secrets=store_secrets,
store_channels=store_channels,
date_created=date_created,
+ use_fetch_service=use_fetch_service,
+ fetch_service_policy=fetch_service_policy,
)
if is_stale is not None:
removeSecurityProxy(recipe).is_stale = is_stale
_______________________________________________
Mailing list: https://launchpad.net/~launchpad-reviewers
Post to : launchpad-reviewers@lists.launchpad.net
Unsubscribe : https://launchpad.net/~launchpad-reviewers
More help : https://help.launchpad.net/ListHelp
