Colin Watson has proposed merging ~cjwatson/launchpad-buildd:charm-snap-build-proxy into launchpad-buildd:master.
Commit message: Honour proxy arguments when building charms Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~cjwatson/launchpad-buildd/+git/launchpad-buildd/+merge/405623 charmcraft essentially always runs pip3 as part of building the charm, and has no obvious way to provide a local package index. It looks as though we'll need to regard charms as requiring internet access to build. -- Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad-buildd:charm-snap-build-proxy into launchpad-buildd:master.
diff --git a/debian/changelog b/debian/changelog index e66175d..0dcf32e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ launchpad-buildd (198) UNRELEASED; urgency=medium * Run charmcraft in verbose mode. + * Honour proxy arguments when building charms. -- Colin Watson <[email protected]> Fri, 09 Jul 2021 14:08:58 +0100 diff --git a/lpbuildd/charm.py b/lpbuildd/charm.py index e5c5fe9..0838fe7 100644 --- a/lpbuildd/charm.py +++ b/lpbuildd/charm.py @@ -11,6 +11,7 @@ from lpbuildd.debian import ( DebianBuildState, DebianBuildManager, ) +from lpbuildd.snap import SnapBuildProxyMixin RETCODE_SUCCESS = 0 @@ -22,7 +23,7 @@ class CharmBuildState(DebianBuildState): BUILD_CHARM = "BUILD_CHARM" -class CharmBuildManager(DebianBuildManager): +class CharmBuildManager(SnapBuildProxyMixin, DebianBuildManager): """Build a charm.""" backend_name = "lxd" @@ -40,6 +41,9 @@ class CharmBuildManager(DebianBuildManager): self.git_path = extra_args.get("git_path") self.build_path = extra_args.get("build_path") self.channels = extra_args.get("channels", {}) + self.proxy_url = extra_args.get("proxy_url") + self.revocation_endpoint = extra_args.get("revocation_endpoint") + self.proxy_service = None super(CharmBuildManager, self).initiate(files, chroot, extra_args) diff --git a/lpbuildd/target/build_charm.py b/lpbuildd/target/build_charm.py index 95da85e..7610a8b 100644 --- a/lpbuildd/target/build_charm.py +++ b/lpbuildd/target/build_charm.py @@ -2,7 +2,6 @@ # GNU Affero General Public License version 3 (see the file LICENSE). from __future__ import print_function -import functools __metaclass__ = type @@ -14,6 +13,7 @@ import sys from lpbuildd.target.backend import check_path_escape from lpbuildd.target.build_snap import SnapChannelsAction from lpbuildd.target.operation import Operation +from lpbuildd.target.snapbuildproxy import SnapBuildProxyOperationMixin from lpbuildd.target.snapstore import SnapStoreOperationMixin from lpbuildd.target.vcs import VCSOperationMixin @@ -25,7 +25,8 @@ RETCODE_FAILURE_BUILD = 201 logger = logging.getLogger(__name__) -class BuildCharm(VCSOperationMixin, SnapStoreOperationMixin, Operation): +class BuildCharm(SnapBuildProxyOperationMixin, VCSOperationMixin, + SnapStoreOperationMixin, Operation): description = "Build a charm." @@ -69,6 +70,9 @@ class BuildCharm(VCSOperationMixin, SnapStoreOperationMixin, Operation): def install(self): logger.info("Running install phase") deps = [] + if self.args.proxy_url: + deps.extend(self.proxy_deps) + self.install_git_proxy() if self.args.backend == "lxd": # udev is installed explicitly to work around # https://bugs.launchpad.net/snapd/+bug/1731519. @@ -99,7 +103,8 @@ class BuildCharm(VCSOperationMixin, SnapStoreOperationMixin, Operation): def repo(self): """Collect git or bzr branch.""" logger.info("Running repo phase...") - self.vcs_fetch(self.args.name, cwd="/home/buildd") + env = self.build_proxy_environment(proxy_url=self.args.proxy_url) + self.vcs_fetch(self.args.name, cwd="/home/buildd", env=env) self.save_status(self.buildd_path) def build(self): @@ -109,8 +114,13 @@ class BuildCharm(VCSOperationMixin, SnapStoreOperationMixin, Operation): self.args.name, self.args.build_path) check_path_escape(self.buildd_path, build_context_path) + env = OrderedDict() + if self.args.proxy_url: + env["http_proxy"] = self.args.proxy_url + env["https_proxy"] = self.args.proxy_url + env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy" args = ["charmcraft", "build", "-v", "-f", build_context_path] - self.run_build_command(args) + self.run_build_command(args, env=env) def run(self): try: diff --git a/lpbuildd/target/tests/test_build_charm.py b/lpbuildd/target/tests/test_build_charm.py index 4e77943..8be840e 100644 --- a/lpbuildd/target/tests/test_build_charm.py +++ b/lpbuildd/target/tests/test_build_charm.py @@ -5,6 +5,7 @@ __metaclass__ = type import json import os +import stat import subprocess from textwrap import dedent @@ -13,6 +14,7 @@ from fixtures import ( TempDir, ) import responses +from systemfixtures import FakeFilesystem from testtools.matchers import ( AnyMatch, Equals, @@ -171,11 +173,11 @@ class TestBuildCharm(TestCase): "--backend=fake", "--series=xenial", "--arch=amd64", "1", "--git-repository", "lp:foo", "--snap-store-proxy-url", "http://snap-store-proxy.example/";, - "test-snap", + "test-image", ] - build_snap = parse_args(args=args).operation - build_snap.install() - self.assertThat(build_snap.backend.run.calls, MatchesListwise([ + build_charm = parse_args(args=args).operation + build_charm.install() + self.assertThat(build_charm.backend.run.calls, MatchesListwise([ RanAptGet("install", "git"), RanCommand( ["snap", "ack", "/dev/stdin"], input_text=store_assertion), @@ -184,6 +186,31 @@ class TestBuildCharm(TestCase): RanCommand(["mkdir", "-p", "/home/buildd"]), ])) + def test_install_proxy(self): + args = [ + "build-charm", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--git-repository", "lp:foo", + "--proxy-url", "http://proxy.example:3128/";, + "test-image", + ] + build_charm = parse_args(args=args).operation + build_charm.bin = "/builderbin" + self.useFixture(FakeFilesystem()).add("/builderbin") + os.mkdir("/builderbin") + with open("/builderbin/snap-git-proxy", "w") as proxy_script: + proxy_script.write("proxy script\n") + os.fchmod(proxy_script.fileno(), 0o755) + build_charm.install() + self.assertThat(build_charm.backend.run.calls, MatchesListwise([ + RanAptGet("install", "python3", "socat", "git"), + RanSnap("install", "charmcraft"), + RanCommand(["mkdir", "-p", "/home/buildd"]), + ])) + self.assertEqual( + (b"proxy script\n", stat.S_IFREG | 0o755), + build_charm.backend.backend_fs["/usr/local/bin/snap-git-proxy"]) + def test_repo_bzr(self): args = [ "build-charm", @@ -284,6 +311,39 @@ class TestBuildCharm(TestCase): with open(status_path) as status: self.assertEqual({"revision_id": "0" * 40}, json.load(status)) + def test_repo_proxy(self): + args = [ + "build-charm", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--git-repository", "lp:foo", + "--proxy-url", "http://proxy.example:3128/";, + "test-image", + ] + build_charm = parse_args(args=args).operation + build_charm.backend.build_path = self.useFixture(TempDir()).path + build_charm.backend.run = FakeRevisionID("0" * 40) + build_charm.repo() + env = { + "http_proxy": "http://proxy.example:3128/";, + "https_proxy": "http://proxy.example:3128/";, + "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy", + } + self.assertThat(build_charm.backend.run.calls, MatchesListwise([ + RanBuildCommand( + ["git", "clone", "lp:foo", "test-image"], + cwd="/home/buildd", **env), + RanBuildCommand( + ["git", "submodule", "update", "--init", "--recursive"], + cwd="/home/buildd/test-image", **env), + RanBuildCommand( + ["git", "rev-parse", "HEAD^{}"], + cwd="/home/buildd/test-image", get_output=True, + universal_newlines=True), + ])) + status_path = os.path.join(build_charm.backend.build_path, "status") + with open(status_path) as status: + self.assertEqual({"revision_id": "0" * 40}, json.load(status)) + def test_build(self): args = [ "build-charm", @@ -317,6 +377,27 @@ class TestBuildCharm(TestCase): cwd="/home/buildd/test-image"), ])) + def test_build_proxy(self): + args = [ + "build-charm", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--branch", "lp:foo", "--proxy-url", "http://proxy.example:3128/";, + "test-image", + ] + build_charm = parse_args(args=args).operation + build_charm.build() + env = { + "http_proxy": "http://proxy.example:3128/";, + "https_proxy": "http://proxy.example:3128/";, + "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy", + } + self.assertThat(build_charm.backend.run.calls, MatchesListwise([ + RanBuildCommand( + ["charmcraft", "build", "-v", "-f", + "/home/buildd/test-image/."], + cwd="/home/buildd/test-image", **env), + ])) + def test_run_succeeds(self): args = [ "build-charm", diff --git a/lpbuildd/tests/test_charm.py b/lpbuildd/tests/test_charm.py index 407f732..8610bad 100644 --- a/lpbuildd/tests/test_charm.py +++ b/lpbuildd/tests/test_charm.py @@ -4,6 +4,10 @@ __metaclass__ = type import os +try: + from unittest import mock +except ImportError: + import mock from fixtures import ( EnvironmentVariable, @@ -137,3 +141,15 @@ class TestCharmBuildManagerIteration(TestCase): self.assertEqual( self.buildmanager.iterate, self.buildmanager.iterators[-1]) self.assertFalse(self.builder.wasCalled("buildFail")) + + @mock.patch('lpbuildd.snap.urlopen') + def test_revokeProxyToken(self, urlopen_mock): + self.buildmanager.revocation_endpoint = "http://revoke_endpoint"; + self.buildmanager.proxy_url = "http://username:password@proxy_url"; + self.buildmanager.revokeProxyToken() + self.assertEqual(1, urlopen_mock.call_count) + request = urlopen_mock.call_args[0][0] + self.assertEqual( + {'Authorization': "Basic dXNlcm5hbWU6cGFzc3dvcmQ="}, + request.headers) + self.assertEqual('http://revoke_endpoint', request.get_full_url())
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : [email protected] Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp
