Tom Wardill has proposed merging lp:~twom/launchpad-buildd/initial-docker-build-support into lp:launchpad-buildd.
Commit message: Add initial docker build support Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~twom/launchpad-buildd/initial-docker-build-support/+merge/369775 Add a builder for docker, creating an image following a supplied Dockerfile. Save the image, extract it and then tar each component layer individually for returning/caching in launchpad. -- Your team Launchpad code reviewers is requested to review the proposed merge of lp:~twom/launchpad-buildd/initial-docker-build-support into lp:launchpad-buildd.
=== modified file 'debian/changelog' --- debian/changelog 2019-06-18 16:54:33 +0000 +++ debian/changelog 2019-07-05 15:18:01 +0000 @@ -1,3 +1,9 @@ +launchpad-buildd (177) UNRELEASED; urgency=medium + ++ * Prototype Docker image building support. + + -- Colin Watson <[email protected]> Wed, 05 Jun 2019 15:06:54 +0100 + launchpad-buildd (176) xenial; urgency=medium * Don't rely on /CurrentlyBuilding existing in base images. @@ -726,7 +732,7 @@ memory at once (LP: #1227086). [ Adam Conrad ] - * Tidy up log formatting of the "Already reaped..." message. + * Tidy up log formatting of the "Already reaped..." message. -- Colin Watson <[email protected]> Fri, 27 Sep 2013 13:08:59 +0100 @@ -911,7 +917,7 @@ launchpad-buildd (98) hardy; urgency=low * Add launchpad-buildd dependency on python-apt, as an accomodation for it - being only a Recommends but actually required by python-debian. + being only a Recommends but actually required by python-debian. LP: #890834 -- Martin Pool <[email protected]> Wed, 16 Nov 2011 10:28:48 +1100 @@ -965,7 +971,7 @@ launchpad-buildd (90) hardy; urgency=low - * debhelper is a Build-Depends because it is needed to run 'clean'. + * debhelper is a Build-Depends because it is needed to run 'clean'. * python-lpbuildd conflicts with launchpad-buildd << 88. * Add and adjust build-arch, binary-arch, build-indep to match policy. * Complies with stardards version 3.9.2. === modified file 'lpbuildd/buildd-slave.tac' --- lpbuildd/buildd-slave.tac 2019-02-12 10:35:12 +0000 +++ lpbuildd/buildd-slave.tac 2019-07-05 15:18:01 +0000 @@ -23,6 +23,7 @@ from lpbuildd.binarypackage import BinaryPackageBuildManager from lpbuildd.builder import XMLRPCBuilder +from lpbuildd.docker import DockerBuildManager from lpbuildd.livefs import LiveFilesystemBuildManager from lpbuildd.log import RotatableFileLogObserver from lpbuildd.snap import SnapBuildManager @@ -45,6 +46,7 @@ TranslationTemplatesBuildManager, 'translation-templates') builder.registerManager(LiveFilesystemBuildManager, "livefs") builder.registerManager(SnapBuildManager, "snap") +builder.registerManager(DockerBuildManager, "docker") application = service.Application('Builder') application.addComponent( === added file 'lpbuildd/docker.py' --- lpbuildd/docker.py 1970-01-01 00:00:00 +0000 +++ lpbuildd/docker.py 2019-07-05 15:18:01 +0000 @@ -0,0 +1,167 @@ +# Copyright 2019 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +from __future__ import print_function + +__metaclass__ = type + +import base64 +import json +import os +import tempfile + +from six.moves.configparser import ( + NoOptionError, + NoSectionError, + ) +from six.moves.urllib.error import ( + HTTPError, + URLError, + ) +from six.moves.urllib.parse import urlparse +from six.moves.urllib.request import ( + Request, + urlopen, + ) +from twisted.application import strports + +from lpbuildd.debian import ( + DebianBuildManager, + DebianBuildState, + ) +from lpbuildd.snap import SnapProxyFactory + + +RETCODE_SUCCESS = 0 +RETCODE_FAILURE_INSTALL = 200 +RETCODE_FAILURE_BUILD = 201 + + +class DockerBuildState(DebianBuildState): + BUILD_DOCKER = "BUILD_DOCKER" + + +class DockerBuildManager(DebianBuildManager): + """Build a snap.""" + + backend_name = "lxd" + initial_build_state = DockerBuildState.BUILD_DOCKER + + @property + def needs_sanitized_logs(self): + return True + + def initiate(self, files, chroot, extra_args): + """Initiate a build with a given set of files and chroot.""" + self.name = extra_args["name"] + self.branch = extra_args.get("branch") + self.git_repository = extra_args.get("git_repository") + self.git_path = extra_args.get("git_path") + self.file = extra_args.get("file") + self.proxy_url = extra_args.get("proxy_url") + self.revocation_endpoint = extra_args.get("revocation_endpoint") + self.proxy_service = None + + super(DockerBuildManager, self).initiate(files, chroot, extra_args) + + def startProxy(self): + """Start the local snap proxy, if necessary.""" + if not self.proxy_url: + return [] + proxy_port = self._builder._config.get("snapmanager", "proxyport") + proxy_factory = SnapProxyFactory(self, self.proxy_url, timeout=60) + self.proxy_service = strports.service(proxy_port, proxy_factory) + self.proxy_service.setServiceParent(self._builder.service) + if self.backend_name == "lxd": + proxy_host = self.backend.ipv4_network.ip + else: + proxy_host = "localhost" + return ["--proxy-url", "http://{}:{}/".format(proxy_host, proxy_port)] + + def stopProxy(self): + """Stop the local snap proxy, if necessary.""" + if self.proxy_service is None: + return + self.proxy_service.disownServiceParent() + self.proxy_service = None + + def revokeProxyToken(self): + """Revoke builder proxy token.""" + if not self.revocation_endpoint: + return + self._builder.log("Revoking proxy token...\n") + url = urlparse(self.proxy_url) + auth = "{}:{}".format(url.username, url.password) + headers = { + "Authorization": "Basic {}".format(base64.b64encode(auth)) + } + req = Request(self.revocation_endpoint, None, headers) + req.get_method = lambda: "DELETE" + try: + urlopen(req) + except (HTTPError, URLError) as e: + self._builder.log( + "Unable to revoke token for %s: %s" % (url.username, e)) + + def doRunBuild(self): + """Run the process to build the snap.""" + args = [] + args.extend(self.startProxy()) + if self.revocation_endpoint: + args.extend(["--revocation-endpoint", self.revocation_endpoint]) + if self.branch is not None: + args.extend(["--branch", self.branch]) + if self.git_repository is not None: + args.extend(["--git-repository", self.git_repository]) + if self.git_path is not None: + args.extend(["--git-path", self.git_path]) + if self.file is not None: + args.extend(["--file", self.file]) + try: + snap_store_proxy_url = self._builder._config.get( + "proxy", "snapstore") + args.extend(["--snap-store-proxy-url", snap_store_proxy_url]) + except (NoSectionError, NoOptionError): + pass + args.append(self.name) + self.runTargetSubProcess("build-docker", *args) + + def iterate_BUILD_DOCKER(self, retcode): + """Finished building the Docker image.""" + self.stopProxy() + self.revokeProxyToken() + if retcode == RETCODE_SUCCESS: + print("Returning build status: OK") + return self.deferGatherResults() + elif (retcode >= RETCODE_FAILURE_INSTALL and + retcode <= RETCODE_FAILURE_BUILD): + if not self.alreadyfailed: + self._builder.buildFail() + print("Returning build status: Build failed.") + self.alreadyfailed = True + else: + if not self.alreadyfailed: + self._builder.builderFail() + print("Returning build status: Builder failed.") + self.alreadyfailed = True + self.doReapProcesses(self._state) + + def iterateReap_BUILD_DOCKER(self, retcode): + """Finished reaping after building the Docker image.""" + self._state = DebianBuildState.UMOUNT + self.doUnmounting() + + def gatherResults(self): + """Gather the results of the build and add them to the file cache.""" + self.addWaitingFileFromBackend('/build/manifest.json') + with tempfile.NamedTemporaryFile() as manifest_path: + self.backend.copy_out('/build/manifest.json', manifest_path.name) + with open(manifest_path.name) as manifest_fp: + manifest = json.load(manifest_fp) + + for section in manifest: + layers = section['Layers'] + for layer in layers: + layer_name = layer.split('/')[0] + layer_path = os.path.join('/build/', layer_name + '.tar') + self.addWaitingFileFromBackend(layer_path) === added file 'lpbuildd/target/build_docker.py' --- lpbuildd/target/build_docker.py 1970-01-01 00:00:00 +0000 +++ lpbuildd/target/build_docker.py 2019-07-05 15:18:01 +0000 @@ -0,0 +1,143 @@ +# Copyright 2019 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +from __future__ import print_function + +__metaclass__ = type + +from collections import OrderedDict +import logging +import os.path +import sys + +from lpbuildd.target.operation import Operation +from lpbuildd.target.snapstore import SnapStoreOperationMixin +from lpbuildd.target.vcs import VCSOperationMixin + + +RETCODE_FAILURE_INSTALL = 200 +RETCODE_FAILURE_BUILD = 201 + + +logger = logging.getLogger(__name__) + + +class BuildDocker(VCSOperationMixin, SnapStoreOperationMixin, Operation): + + description = "Build a Docker image." + + @classmethod + def add_arguments(cls, parser): + super(BuildDocker, cls).add_arguments(parser) + parser.add_argument("--proxy-url", help="builder proxy url") + parser.add_argument( + "--revocation-endpoint", + help="builder proxy token revocation endpoint") + parser.add_argument("--file", help="path to Dockerfile in branch") + parser.add_argument("name", help="name of snap to build") + + def __init__(self, args, parser): + super(BuildDocker, self).__init__(args, parser) + self.bin = os.path.dirname(sys.argv[0]) + + def run_build_command(self, args, env=None, **kwargs): + """Run a build command in the target. + + :param args: the command and arguments to run. + :param env: dictionary of additional environment variables to set. + :param kwargs: any other keyword arguments to pass to Backend.run. + """ + full_env = OrderedDict() + full_env["LANG"] = "C.UTF-8" + full_env["SHELL"] = "/bin/sh" + if env: + full_env.update(env) + return self.backend.run(args, env=full_env, **kwargs) + + def install(self): + logger.info("Running install phase...") + deps = [] + if self.args.backend == "lxd": + # udev is installed explicitly to work around + # https://bugs.launchpad.net/snapd/+bug/1731519. + for dep in "snapd", "fuse", "squashfuse", "udev": + if self.backend.is_package_available(dep): + deps.append(dep) + deps.extend(self.vcs_deps) + if self.args.proxy_url: + deps.extend(["python3", "socat"]) + self.backend.run(["apt-get", "-y", "install"] + deps) + if self.args.backend in ("lxd", "fake"): + self.snap_store_set_proxy() + self.backend.run(["snap", "install", "docker"]) + if self.args.proxy_url: + self.backend.copy_in( + os.path.join(self.bin, "snap-git-proxy"), + "/usr/local/bin/snap-git-proxy") + # The docker snap can't see /build, so we have to do our work under + # /home/buildd instead. Make sure it exists. + self.backend.run(["mkdir", "-p", "/home/buildd"]) + + def repo(self): + """Collect git or bzr branch.""" + logger.info("Running repo phase...") + env = OrderedDict() + if self.args.proxy_url: + env["http_proxy"] = self.args.proxy_url + env["https_proxy"] = self.args.proxy_url + env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy" + self.vcs_fetch(self.args.name, cwd="/home/buildd", env=env) + + def build(self): + logger.info("Running build phase...") + args = ["docker", "build", "--no-cache"] + if self.args.proxy_url: + for var in ("http_proxy", "https_proxy"): + args.extend( + ["--build-arg", "{}={}".format(var, self.args.proxy_url)]) + args.extend(["--tag", self.args.name]) + if self.args.file is not None: + args.extend(["--file", self.args.file]) + args.append(os.path.join("/home/buildd", self.args.name)) + self.run_build_command(args) + + # Make extraction directy + self.backend.run(["mkdir", "-p", "/home/buildd/{}-extract".format( + self.args.name)]) + + # save the newly built image + docker_save = "docker save {name} > /build/{name}.tar".format( + name=self.args.name) + save_args = ["/bin/bash", "-c", docker_save] + self.run_build_command(save_args) + + # extract the saved image + extract_args = [ + "tar", "-xf", "/build/{name}.tar".format(name=self.args.name), + "-C", "/build/" + ] + self.run_build_command(extract_args) + + # Tar each layer separately + build_dir_contents = self.backend.listdir('/build') + for content in build_dir_contents: + content_path = os.path.join('/build/', content) + if not self.backend.isdir(content_path): + continue + tar_path = '/build/{}.tar'.format(content) + tar_args = ['tar', '-cvf', tar_path, content_path] + self.run_build_command(tar_args) + + def run(self): + try: + self.install() + except Exception: + logger.exception('Install failed') + return RETCODE_FAILURE_INSTALL + try: + self.repo() + self.build() + except Exception: + logger.exception('Build failed') + return RETCODE_FAILURE_BUILD + return 0 === modified file 'lpbuildd/target/cli.py' --- lpbuildd/target/cli.py 2017-09-08 15:57:18 +0000 +++ lpbuildd/target/cli.py 2019-07-05 15:18:01 +0000 @@ -14,6 +14,7 @@ OverrideSourcesList, Update, ) +from lpbuildd.target.build_docker import BuildDocker from lpbuildd.target.build_livefs import BuildLiveFS from lpbuildd.target.build_snap import BuildSnap from lpbuildd.target.generate_translation_templates import ( @@ -49,6 +50,7 @@ operations = { "add-trusted-keys": AddTrustedKeys, + "build-docker": BuildDocker, "buildlivefs": BuildLiveFS, "buildsnap": BuildSnap, "generate-translation-templates": GenerateTranslationTemplates, === added file 'lpbuildd/target/tests/test_build_docker.py' --- lpbuildd/target/tests/test_build_docker.py 1970-01-01 00:00:00 +0000 +++ lpbuildd/target/tests/test_build_docker.py 2019-07-05 15:18:01 +0000 @@ -0,0 +1,433 @@ +# Copyright 2019 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +__metaclass__ = type + +import os.path +import stat +import subprocess +from textwrap import dedent + +from fixtures import ( + FakeLogger, + TempDir, + ) +import responses +from systemfixtures import FakeFilesystem +from testtools import TestCase +from testtools.matchers import ( + AnyMatch, + Equals, + Is, + MatchesAll, + MatchesDict, + MatchesListwise, + ) + +from lpbuildd.target.build_docker import ( + RETCODE_FAILURE_BUILD, + RETCODE_FAILURE_INSTALL, + ) +from lpbuildd.target.cli import parse_args +from lpbuildd.tests.fakebuilder import FakeMethod + + +class RanCommand(MatchesListwise): + + def __init__(self, args, echo=None, cwd=None, input_text=None, + get_output=None, **env): + kwargs_matcher = {} + if echo is not None: + kwargs_matcher["echo"] = Is(echo) + if cwd: + kwargs_matcher["cwd"] = Equals(cwd) + if input_text: + kwargs_matcher["input_text"] = Equals(input_text) + if get_output is not None: + kwargs_matcher["get_output"] = Is(get_output) + if env: + kwargs_matcher["env"] = MatchesDict( + {key: Equals(value) for key, value in env.items()}) + super(RanCommand, self).__init__( + [Equals((args,)), MatchesDict(kwargs_matcher)]) + + +class RanAptGet(RanCommand): + + def __init__(self, *args): + super(RanAptGet, self).__init__(["apt-get", "-y"] + list(args)) + + +class RanSnap(RanCommand): + + def __init__(self, *args, **kwargs): + super(RanSnap, self).__init__(["snap"] + list(args), **kwargs) + + +class RanBuildCommand(RanCommand): + + def __init__(self, args, **kwargs): + kwargs.setdefault("LANG", "C.UTF-8") + kwargs.setdefault("SHELL", "/bin/sh") + super(RanBuildCommand, self).__init__(args, **kwargs) + + +class TestBuildDocker(TestCase): + + def test_run_build_command_no_env(self): + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--branch", "lp:foo", "test-image", + ] + build_docker = parse_args(args=args).operation + build_docker.run_build_command(["echo", "hello world"]) + self.assertThat(build_docker.backend.run.calls, MatchesListwise([ + RanBuildCommand(["echo", "hello world"]), + ])) + + def test_run_build_command_env(self): + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--branch", "lp:foo", "test-image", + ] + build_docker = parse_args(args=args).operation + build_docker.run_build_command( + ["echo", "hello world"], env={"FOO": "bar baz"}) + self.assertThat(build_docker.backend.run.calls, MatchesListwise([ + RanBuildCommand(["echo", "hello world"], FOO="bar baz"), + ])) + + def test_install_bzr(self): + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--branch", "lp:foo", "test-image" + ] + build_docker = parse_args(args=args).operation + build_docker.install() + self.assertThat(build_docker.backend.run.calls, MatchesListwise([ + RanAptGet("install", "bzr"), + RanSnap("install", "docker"), + RanCommand(["mkdir", "-p", "/home/buildd"]), + ])) + + def test_install_git(self): + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--git-repository", "lp:foo", "test-image" + ] + build_docker = parse_args(args=args).operation + build_docker.install() + self.assertThat(build_docker.backend.run.calls, MatchesListwise([ + RanAptGet("install", "git"), + RanSnap("install", "docker"), + RanCommand(["mkdir", "-p", "/home/buildd"]), + ])) + + @responses.activate + def test_install_snap_store_proxy(self): + store_assertion = dedent("""\ + type: store + store: store-id + url: http://snap-store-proxy.example + + body + """) + + def respond(request): + return 200, {"X-Assertion-Store-Id": "store-id"}, store_assertion + + responses.add_callback( + "GET", "http://snap-store-proxy.example/v2/auth/store/assertions";, + callback=respond) + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--git-repository", "lp:foo", + "--snap-store-proxy-url", "http://snap-store-proxy.example/";, + "test-image", + ] + build_snap = parse_args(args=args).operation + build_snap.install() + self.assertThat(build_snap.backend.run.calls, MatchesListwise([ + RanAptGet("install", "git"), + RanSnap("ack", "/dev/stdin", input_text=store_assertion), + RanSnap("set", "core", "proxy.store=store-id"), + RanSnap("install", "docker"), + RanCommand(["mkdir", "-p", "/home/buildd"]), + ])) + + def test_install_proxy(self): + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--git-repository", "lp:foo", + "--proxy-url", "http://proxy.example:3128/";, + "test-image", + ] + build_docker = parse_args(args=args).operation + build_docker.bin = "/builderbin" + self.useFixture(FakeFilesystem()).add("/builderbin") + os.mkdir("/builderbin") + with open("/builderbin/snap-git-proxy", "w") as proxy_script: + proxy_script.write("proxy script\n") + os.fchmod(proxy_script.fileno(), 0o755) + build_docker.install() + self.assertThat(build_docker.backend.run.calls, MatchesListwise([ + RanAptGet("install", "git", "python3", "socat"), + RanSnap("install", "docker"), + RanCommand(["mkdir", "-p", "/home/buildd"]), + ])) + self.assertEqual( + (b"proxy script\n", stat.S_IFREG | 0o755), + build_docker.backend.backend_fs["/usr/local/bin/snap-git-proxy"]) + + def test_repo_bzr(self): + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--branch", "lp:foo", "test-image", + ] + build_docker = parse_args(args=args).operation + build_docker.backend.build_path = self.useFixture(TempDir()).path + build_docker.backend.run = FakeMethod() + build_docker.repo() + self.assertThat(build_docker.backend.run.calls, MatchesListwise([ + RanBuildCommand( + ["bzr", "branch", "lp:foo", "test-image"], cwd="/home/buildd"), + ])) + + def test_repo_git(self): + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--git-repository", "lp:foo", "test-image", + ] + build_docker = parse_args(args=args).operation + build_docker.backend.build_path = self.useFixture(TempDir()).path + build_docker.backend.run = FakeMethod() + build_docker.repo() + self.assertThat(build_docker.backend.run.calls, MatchesListwise([ + RanBuildCommand( + ["git", "clone", "lp:foo", "test-image"], cwd="/home/buildd"), + RanBuildCommand( + ["git", "submodule", "update", "--init", "--recursive"], + cwd="/home/buildd/test-image"), + ])) + + def test_repo_git_with_path(self): + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--git-repository", "lp:foo", "--git-path", "next", "test-image", + ] + build_docker = parse_args(args=args).operation + build_docker.backend.build_path = self.useFixture(TempDir()).path + build_docker.backend.run = FakeMethod() + build_docker.repo() + self.assertThat(build_docker.backend.run.calls, MatchesListwise([ + RanBuildCommand( + ["git", "clone", "-b", "next", "lp:foo", "test-image"], + cwd="/home/buildd"), + RanBuildCommand( + ["git", "submodule", "update", "--init", "--recursive"], + cwd="/home/buildd/test-image"), + ])) + + def test_repo_git_with_tag_path(self): + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--git-repository", "lp:foo", "--git-path", "refs/tags/1.0", + "test-image", + ] + build_docker = parse_args(args=args).operation + build_docker.backend.build_path = self.useFixture(TempDir()).path + build_docker.backend.run = FakeMethod() + build_docker.repo() + self.assertThat(build_docker.backend.run.calls, MatchesListwise([ + RanBuildCommand( + ["git", "clone", "-b", "1.0", "lp:foo", "test-image"], + cwd="/home/buildd"), + RanBuildCommand( + ["git", "submodule", "update", "--init", "--recursive"], + cwd="/home/buildd/test-image"), + ])) + + def test_repo_proxy(self): + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--git-repository", "lp:foo", + "--proxy-url", "http://proxy.example:3128/";, + "test-image", + ] + build_docker = parse_args(args=args).operation + build_docker.backend.build_path = self.useFixture(TempDir()).path + build_docker.backend.run = FakeMethod() + build_docker.repo() + env = { + "http_proxy": "http://proxy.example:3128/";, + "https_proxy": "http://proxy.example:3128/";, + "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy", + } + self.assertThat(build_docker.backend.run.calls, MatchesListwise([ + RanBuildCommand( + ["git", "clone", "lp:foo", "test-image"], + cwd="/home/buildd", **env), + RanBuildCommand( + ["git", "submodule", "update", "--init", "--recursive"], + cwd="/home/buildd/test-image", **env), + ])) + + def test_build(self): + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--branch", "lp:foo", "test-image", + ] + build_docker = parse_args(args=args).operation + build_docker.backend.add_dir('/build/test-directory') + build_docker.build() + self.assertThat(build_docker.backend.run.calls, MatchesListwise([ + RanBuildCommand( + ["docker", "build", "--no-cache", "--tag", "test-image", + "/home/buildd/test-image"]), + RanCommand(["mkdir", "-p", "/home/buildd/test-image-extract"]), + RanBuildCommand([ + '/bin/bash', '-c', + 'docker save test-image > /build/test-image.tar']), + RanBuildCommand([ + 'tar', '-xf', '/build/test-image.tar', '-C', '/build/']), + RanBuildCommand([ + 'tar', '-cvf', '/build/test-directory.tar', + '/build/test-directory']), + ])) + + def test_build_with_file(self): + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--branch", "lp:foo", "--file", "build-aux/Dockerfile", + "test-image", + ] + build_docker = parse_args(args=args).operation + build_docker.backend.add_dir('/build/test-directory') + build_docker.build() + self.assertThat(build_docker.backend.run.calls, MatchesListwise([ + RanBuildCommand( + ["docker", "build", "--no-cache", "--tag", "test-image", + "--file", "build-aux/Dockerfile", "/home/buildd/test-image"]), + RanCommand(["mkdir", "-p", "/home/buildd/test-image-extract"]), + RanBuildCommand([ + '/bin/bash', '-c', + 'docker save test-image > /build/test-image.tar']), + RanBuildCommand([ + 'tar', '-xf', '/build/test-image.tar', '-C', '/build/']), + RanBuildCommand([ + 'tar', '-cvf', '/build/test-directory.tar', + '/build/test-directory']), + ])) + + def test_build_proxy(self): + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--branch", "lp:foo", "--proxy-url", "http://proxy.example:3128/";, + "test-image", + ] + build_docker = parse_args(args=args).operation + build_docker.backend.add_dir('/build/test-directory') + build_docker.build() + self.assertThat(build_docker.backend.run.calls, MatchesListwise([ + RanBuildCommand( + ["docker", "build", "--no-cache", + "--build-arg", "http_proxy=http://proxy.example:3128/";, + "--build-arg", "https_proxy=http://proxy.example:3128/";, + "--tag", "test-image", "/home/buildd/test-image"]), + RanCommand(["mkdir", "-p", "/home/buildd/test-image-extract"]), + RanBuildCommand([ + '/bin/bash', '-c', + 'docker save test-image > /build/test-image.tar']), + RanBuildCommand([ + 'tar', '-xf', '/build/test-image.tar', '-C', '/build/']), + RanBuildCommand([ + 'tar', '-cvf', '/build/test-directory.tar', + '/build/test-directory']), + ])) + + def test_run_succeeds(self): + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--branch", "lp:foo", "test-image", + ] + build_docker = parse_args(args=args).operation + build_docker.backend.build_path = self.useFixture(TempDir()).path + build_docker.backend.run = FakeMethod() + self.assertEqual(0, build_docker.run()) + self.assertThat(build_docker.backend.run.calls, MatchesAll( + AnyMatch(RanAptGet("install", "bzr")), + AnyMatch(RanSnap("install", "docker")), + AnyMatch(RanBuildCommand( + ["bzr", "branch", "lp:foo", "test-image"], + cwd="/home/buildd")), + AnyMatch(RanBuildCommand( + ["docker", "build", "--no-cache", "--tag", "test-image", + "/home/buildd/test-image"])), + )) + + def test_run_install_fails(self): + class FailInstall(FakeMethod): + def __call__(self, run_args, *args, **kwargs): + super(FailInstall, self).__call__(run_args, *args, **kwargs) + if run_args[0] == "apt-get": + raise subprocess.CalledProcessError(1, run_args) + + self.useFixture(FakeLogger()) + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--branch", "lp:foo", "test-image", + ] + build_docker = parse_args(args=args).operation + build_docker.backend.run = FailInstall() + self.assertEqual(RETCODE_FAILURE_INSTALL, build_docker.run()) + + def test_run_repo_fails(self): + class FailRepo(FakeMethod): + def __call__(self, run_args, *args, **kwargs): + super(FailRepo, self).__call__(run_args, *args, **kwargs) + if run_args[:2] == ["bzr", "branch"]: + raise subprocess.CalledProcessError(1, run_args) + + self.useFixture(FakeLogger()) + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--branch", "lp:foo", "test-image", + ] + build_docker = parse_args(args=args).operation + build_docker.backend.run = FailRepo() + self.assertEqual(RETCODE_FAILURE_BUILD, build_docker.run()) + + def test_run_build_fails(self): + class FailBuild(FakeMethod): + def __call__(self, run_args, *args, **kwargs): + super(FailBuild, self).__call__(run_args, *args, **kwargs) + if run_args[0] == "docker": + raise subprocess.CalledProcessError(1, run_args) + + self.useFixture(FakeLogger()) + args = [ + "build-docker", + "--backend=fake", "--series=xenial", "--arch=amd64", "1", + "--branch", "lp:foo", "test-image", + ] + build_docker = parse_args(args=args).operation + build_docker.backend.build_path = self.useFixture(TempDir()).path + build_docker.backend.run = FailBuild() + self.assertEqual(RETCODE_FAILURE_BUILD, build_docker.run()) === added file 'lpbuildd/tests/test_docker.py' --- lpbuildd/tests/test_docker.py 1970-01-01 00:00:00 +0000 +++ lpbuildd/tests/test_docker.py 2019-07-05 15:18:01 +0000 @@ -0,0 +1,197 @@ +# Copyright 2019 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +__metaclass__ = type + +import os + +from fixtures import ( + EnvironmentVariable, + TempDir, + ) +from testtools import TestCase +from testtools.deferredruntest import AsynchronousDeferredRunTest +from twisted.internet import defer + +from lpbuildd.docker import ( + DockerBuildManager, + DockerBuildState, + ) +from lpbuildd.tests.fakebuilder import FakeBuilder +from lpbuildd.tests.matchers import HasWaitingFiles + + +class MockBuildManager(DockerBuildManager): + def __init__(self, *args, **kwargs): + super(MockBuildManager, self).__init__(*args, **kwargs) + self.commands = [] + self.iterators = [] + + def runSubProcess(self, path, command, iterate=None, env=None): + self.commands.append([path] + command) + if iterate is None: + iterate = self.iterate + self.iterators.append(iterate) + return 0 + + +class TestDockerBuildManagerIteration(TestCase): + """Run DockerBuildManager through its iteration steps.""" + + run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=5) + + def setUp(self): + super(TestDockerBuildManagerIteration, self).setUp() + self.working_dir = self.useFixture(TempDir()).path + builder_dir = os.path.join(self.working_dir, "builder") + home_dir = os.path.join(self.working_dir, "home") + for dir in (builder_dir, home_dir): + os.mkdir(dir) + self.useFixture(EnvironmentVariable("HOME", home_dir)) + self.builder = FakeBuilder(builder_dir) + self.buildid = "123" + self.buildmanager = MockBuildManager(self.builder, self.buildid) + self.buildmanager._cachepath = self.builder._cachepath + + def getState(self): + """Retrieve build manager's state.""" + return self.buildmanager._state + + @defer.inlineCallbacks + def startBuild(self, args=None, options=None): + # The build manager's iterate() kicks off the consecutive states + # after INIT. + extra_args = { + "series": "xenial", + "arch_tag": "i386", + "name": "test-image", + } + if args is not None: + extra_args.update(args) + original_backend_name = self.buildmanager.backend_name + self.buildmanager.backend_name = "fake" + self.buildmanager.initiate({}, "chroot.tar.gz", extra_args) + self.buildmanager.backend_name = original_backend_name + + # Skip states that are done in DebianBuildManager to the state + # directly before BUILD_DOCKER. + self.buildmanager._state = DockerBuildState.UPDATE + + # BUILD_DOCKER: Run the builder's payload to build the snap package. + yield self.buildmanager.iterate(0) + self.assertEqual(DockerBuildState.BUILD_DOCKER, self.getState()) + expected_command = [ + "sharepath/bin/in-target", "in-target", "build-docker", + "--backend=lxd", "--series=xenial", "--arch=i386", self.buildid, + ] + if options is not None: + expected_command.extend(options) + expected_command.append("test-image") + self.assertEqual(expected_command, self.buildmanager.commands[-1]) + self.assertEqual( + self.buildmanager.iterate, self.buildmanager.iterators[-1]) + self.assertFalse(self.builder.wasCalled("chrootFail")) + + @defer.inlineCallbacks + def test_iterate(self): + # The build manager iterates a normal build from start to finish. + args = { + "git_repository": "https://git.launchpad.dev/~example/+git/snap";, + "git_path": "master", + } + expected_options = [ + "--git-repository", "https://git.launchpad.dev/~example/+git/snap";, + "--git-path", "master", + ] + yield self.startBuild(args, expected_options) + + log_path = os.path.join(self.buildmanager._cachepath, "buildlog") + with open(log_path, "w") as log: + log.write("I am a build log.") + + self.buildmanager.backend.add_file("/build/manifest.json", b"[]") + + # After building the package, reap processes. + yield self.buildmanager.iterate(0) + expected_command = [ + "sharepath/bin/in-target", "in-target", "scan-for-processes", + "--backend=lxd", "--series=xenial", "--arch=i386", self.buildid, + ] + self.assertEqual(DockerBuildState.BUILD_DOCKER, self.getState()) + self.assertEqual(expected_command, self.buildmanager.commands[-1]) + self.assertNotEqual( + self.buildmanager.iterate, self.buildmanager.iterators[-1]) + self.assertFalse(self.builder.wasCalled("buildFail")) + self.assertThat(self.builder, HasWaitingFiles.byEquality({ + "manifest.json": b"[]", + })) + + # Control returns to the DebianBuildManager in the UMOUNT state. + self.buildmanager.iterateReap(self.getState(), 0) + expected_command = [ + "sharepath/bin/in-target", "in-target", "umount-chroot", + "--backend=lxd", "--series=xenial", "--arch=i386", self.buildid, + ] + self.assertEqual(DockerBuildState.UMOUNT, self.getState()) + self.assertEqual(expected_command, self.buildmanager.commands[-1]) + self.assertEqual( + self.buildmanager.iterate, self.buildmanager.iterators[-1]) + self.assertFalse(self.builder.wasCalled("buildFail")) + + @defer.inlineCallbacks + def test_iterate_with_file(self): + # The build manager iterates a build that specifies a non-default + # Dockerfile location from start to finish. + args = { + "git_repository": "https://git.launchpad.dev/~example/+git/snap";, + "git_path": "master", + "file": "build-aux/Dockerfile", + } + expected_options = [ + "--git-repository", "https://git.launchpad.dev/~example/+git/snap";, + "--git-path", "master", + "--file", "build-aux/Dockerfile", + ] + yield self.startBuild(args, expected_options) + + log_path = os.path.join(self.buildmanager._cachepath, "buildlog") + with open(log_path, "w") as log: + log.write("I am a build log.") + + self.buildmanager.backend.add_file("/build/manifest.json", b"[]") + + # After building the package, reap processes. + yield self.buildmanager.iterate(0) + expected_command = [ + "sharepath/bin/in-target", "in-target", "scan-for-processes", + "--backend=lxd", "--series=xenial", "--arch=i386", self.buildid, + ] + self.assertEqual(DockerBuildState.BUILD_DOCKER, self.getState()) + self.assertEqual(expected_command, self.buildmanager.commands[-1]) + self.assertNotEqual( + self.buildmanager.iterate, self.buildmanager.iterators[-1]) + self.assertFalse(self.builder.wasCalled("buildFail")) + self.assertThat(self.builder, HasWaitingFiles.byEquality({ + "manifest.json": b"[]", + })) + + # Control returns to the DebianBuildManager in the UMOUNT state. + self.buildmanager.iterateReap(self.getState(), 0) + expected_command = [ + "sharepath/bin/in-target", "in-target", "umount-chroot", + "--backend=lxd", "--series=xenial", "--arch=i386", self.buildid, + ] + self.assertEqual(DockerBuildState.UMOUNT, self.getState()) + self.assertEqual(expected_command, self.buildmanager.commands[-1]) + self.assertEqual( + self.buildmanager.iterate, self.buildmanager.iterators[-1]) + self.assertFalse(self.builder.wasCalled("buildFail")) + + @defer.inlineCallbacks + def test_iterate_snap_store_proxy(self): + # The build manager can be told to use a snap store proxy. + self.builder._config.set( + "proxy", "snapstore", "http://snap-store-proxy.example/";) + expected_options = [ + "--snap-store-proxy-url", "http://snap-store-proxy.example/";] + yield self.startBuild(options=expected_options)
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : [email protected] Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp
