Quentin Debhi has proposed merging ~ruinedyourlife/launchpad-buildd:feat-source-build-base into launchpad-buildd:master.
Commit message: Add the capability to build sources Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~ruinedyourlife/launchpad-buildd/+git/launchpad-buildd/+merge/471812 -- Your team Launchpad code reviewers is requested to review the proposed merge of ~ruinedyourlife/launchpad-buildd:feat-source-build-base into launchpad-buildd:master.
diff --git a/debian/changelog b/debian/changelog index 6897374..23fb469 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +launchpad-buildd (242) UNRELEASED; urgency=medium + + [ Quentin Debhi ] + * Add sourcecraft build type. + + -- Quentin Debhi <[email protected]> Thu, 22 Aug 2024 14:09:58 +0200 + launchpad-buildd (241) focal; urgency=medium [ Quentin Debhi ] diff --git a/lpbuildd/buildd.tac b/lpbuildd/buildd.tac index 41da099..7e968eb 100644 --- a/lpbuildd/buildd.tac +++ b/lpbuildd/buildd.tac @@ -23,6 +23,7 @@ from lpbuildd.livefs import LiveFilesystemBuildManager from lpbuildd.log import RotatableFileLogObserver from lpbuildd.oci import OCIBuildManager from lpbuildd.rock import RockBuildManager +from lpbuildd.source import SourceBuildManager from lpbuildd.snap import SnapBuildManager from lpbuildd.sourcepackagerecipe import SourcePackageRecipeBuildManager from lpbuildd.translationtemplates import TranslationTemplatesBuildManager @@ -47,6 +48,7 @@ builder.registerManager(OCIBuildManager, "oci") builder.registerManager(CharmBuildManager, "charm") builder.registerManager(CIBuildManager, "ci") builder.registerManager(RockBuildManager, "rock") +builder.registerManager(SourceBuildManager, "source") application = service.Application("Builder") application.addComponent( diff --git a/lpbuildd/source.py b/lpbuildd/source.py new file mode 100644 index 0000000..b0aac67 --- /dev/null +++ b/lpbuildd/source.py @@ -0,0 +1,96 @@ +import os + +from lpbuildd.debian import DebianBuildManager, DebianBuildState +from lpbuildd.proxy import BuildManagerProxyMixin + +RETCODE_SUCCESS = 0 +RETCODE_FAILURE_INSTALL = 200 +RETCODE_FAILURE_BUILD = 201 + + +class SourceBuildState(DebianBuildState): + BUILD_SOURCE = "BUILD_SOURCE" + + +class SourceBuildManager(BuildManagerProxyMixin, DebianBuildManager): + """Build a source.""" + + backend_name = "lxd" + initial_build_state = SourceBuildState.BUILD_SOURCE + + @property + def needs_sanitized_logs(self): + return True + + def initiate(self, files, chroot, extra_args): + """Initiate a build with a given set of files and chroot.""" + self.name = extra_args["name"] + self.branch = extra_args.get("branch") + self.git_repository = extra_args.get("git_repository") + self.git_path = extra_args.get("git_path") + self.build_path = extra_args.get("build_path") + self.channels = extra_args.get("channels", {}) + self.proxy_url = extra_args.get("proxy_url") + self.revocation_endpoint = extra_args.get("revocation_endpoint") + self.proxy_service = None + + super().initiate(files, chroot, extra_args) + + def doRunBuild(self): + """Run the process to build the source.""" + args = [] + args.extend(self.startProxy()) + if self.revocation_endpoint: + args.extend(["--revocation-endpoint", self.revocation_endpoint]) + for snap, channel in sorted(self.channels.items()): + args.extend(["--channel", f"{snap}={channel}"]) + if self.branch is not None: + args.extend(["--branch", self.branch]) + if self.git_repository is not None: + args.extend(["--git-repository", self.git_repository]) + if self.git_path is not None: + args.extend(["--git-path", self.git_path]) + if self.build_path is not None: + args.extend(["--build-path", self.build_path]) + args.append(self.name) + self.runTargetSubProcess("build-source", *args) + + def iterate_BUILD_SOURCE(self, retcode): + """Finished building the source.""" + self.stopProxy() + self.revokeProxyToken() + if retcode == RETCODE_SUCCESS: + print("[source] Returning build status: OK") + return self.deferGatherResults() + elif ( + retcode >= RETCODE_FAILURE_INSTALL + and retcode <= RETCODE_FAILURE_BUILD + ): + if not self.alreadyfailed: + self._builder.buildFail() + print("[source] Returning build status: Builder failed.") + self.alreadyfailed = True + else: + if not self.alreadyfailed: + self._builder.buildFail() + print("[source] Returning build status: Build failed.") + self.alreadyfailed = True + self.doReapProcesses(self._state) + + def iterateReap_BUILD_SOURCE(self, retcode): + """Finished reaping after building the source.""" + self._state = DebianBuildState.UMOUNT + self.doUnmounting() + + def gatherResults(self): + """Gather the results of the build and add them to the file cache.""" + output_path = os.path.join("/home/buildd", self.name) + if self.build_path is not None: + output_path = os.path.join(output_path, self.build_path) + if self.backend.path_exists(output_path): + for entry in sorted(self.backend.listdir(output_path)): + path = os.path.join(output_path, entry) + if self.backend.islink(path): + continue + # + self.addWaitingFileFromBackend(path) diff --git a/lpbuildd/target/build_source.py b/lpbuildd/target/build_source.py new file mode 100644 index 0000000..370371c --- /dev/null +++ b/lpbuildd/target/build_source.py @@ -0,0 +1,120 @@ +import logging +import os + +from lpbuildd.target.backend import check_path_escape +from lpbuildd.target.build_snap import SnapChannelsAction +from lpbuildd.target.operation import Operation +from lpbuildd.target.proxy import BuilderProxyOperationMixin +from lpbuildd.target.snapstore import SnapStoreOperationMixin +from lpbuildd.target.vcs import VCSOperationMixin + +RETCODE_FAILURE_INSTALL = 200 +RETCODE_FAILURE_BUILD = 201 + + +logger = logging.getLogger(__name__) + + +class BuildSource( + BuilderProxyOperationMixin, + VCSOperationMixin, + SnapStoreOperationMixin, + Operation, +): + description = "Build a source." + + @classmethod + def add_arguments(cls, parser): + super().add_arguments(parser) + parser.add_argument( + "--channel", + action=SnapChannelsAction, + metavar="SNAP=CHANNEL", + dest="channels", + default={}, + help="install SNAP from CHANNEL", + ) + parser.add_argument( + "--build-path", default=".", help="location of source to build." + ) + parser.add_argument("name", help="name of source to build") + + def __init__(self, args, parser): + super().__init__(args, parser) + self.buildd_path = os.path.join("/home/buildd", self.args.name) + + def install(self): + logger.info("Running install phase") + deps = [] + if self.args.proxy_url: + deps.extend(self.proxy_deps) + self.install_git_proxy() + if self.backend.supports_snapd: + # udev is installed explicitly to work around + # https://bugs.launchpad.net/snapd/+bug/1731519. + # Low maintenance: we can keep udevs as a dependency + # since it is a low-level system dependency, + # and since it might be broken for older versions. + for dep in "snapd", "fuse", "squashfuse", "udev": + if self.backend.is_package_available(dep): + deps.append(dep) + deps.extend(self.vcs_deps) + # See charmcraft.provider.CharmcraftBuilddBaseConfiguration.setup. + self.backend.run(["apt-get", "-y", "install"] + deps) + if self.backend.supports_snapd: + self.snap_store_set_proxy() + for snap_name, channel in sorted(self.args.channels.items()): + # sourcecraft is handled separately, since it requires --classic, + # which disables all sandboxing to ensure it runs with no strict + # confinement. + if snap_name != "sourcecraft": + self.backend.run( + ["snap", "install", "--channel=%s" % channel, snap_name] + ) + if "sourcecraft" in self.args.channels: + self.backend.run( + [ + "snap", + "install", + "--classic", + "--channel=%s" % self.args.channels["sourcecraft"], + "sourcecraft", + ] + ) + else: + self.backend.run(["snap", "install", "--classic", "sourcecraft"]) + # With classic confinement, the snap can access the whole system. + # We could build the source in /build, but we are using /home/buildd + # for consistency with other build types. + self.backend.run(["mkdir", "-p", "/home/buildd"]) + + def repo(self): + """Collect git or bzr branch.""" + logger.info("Running repo phase...") + env = self.build_proxy_environment(proxy_url=self.args.proxy_url) + self.vcs_fetch(self.args.name, cwd="/home/buildd", env=env) + self.vcs_update_status(self.buildd_path) + + def build(self): + logger.info("Running build phase...") + build_context_path = os.path.join( + "/home/buildd", self.args.name, self.args.build_path + ) + check_path_escape(self.buildd_path, build_context_path) + env = self.build_proxy_environment(proxy_url=self.args.proxy_url) + args = ["sourcecraft", "pack", "-v", "--destructive-mode"] + self.run_build_command(args, env=env, cwd=build_context_path) + + def run(self): + try: + self.install() + except Exception: + logger.exception("Install failed") + return RETCODE_FAILURE_INSTALL + try: + self.repo() + self.build() + except Exception: + logger.exception("Build failed") + return RETCODE_FAILURE_BUILD + return 0 diff --git a/lpbuildd/target/cli.py b/lpbuildd/target/cli.py index 8d2ade2..fe7bc7b 100644 --- a/lpbuildd/target/cli.py +++ b/lpbuildd/target/cli.py @@ -11,6 +11,7 @@ from lpbuildd.target.build_livefs import BuildLiveFS from lpbuildd.target.build_oci import BuildOCI from lpbuildd.target.build_rock import BuildRock from lpbuildd.target.build_snap import BuildSnap +from lpbuildd.target.build_source import BuildSource from lpbuildd.target.generate_translation_templates import ( GenerateTranslationTemplates, ) @@ -48,6 +49,7 @@ operations = { "build-oci": BuildOCI, "build-charm": BuildCharm, "build-rock": BuildRock, + "build-source": BuildSource, "buildlivefs": BuildLiveFS, "buildsnap": BuildSnap, "generate-translation-templates": GenerateTranslationTemplates, diff --git a/lpbuildd/target/tests/test_build_source.py b/lpbuildd/target/tests/test_build_source.py new file mode 100644 index 0000000..c8a4848 --- /dev/null +++ b/lpbuildd/target/tests/test_build_source.py @@ -0,0 +1,759 @@ +import json +import os +import stat +import subprocess +from textwrap import dedent + +import responses +from fixtures import FakeLogger, TempDir +from systemfixtures import FakeFilesystem +from testtools.matchers import AnyMatch, MatchesAll, MatchesListwise +from testtools.testcase import TestCase + +from lpbuildd.target.backend import InvalidBuildFilePath +from lpbuildd.target.build_source import ( + RETCODE_FAILURE_BUILD, + RETCODE_FAILURE_INSTALL, +) +from lpbuildd.target.cli import parse_args +from lpbuildd.target.tests.matchers import ( + RanAptGet, + RanBuildCommand, + RanCommand, +) +from lpbuildd.target.tests.test_build_snap import FakeRevisionID, RanSnap +from lpbuildd.tests.fakebuilder import FakeMethod + + +class TestBuildSource(TestCase): + def test_run_build_command_no_env(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--branch", + "lp:foo", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.run_build_command(["echo", "hello world"]) + self.assertThat( + build_source.backend.run.calls, + MatchesListwise( + [ + RanBuildCommand( + ["echo", "hello world"], cwd="/home/buildd/test-image" + ), + ] + ), + ) + + def test_run_build_command_env(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--branch", + "lp:foo", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.run_build_command( + ["echo", "hello world"], env={"FOO": "bar baz"} + ) + self.assertThat( + build_source.backend.run.calls, + MatchesListwise( + [ + RanBuildCommand( + ["echo", "hello world"], + FOO="bar baz", + cwd="/home/buildd/test-image", + ) + ] + ), + ) + + def test_install_channels(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--channel=core=candidate", + "--channel=core18=beta", + "--channel=sourcecraft=edge", + "--branch", + "lp:foo", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.install() + self.assertThat( + build_source.backend.run.calls, + MatchesListwise( + [ + RanAptGet( + "install", "bzr" + ), + RanSnap("install", "--channel=candidate", "core"), + RanSnap("install", "--channel=beta", "core18"), + RanSnap( + "install", "--classic", "--channel=edge", "sourcecraft" + ), + RanCommand(["mkdir", "-p", "/home/buildd"]), + ] + ), + ) + + def test_install_bzr(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--branch", + "lp:foo", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.install() + self.assertThat( + build_source.backend.run.calls, + MatchesListwise( + [ + RanAptGet( + "install", "bzr" + ), + RanSnap("install", "--classic", "sourcecraft"), + RanCommand(["mkdir", "-p", "/home/buildd"]), + ] + ), + ) + + def test_install_git(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--git-repository", + "lp:foo", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.install() + self.assertThat( + build_source.backend.run.calls, + MatchesListwise( + [ + RanAptGet( + "install", "git" + ), + RanSnap("install", "--classic", "sourcecraft"), + RanCommand(["mkdir", "-p", "/home/buildd"]), + ] + ), + ) + + @responses.activate + def test_install_snap_store_proxy(self): + store_assertion = dedent( + """\ + type: store + store: store-id + url: http://snap-store-proxy.example + + body + """ + ) + + def respond(request): + return 200, {"X-Assertion-Store-Id": "store-id"}, store_assertion + + responses.add_callback( + "GET", + "http://snap-store-proxy.example/v2/auth/store/assertions";, + callback=respond, + ) + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--git-repository", + "lp:foo", + "--snap-store-proxy-url", + "http://snap-store-proxy.example/";, + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.install() + self.assertThat( + build_source.backend.run.calls, + MatchesListwise( + [ + RanAptGet( + "install", "git" + ), + RanCommand( + ["snap", "ack", "/dev/stdin"], + input_text=store_assertion, + ), + RanCommand( + ["snap", "set", "core", "proxy.store=store-id"] + ), + RanSnap("install", "--classic", "sourcecraft"), + RanCommand(["mkdir", "-p", "/home/buildd"]), + ] + ), + ) + + def test_install_proxy(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--git-repository", + "lp:foo", + "--proxy-url", + "http://proxy.example:3128/";, + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.bin = "/builderbin" + self.useFixture(FakeFilesystem()).add("/builderbin") + os.mkdir("/builderbin") + with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script: + proxy_script.write("proxy script\n") + os.fchmod(proxy_script.fileno(), 0o755) + build_source.install() + self.assertThat( + build_source.backend.run.calls, + MatchesListwise( + [ + RanAptGet( + "install", + "python3", + "socat", + "git", + ), + RanSnap("install", "--classic", "sourcecraft"), + RanCommand(["mkdir", "-p", "/home/buildd"]), + ] + ), + ) + self.assertEqual( + (b"proxy script\n", stat.S_IFREG | 0o755), + build_source.backend.backend_fs[ + "/usr/local/bin/lpbuildd-git-proxy" + ], + ) + + def test_repo_bzr(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--branch", + "lp:foo", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.backend.build_path = self.useFixture(TempDir()).path + build_source.backend.run = FakeRevisionID("42") + build_source.repo() + self.assertThat( + build_source.backend.run.calls, + MatchesListwise( + [ + RanBuildCommand( + ["bzr", "branch", "lp:foo", "test-image"], + cwd="/home/buildd", + ), + RanBuildCommand( + ["bzr", "revno"], + cwd="/home/buildd/test-image", + get_output=True, + universal_newlines=True, + ), + ] + ), + ) + status_path = os.path.join(build_source.backend.build_path, "status") + with open(status_path) as status: + self.assertEqual({"revision_id": "42"}, json.load(status)) + + def test_repo_git(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--git-repository", + "lp:foo", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.backend.build_path = self.useFixture(TempDir()).path + build_source.backend.run = FakeRevisionID("0" * 40) + build_source.repo() + self.assertThat( + build_source.backend.run.calls, + MatchesListwise( + [ + RanBuildCommand( + ["git", "clone", "-n", "lp:foo", "test-image"], + cwd="/home/buildd", + ), + RanBuildCommand( + ["git", "checkout", "-q", "HEAD"], + cwd="/home/buildd/test-image", + ), + RanBuildCommand( + [ + "git", + "submodule", + "update", + "--init", + "--recursive", + ], + cwd="/home/buildd/test-image", + ), + RanBuildCommand( + ["git", "rev-parse", "HEAD^{}"], + cwd="/home/buildd/test-image", + get_output=True, + universal_newlines=True, + ), + ] + ), + ) + status_path = os.path.join(build_source.backend.build_path, "status") + with open(status_path) as status: + self.assertEqual({"revision_id": "0" * 40}, json.load(status)) + + def test_repo_git_with_path(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--git-repository", + "lp:foo", + "--git-path", + "next", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.backend.build_path = self.useFixture(TempDir()).path + build_source.backend.run = FakeRevisionID("0" * 40) + build_source.repo() + self.assertThat( + build_source.backend.run.calls, + MatchesListwise( + [ + RanBuildCommand( + ["git", "clone", "-n", "lp:foo", "test-image"], + cwd="/home/buildd", + ), + RanBuildCommand( + ["git", "checkout", "-q", "next"], + cwd="/home/buildd/test-image", + ), + RanBuildCommand( + [ + "git", + "submodule", + "update", + "--init", + "--recursive", + ], + cwd="/home/buildd/test-image", + ), + RanBuildCommand( + ["git", "rev-parse", "next^{}"], + cwd="/home/buildd/test-image", + get_output=True, + universal_newlines=True, + ), + ] + ), + ) + status_path = os.path.join(build_source.backend.build_path, "status") + with open(status_path) as status: + self.assertEqual({"revision_id": "0" * 40}, json.load(status)) + + def test_repo_git_with_tag_path(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--git-repository", + "lp:foo", + "--git-path", + "refs/tags/1.0", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.backend.build_path = self.useFixture(TempDir()).path + build_source.backend.run = FakeRevisionID("0" * 40) + build_source.repo() + self.assertThat( + build_source.backend.run.calls, + MatchesListwise( + [ + RanBuildCommand( + ["git", "clone", "-n", "lp:foo", "test-image"], + cwd="/home/buildd", + ), + RanBuildCommand( + ["git", "checkout", "-q", "refs/tags/1.0"], + cwd="/home/buildd/test-image", + ), + RanBuildCommand( + [ + "git", + "submodule", + "update", + "--init", + "--recursive", + ], + cwd="/home/buildd/test-image", + ), + RanBuildCommand( + ["git", "rev-parse", "refs/tags/1.0^{}"], + cwd="/home/buildd/test-image", + get_output=True, + universal_newlines=True, + ), + ] + ), + ) + status_path = os.path.join(build_source.backend.build_path, "status") + with open(status_path) as status: + self.assertEqual({"revision_id": "0" * 40}, json.load(status)) + + def test_repo_proxy(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--git-repository", + "lp:foo", + "--proxy-url", + "http://proxy.example:3128/";, + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.backend.build_path = self.useFixture(TempDir()).path + build_source.backend.run = FakeRevisionID("0" * 40) + build_source.repo() + env = { + "http_proxy": "http://proxy.example:3128/";, + "https_proxy": "http://proxy.example:3128/";, + "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy", + "SNAPPY_STORE_NO_CDN": "1", + } + self.assertThat( + build_source.backend.run.calls, + MatchesListwise( + [ + RanBuildCommand( + ["git", "clone", "-n", "lp:foo", "test-image"], + cwd="/home/buildd", + **env, + ), + RanBuildCommand( + ["git", "checkout", "-q", "HEAD"], + cwd="/home/buildd/test-image", + **env, + ), + RanBuildCommand( + [ + "git", + "submodule", + "update", + "--init", + "--recursive", + ], + cwd="/home/buildd/test-image", + **env, + ), + RanBuildCommand( + ["git", "rev-parse", "HEAD^{}"], + cwd="/home/buildd/test-image", + get_output=True, + universal_newlines=True, + ), + ] + ), + ) + status_path = os.path.join(build_source.backend.build_path, "status") + with open(status_path) as status: + self.assertEqual({"revision_id": "0" * 40}, json.load(status)) + + def test_build(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--branch", + "lp:foo", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.backend.add_dir("/build/test-directory") + build_source.build() + self.assertThat( + build_source.backend.run.calls, + MatchesListwise( + [ + RanBuildCommand( + ["sourcecraft", "pack", "-v", "--destructive-mode"], + cwd="/home/buildd/test-image/.", + ), + ] + ), + ) + + def test_build_with_path(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--branch", + "lp:foo", + "--build-path", + "build-aux/", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.backend.add_dir("/build/test-directory") + build_source.build() + self.assertThat( + build_source.backend.run.calls, + MatchesListwise( + [ + RanBuildCommand( + ["sourcecraft", "pack", "-v", "--destructive-mode"], + cwd="/home/buildd/test-image/build-aux/", + ), + ] + ), + ) + + def test_build_proxy(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--branch", + "lp:foo", + "--proxy-url", + "http://proxy.example:3128/";, + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.build() + env = { + "http_proxy": "http://proxy.example:3128/";, + "https_proxy": "http://proxy.example:3128/";, + "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy", + "SNAPPY_STORE_NO_CDN": "1", + } + self.assertThat( + build_source.backend.run.calls, + MatchesListwise( + [ + RanBuildCommand( + ["sourcecraft", "pack", "-v", "--destructive-mode"], + cwd="/home/buildd/test-image/.", + **env, + ), + ] + ), + ) + + def test_run_succeeds(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--branch", + "lp:foo", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.backend.build_path = self.useFixture(TempDir()).path + build_source.backend.run = FakeRevisionID("42") + self.assertEqual(0, build_source.run()) + self.assertThat( + build_source.backend.run.calls, + MatchesAll( + AnyMatch( + RanAptGet( + "install", "bzr" + ), + ), + AnyMatch( + RanBuildCommand( + ["bzr", "branch", "lp:foo", "test-image"], + cwd="/home/buildd", + ) + ), + AnyMatch( + RanBuildCommand( + ["sourcecraft", "pack", "-v", "--destructive-mode"], + cwd="/home/buildd/test-image/.", + ) + ), + ), + ) + + def test_run_install_fails(self): + class FailInstall(FakeMethod): + def __call__(self, run_args, *args, **kwargs): + super().__call__(run_args, *args, **kwargs) + if run_args[0] == "apt-get": + raise subprocess.CalledProcessError(1, run_args) + + self.useFixture(FakeLogger()) + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--branch", + "lp:foo", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.backend.run = FailInstall() + self.assertEqual(RETCODE_FAILURE_INSTALL, build_source.run()) + + def test_run_repo_fails(self): + class FailRepo(FakeMethod): + def __call__(self, run_args, *args, **kwargs): + super().__call__(run_args, *args, **kwargs) + if run_args[:2] == ["bzr", "branch"]: + raise subprocess.CalledProcessError(1, run_args) + + self.useFixture(FakeLogger()) + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--branch", + "lp:foo", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.backend.run = FailRepo() + self.assertEqual(RETCODE_FAILURE_BUILD, build_source.run()) + + def test_run_build_fails(self): + class FailBuild(FakeMethod): + def __call__(self, run_args, *args, **kwargs): + super().__call__(run_args, *args, **kwargs) + if run_args[0] == "sourcecraft": + raise subprocess.CalledProcessError(1, run_args) + + self.useFixture(FakeLogger()) + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--branch", + "lp:foo", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.backend.build_path = self.useFixture(TempDir()).path + build_source.backend.run = FailBuild() + self.assertEqual(RETCODE_FAILURE_BUILD, build_source.run()) + + def test_build_with_invalid_build_path_parent(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--branch", + "lp:foo", + "--build-path", + "../", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.backend.add_dir("/build/test-directory") + self.assertRaises(InvalidBuildFilePath, build_source.build) + + def test_build_with_invalid_build_path_absolute(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--branch", + "lp:foo", + "--build-path", + "/etc", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.backend.add_dir("/build/test-directory") + self.assertRaises(InvalidBuildFilePath, build_source.build) + + def test_build_with_invalid_build_path_symlink(self): + args = [ + "build-source", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--branch", + "lp:foo", + "--build-path", + "build/", + "test-image", + ] + build_source = parse_args(args=args).operation + build_source.buildd_path = self.useFixture(TempDir()).path + os.symlink( + "/etc/hosts", os.path.join(build_source.buildd_path, "build") + ) + self.assertRaises(InvalidBuildFilePath, build_source.build) diff --git a/lpbuildd/tests/test_source.py b/lpbuildd/tests/test_source.py new file mode 100644 index 0000000..06d9ff7 --- /dev/null +++ b/lpbuildd/tests/test_source.py @@ -0,0 +1,242 @@ +import base64 +import os + +import responses +from fixtures import EnvironmentVariable, TempDir +from testtools import TestCase +from testtools.deferredruntest import AsynchronousDeferredRunTest +from twisted.internet import defer + +from lpbuildd.source import SourceBuildManager, SourceBuildState +from lpbuildd.tests.fakebuilder import FakeBuilder +from lpbuildd.tests.matchers import HasWaitingFiles + + +class MockBuildManager(SourceBuildManager): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.commands = [] + self.iterators = [] + + def runSubProcess(self, path, command, iterate=None, env=None): + self.commands.append([path] + command) + if iterate is None: + iterate = self.iterate + self.iterators.append(iterate) + return 0 + + +class TestSourceBuildManagerIteration(TestCase): + """Run SourceBuildManager through its iteration steps.""" + + run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=5) + + def setUp(self): + super().setUp() + self.working_dir = self.useFixture(TempDir()).path + builder_dir = os.path.join(self.working_dir, "builder") + home_dir = os.path.join(self.working_dir, "home") + for dir in (builder_dir, home_dir): + os.mkdir(dir) + self.useFixture(EnvironmentVariable("HOME", home_dir)) + self.builder = FakeBuilder(builder_dir) + self.buildid = "123" + self.buildmanager = MockBuildManager(self.builder, self.buildid) + self.buildmanager._cachepath = self.builder._cachepath + + def getState(self): + """Retrieve build manager's state.""" + return self.buildmanager._state + + @defer.inlineCallbacks + def startBuild(self, args=None, options=None): + # The build manager's iterate() kicks off the consecutive states + # after INIT. + extra_args = { + "series": "xenial", + "arch_tag": "i386", + "name": "test-source", + } + if args is not None: + extra_args.update(args) + original_backend_name = self.buildmanager.backend_name + self.buildmanager.backend_name = "fake" + self.buildmanager.initiate({}, "chroot.tar.gz", extra_args) + self.buildmanager.backend_name = original_backend_name + + # Skip states that are done in DebianBuildManager to the state + # directly before BUILD_SOURCE. + self.buildmanager._state = SourceBuildState.UPDATE + + # BUILD_SOURCE: Run the builder's payload to build the source. + yield self.buildmanager.iterate(0) + self.assertEqual(SourceBuildState.BUILD_SOURCE, self.getState()) + expected_command = [ + "sharepath/bin/in-target", + "in-target", + "build-source", + "--backend=lxd", + "--series=xenial", + "--arch=i386", + self.buildid, + ] + if options is not None: + expected_command.extend(options) + expected_command.append("test-source") + self.assertEqual(expected_command, self.buildmanager.commands[-1]) + self.assertEqual( + self.buildmanager.iterate, self.buildmanager.iterators[-1] + ) + self.assertFalse(self.builder.wasCalled("chrootFail")) + + def test_status(self): + # The build manager returns saved status information on request. + self.assertEqual({}, self.buildmanager.status()) + status_path = os.path.join( + self.working_dir, "home", "build-%s" % self.buildid, "status" + ) + os.makedirs(os.path.dirname(status_path)) + with open(status_path, "w") as status_file: + status_file.write('{"revision_id": "foo"}') + self.assertEqual({"revision_id": "foo"}, self.buildmanager.status()) + + @defer.inlineCallbacks + def test_iterate(self): + # The build manager iterates a normal build from start to finish. + args = { + "git_repository": "https://git.launchpad.dev/~example/+git/source";, + "git_path": "master", + } + expected_options = [ + "--git-repository", + "https://git.launchpad.dev/~example/+git/source";, + "--git-path", + "master", + ] + yield self.startBuild(args, expected_options) + + log_path = os.path.join(self.buildmanager._cachepath, "buildlog") + with open(log_path, "w") as log: + log.write("I am a build log.") + + self.buildmanager.backend.add_file( + "/home/buildd/test-source/test-source_0_all.source", b"I am sourceing." + ) + + # After building the package, reap processes. + yield self.buildmanager.iterate(0) + expected_command = [ + "sharepath/bin/in-target", + "in-target", + "scan-for-processes", + "--backend=lxd", + "--series=xenial", + "--arch=i386", + self.buildid, + ] + + self.assertEqual(SourceBuildState.BUILD_SOURCE, self.getState()) + self.assertEqual(expected_command, self.buildmanager.commands[-1]) + self.assertNotEqual( + self.buildmanager.iterate, self.buildmanager.iterators[-1] + ) + self.assertFalse(self.builder.wasCalled("buildFail")) + self.assertThat( + self.builder, + HasWaitingFiles.byEquality( + { + "test-source_0_all.source": b"I am sourceing.", + } + ), + ) + + # Control returns to the DebianBuildManager in the UMOUNT state. + self.buildmanager.iterateReap(self.getState(), 0) + expected_command = [ + "sharepath/bin/in-target", + "in-target", + "umount-chroot", + "--backend=lxd", + "--series=xenial", + "--arch=i386", + self.buildid, + ] + self.assertEqual(SourceBuildState.UMOUNT, self.getState()) + self.assertEqual(expected_command, self.buildmanager.commands[-1]) + self.assertEqual( + self.buildmanager.iterate, self.buildmanager.iterators[-1] + ) + self.assertFalse(self.builder.wasCalled("buildFail")) + + @defer.inlineCallbacks + def test_iterate_build_path(self): + # The build manager iterates a build using build_path from start to + # finish. + args = { + "git_repository": "https://git.launchpad.dev/~example/+git/source";, + "git_path": "master", + "build_path": "source", + } + expected_options = [ + "--git-repository", + "https://git.launchpad.dev/~example/+git/source";, + "--git-path", + "master", + "--build-path", + "source", + ] + yield self.startBuild(args, expected_options) + + log_path = os.path.join(self.buildmanager._cachepath, "buildlog") + with open(log_path, "w") as log: + log.write("I am a build log.") + + self.buildmanager.backend.add_file( + "/home/buildd/test-source/source/test-source_0_all.source", + b"I am sourceing.", + ) + + # After building the package, reap processes. + yield self.buildmanager.iterate(0) + expected_command = [ + "sharepath/bin/in-target", + "in-target", + "scan-for-processes", + "--backend=lxd", + "--series=xenial", + "--arch=i386", + self.buildid, + ] + + self.assertEqual(SourceBuildState.BUILD_SOURCE, self.getState()) + self.assertEqual(expected_command, self.buildmanager.commands[-1]) + self.assertNotEqual( + self.buildmanager.iterate, self.buildmanager.iterators[-1] + ) + self.assertFalse(self.builder.wasCalled("buildFail")) + self.assertThat( + self.builder, + HasWaitingFiles.byEquality( + { + "test-source_0_all.source": b"I am sourceing.", + } + ), + ) + + # Control returns to the DebianBuildManager in the UMOUNT state. + self.buildmanager.iterateReap(self.getState(), 0) + expected_command = [ + "sharepath/bin/in-target", + "in-target", + "umount-chroot", + "--backend=lxd", + "--series=xenial", + "--arch=i386", + self.buildid, + ] + self.assertEqual(SourceBuildState.UMOUNT, self.getState()) + self.assertEqual(expected_command, self.buildmanager.commands[-1]) + self.assertEqual( + self.buildmanager.iterate, self.buildmanager.iterators[-1] + ) + self.assertFalse(self.builder.wasCalled("buildFail"))
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : [email protected] Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp
