Colin Watson has proposed merging ~cjwatson/launchpad-buildd:ci-clamav into launchpad-buildd:master.
Commit message: Add optional malware scanning at the end of CI build jobs Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~cjwatson/launchpad-buildd/+git/launchpad-buildd/+merge/430040 This is currently implemented using clamav. It's probably not yet amazingly effective, and I expect we'd need to start doing on-access scanning in order to get much better, but it gives us a starting point. -- Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad-buildd:ci-clamav into launchpad-buildd:master.
diff --git a/debian/changelog b/debian/changelog index aa2253a..0d35fce 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ launchpad-buildd (222) UNRELEASED; urgency=medium * Remove use of six. + * Add optional malware scanning at the end of CI build jobs, currently + implemented using clamav. -- Colin Watson <[email protected]> Mon, 12 Sep 2022 09:50:13 +0100 diff --git a/lpbuildd/ci.py b/lpbuildd/ci.py index c49a24c..6083298 100644 --- a/lpbuildd/ci.py +++ b/lpbuildd/ci.py @@ -64,6 +64,7 @@ class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager): self.environment_variables = extra_args.get("environment_variables") self.plugin_settings = extra_args.get("plugin_settings") self.secrets = extra_args.get("secrets") + self.scan_malware = extra_args.get("scan_malware", False) super().initiate(files, chroot, extra_args) @@ -82,6 +83,8 @@ class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager): args.extend(["--git-repository", self.git_repository]) if self.git_path is not None: args.extend(["--git-path", self.git_path]) + if self.scan_malware: + args.append("--scan-malware") try: snap_store_proxy_url = self._builder._config.get( "proxy", "snapstore") @@ -164,6 +167,8 @@ class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager): ) args.extend( ["--secrets", "/build/.launchpad-secrets.yaml"]) + if self.scan_malware: + args.append("--scan-malware") job_name, job_index = self.current_job self.current_job_id = _make_job_id(job_name, job_index) diff --git a/lpbuildd/target/run_ci.py b/lpbuildd/target/run_ci.py index 8f23b01..491943b 100644 --- a/lpbuildd/target/run_ci.py +++ b/lpbuildd/target/run_ci.py @@ -31,6 +31,12 @@ class RunCIPrepare(BuilderProxyOperationMixin, VCSOperationMixin, parser.add_argument( "--channel", action=SnapChannelsAction, metavar="SNAP=CHANNEL", dest="channels", default={}, help="install SNAP from CHANNEL") + parser.add_argument( + "--scan-malware", + action="store_true", + default=False, + help="perform malware scans on output files", + ) def install(self): logger.info("Running install phase...") @@ -43,6 +49,8 @@ class RunCIPrepare(BuilderProxyOperationMixin, VCSOperationMixin, if self.backend.is_package_available(dep): deps.append(dep) deps.extend(self.vcs_deps) + if self.args.scan_malware: + deps.append("clamav") self.backend.run(["apt-get", "-y", "install"] + deps) if self.backend.supports_snapd: self.snap_store_set_proxy() @@ -59,6 +67,16 @@ class RunCIPrepare(BuilderProxyOperationMixin, VCSOperationMixin, cmd.append(snap_name) self.backend.run(cmd) self.backend.run(["lxd", "init", "--auto"]) + if self.args.scan_malware: + # lpbuildd.target.lxd configures the container not to run most + # services, which is convenient since it allows us to ensure + # that ClamAV's database is up to date before proceeding. + kwargs = {} + env = self.build_proxy_environment(proxy_url=self.args.proxy_url) + if env: + kwargs["env"] = env + logger.info("Downloading malware definitions...") + self.backend.run(["freshclam", "--quiet"], **kwargs) def repo(self): """Collect VCS branch.""" @@ -121,6 +139,12 @@ class RunCI(BuilderProxyOperationMixin, Operation): type=str, help="secrets where the key and the value are separated by =", ) + parser.add_argument( + "--scan-malware", + action="store_true", + default=False, + help="perform malware scans on output files", + ) def run_job(self): logger.info("Running job phase...") @@ -172,6 +196,10 @@ class RunCI(BuilderProxyOperationMixin, Operation): ] self.run_build_command(args, env=env) + if self.args.scan_malware: + clamscan = ["clamscan", "--recursive", job_output_path] + self.run_build_command(clamscan, env=env) + def run(self): try: self.run_job() diff --git a/lpbuildd/target/tests/test_run_ci.py b/lpbuildd/target/tests/test_run_ci.py index 0b0b77b..ba941ea 100644 --- a/lpbuildd/target/tests/test_run_ci.py +++ b/lpbuildd/target/tests/test_run_ci.py @@ -141,6 +141,53 @@ class TestRunCIPrepare(TestCase): RanCommand(["lxd", "init", "--auto"]), ])) + def test_install_scan_malware(self): + args = [ + "run-ci-prepare", + "--backend=fake", "--series=focal", "--arch=amd64", "1", + "--git-repository", "lp:foo", + "--scan-malware", + ] + run_ci_prepare = parse_args(args=args).operation + run_ci_prepare.install() + self.assertThat(run_ci_prepare.backend.run.calls, MatchesListwise([ + RanAptGet("install", "git", "clamav"), + RanSnap("install", "lxd"), + RanSnap("install", "--classic", "lpcraft"), + RanCommand(["lxd", "init", "--auto"]), + RanCommand(["freshclam", "--quiet"]), + ])) + + def test_install_scan_malware_proxy(self): + args = [ + "run-ci-prepare", + "--backend=fake", "--series=focal", "--arch=amd64", "1", + "--git-repository", "lp:foo", + "--proxy-url", "http://proxy.example:3128/";, + "--scan-malware", + ] + run_ci_prepare = parse_args(args=args).operation + run_ci_prepare.bin = "/builderbin" + self.useFixture(FakeFilesystem()).add("/builderbin") + os.mkdir("/builderbin") + with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script: + proxy_script.write("proxy script\n") + os.fchmod(proxy_script.fileno(), 0o755) + run_ci_prepare.install() + env = { + "http_proxy": "http://proxy.example:3128/";, + "https_proxy": "http://proxy.example:3128/";, + "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy", + "SNAPPY_STORE_NO_CDN": "1", + } + self.assertThat(run_ci_prepare.backend.run.calls, MatchesListwise([ + RanAptGet("install", "python3", "socat", "git", "clamav"), + RanSnap("install", "lxd"), + RanSnap("install", "--classic", "lpcraft"), + RanCommand(["lxd", "init", "--auto"]), + RanCommand(["freshclam", "--quiet"], **env), + ])) + def test_repo_git(self): args = [ "run-ci-prepare", @@ -440,6 +487,47 @@ class TestRunCI(TestCase): ], cwd="/build/tree"), ])) + def test_run_job_scan_malware_succeeds(self): + args = [ + "run-ci", + "--backend=fake", "--series=focal", "--arch=amd64", "1", + "--scan-malware", + "test", "0", + ] + run_ci = parse_args(args=args).operation + run_ci.run_job() + self.assertThat(run_ci.backend.run.calls, MatchesListwise([ + RanCommand(["mkdir", "-p", "/build/output/test/0"]), + RanBuildCommand([ + "/bin/bash", "-o", "pipefail", "-c", + "lpcraft -v run-one --output-directory /build/output " + "test 0 " + "2>&1 " + "| tee /build/output/test/0/log", + ], cwd="/build/tree"), + RanBuildCommand( + ["clamscan", "--recursive", "/build/output/test/0"], + cwd="/build/tree"), + ])) + + def test_run_job_scan_malware_fails(self): + class FailClamscan(FakeMethod): + def __call__(self, run_args, *args, **kwargs): + super().__call__(run_args, *args, **kwargs) + if run_args[0] == "clamscan": + raise subprocess.CalledProcessError(1, run_args) + + self.useFixture(FakeLogger()) + args = [ + "run-ci", + "--backend=fake", "--series=focal", "--arch=amd64", "1", + "--scan-malware", + "test", "0", + ] + run_ci = parse_args(args=args).operation + run_ci.backend.run = FailClamscan() + self.assertRaises(subprocess.CalledProcessError, run_ci.run_job) + def test_run_succeeds(self): args = [ "run-ci", diff --git a/lpbuildd/tests/test_ci.py b/lpbuildd/tests/test_ci.py index 88dda19..bf1b468 100644 --- a/lpbuildd/tests/test_ci.py +++ b/lpbuildd/tests/test_ci.py @@ -128,11 +128,13 @@ class TestCIBuildManagerIteration(TestCase): }, "secrets": { "auth": "user:pass", - } + }, + "scan_malware": True, } expected_prepare_options = [ "--git-repository", "https://git.launchpad.test/~example/+git/ci";, "--git-path", "main", + "--scan-malware", ] yield self.startBuild(args, expected_prepare_options) @@ -145,6 +147,7 @@ class TestCIBuildManagerIteration(TestCase): "--plugin-setting", "miniconda_conda_channel=https://user:[email protected]/artifactory/soss-conda-stable-local/";, # noqa: E501 "--plugin-setting", "foo=bar", "--secrets", "/build/.launchpad-secrets.yaml", + "--scan-malware", ] yield self.expectRunJob("build", "0", options=expected_job_options) self.buildmanager.backend.add_file(
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : [email protected] Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp
