Hi, The attached patch series implements initial version of the gbp-buildpackage-rpm tool.
The patches modifying gbp.command_wrappers:Command (patches 0011 and 0012) are not strictly required but they make the output from nosetests much cleaner. These patches, plus some additional features extending gbp-buildpackage and gbp-buildpackage-rpm, can also be found on the feature/buildpackage-rpm branch in my Github repository: https://github.com/marquiz/git-buildpackage-rpm Best Regards, Markus Lehtonen
>From ff5911b46a6ed16084cdd6d50580e2d22c9e1991 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen <[email protected]> Date: Fri, 27 Jun 2014 08:36:29 +0300 Subject: [PATCH 01/14] GitRepository/has_submodules: add treeish argument For defining a Git treeish which to look into, instead of the current working copy. Signed-off-by: Markus Lehtonen <[email protected]> --- gbp/git/repository.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/gbp/git/repository.py b/gbp/git/repository.py index c261e40..a0995b7 100644 --- a/gbp/git/repository.py +++ b/gbp/git/repository.py @@ -1735,18 +1735,23 @@ class GitRepository(object): #{ Submodules - def has_submodules(self): + def has_submodules(self, treeish=None): """ Does the repo have any submodules? + @param treeish: look into treeish + @type treeish: C{str} @return: C{True} if the repository has any submodules, C{False} otherwise @rtype: C{bool} """ - if os.path.exists(os.path.join(self.path, '.gitmodules')): + if treeish: + try: + self.show('%s:.gitmodules' % treeish) + except GitRepositoryError: + return False return True - else: - return False + return os.path.exists(os.path.join(self.path, '.gitmodules')) def add_submodule(self, repo_path): -- 1.8.4.5
>From 0bb43cf3ab3be6ba5085a014b42ea0b946a5e291 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen <[email protected]> Date: Tue, 15 May 2012 16:37:33 +0300 Subject: [PATCH 02/14] common/buildpackage: support for different archive formats Adds support for defining the archive format of the output of git_archive_single(), e.g. 'zip'. Defaults to 'tar', as before. Signed-off-by: Ed Bartosh <[email protected]> Signed-off-by: Markus Lehtonen <[email protected]> --- debian/control | 2 +- gbp/command_wrappers.py | 10 ++++++++ gbp/scripts/common/buildpackage.py | 48 ++++++++++++++++++++++---------------- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/debian/control b/debian/control index 3e42326..4992af4 100644 --- a/debian/control +++ b/debian/control @@ -45,7 +45,7 @@ Depends: ${python:Depends}, python-dateutil, python-pkg-resources, Recommends: pristine-tar (>= 0.5), cowbuilder, python-requests -Suggests: python-notify, unzip +Suggests: python-notify, unzip, zipmerge Description: Suite to help with Debian packages in Git repositories This package contains the following tools: * gbp import-{dsc,dscs}: import existing Debian source packages into a git diff --git a/gbp/command_wrappers.py b/gbp/command_wrappers.py index b8bd21f..f1312e4 100644 --- a/gbp/command_wrappers.py +++ b/gbp/command_wrappers.py @@ -233,6 +233,16 @@ class UnpackZipArchive(Command): self.run_error = 'Couldn\'t unpack "%s"' % self.archive +class CatenateZipArchive(Command): + """Wrap zipmerge tool to catenate a zip file with the next""" + def __init__(self, archive, **kwargs): + self.archive = archive + Command.__init__(self, 'zipmerge', [archive], **kwargs) + + def __call__(self, target): + Command.__call__(self, [target]) + + class GitCommand(Command): "Mother/Father of all git commands" def __init__(self, cmd, args=[], **kwargs): diff --git a/gbp/scripts/common/buildpackage.py b/gbp/scripts/common/buildpackage.py index 2e53b78..f95147e 100644 --- a/gbp/scripts/common/buildpackage.py +++ b/gbp/scripts/common/buildpackage.py @@ -22,7 +22,7 @@ import os, os.path import pipes import tempfile import shutil -from gbp.command_wrappers import (CatenateTarArchive) +from gbp.command_wrappers import (CatenateTarArchive, CatenateZipArchive) from gbp.errors import GbpError import gbp.log @@ -50,51 +50,59 @@ def sanitize_prefix(prefix): return '/' -def git_archive_submodules(repo, treeish, output, prefix, comp_type, comp_level, comp_opts): +def git_archive_submodules(repo, treeish, output, prefix, comp_type, comp_level, + comp_opts, format='tar'): """ - Create tar.gz of an archive with submodules + Create a source tree archive with submodules. - since git-archive always writes an end of tarfile trailer we concatenate + Since git-archive always writes an end of tarfile trailer we concatenate the generated archives using tar and compress the result. Exception handling is left to the caller. """ prefix = sanitize_prefix(prefix) - tarfile = output.rsplit('.', 1)[0] tempdir = tempfile.mkdtemp() - submodule_tarfile = os.path.join(tempdir, "submodule.tar") + main_archive = os.path.join(tempdir, "main.%s" % format) + submodule_archive = os.path.join(tempdir, "submodule.%s" % format) try: - # generate main tarfile - repo.archive(format='tar', prefix=prefix, - output=tarfile, treeish=treeish) + # generate main (tmp) archive + repo.archive(format=format, prefix=prefix, + output=main_archive, treeish=treeish) - # generate each submodule's tarfile and append it to the main archive + # generate each submodule's arhive and append it to the main archive for (subdir, commit) in repo.get_submodules(treeish): tarpath = [subdir, subdir[2:]][subdir.startswith("./")] gbp.log.debug("Processing submodule %s (%s)" % (subdir, commit[0:8])) - repo.archive(format='tar', prefix='%s%s/' % (prefix, tarpath), - output=submodule_tarfile, treeish=commit, cwd=subdir) - CatenateTarArchive(tarfile)(submodule_tarfile) + repo.archive(format=format, prefix='%s%s/' % (prefix, tarpath), + output=submodule_archive, treeish=commit, cwd=subdir) + if format == 'tar': + CatenateTarArchive(main_archive)(submodule_archive) + elif format == 'zip': + CatenateZipArchive(main_archive)(submodule_archive) # compress the output - ret = os.system("%s -%s %s %s" % (comp_type, comp_level, comp_opts, tarfile)) - if ret: - raise GbpError("Error creating %s: %d" % (output, ret)) + if comp_type: + ret = os.system("%s --stdout -%s %s %s > %s" % (comp_type, comp_level, comp_opts, main_archive, output)) + if ret: + raise GbpError("Error creating %s: %d" % (output, ret)) + else: + shutil.move(main_archive, output) finally: shutil.rmtree(tempdir) -def git_archive_single(treeish, output, prefix, comp_type, comp_level, comp_opts): +def git_archive_single(treeish, output, prefix, comp_type, comp_level, comp_opts, format='tar'): """ - Create tar.gz of an archive without submodules + Create an archive without submodules Exception handling is left to the caller. """ prefix = sanitize_prefix(prefix) pipe = pipes.Template() - pipe.prepend("git archive --format=tar --prefix=%s %s" % (prefix, treeish), '.-') - pipe.append('%s -c -%s %s' % (comp_type, comp_level, comp_opts), '--') + pipe.prepend("git archive --format=%s --prefix=%s %s" % (format, prefix, treeish), '.-') + if comp_type: + pipe.append('%s -c -%s %s' % (comp_type, comp_level, comp_opts), '--') ret = pipe.copy('', output) if ret: raise GbpError("Error creating %s: %d" % (output, ret)) -- 1.8.4.5
>From 15baf2e61712404172499f387e80011666f5741d Mon Sep 17 00:00:00 2001 From: Markus Lehtonen <[email protected]> Date: Tue, 13 May 2014 16:41:32 +0300 Subject: [PATCH 03/14] notifications: make notify() more generic In order to be able to re-use it in gbp-buildpackage-rpm. Signed-off-by: Markus Lehtonen <[email protected]> --- gbp/notifications.py | 5 ++--- gbp/scripts/buildpackage.py | 11 ++++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/gbp/notifications.py b/gbp/notifications.py index 9d9071a..d628b26 100644 --- a/gbp/notifications.py +++ b/gbp/notifications.py @@ -53,7 +53,7 @@ def send_notification(summary, msg): return True -def notify(cp, success, notify_opt): +def notify(summary, message, notify_opt): """ Send a notifications @return: False on error @@ -66,6 +66,5 @@ def notify(cp, success, notify_opt): if not enable: return [True, False][notify_opt.is_on()] - summary, msg = build_msg(cp, success) - return notify_opt.do(send_notification, summary, msg) + return notify_opt.do(send_notification, summary, message) diff --git a/gbp/scripts/buildpackage.py b/gbp/scripts/buildpackage.py index 1c48c05..87d1710 100755 --- a/gbp/scripts/buildpackage.py +++ b/gbp/scripts/buildpackage.py @@ -636,11 +636,12 @@ def main(argv): if options.export_dir and options.purge and not retval: RemoveTree(export_dir)() - if source and not gbp.notifications.notify(source.changelog, - not retval, - options.notify): - gbp.log.err("Failed to send notification") - retval = 1 + if source: + summary, msg = gbp.notifications.build_msg(source.changelog, + not retval) + if not gbp.notifications.notify(summary, msg, options.notify): + gbp.log.err("Failed to send notification") + retval = 1 return retval -- 1.8.4.5
>From 2ada7ead1ea7f4f81a2bedc3995086336b5a1254 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen <[email protected]> Date: Mon, 2 Jun 2014 10:14:52 +0300 Subject: [PATCH 04/14] tristate: implement __nonzero__() method Returns False if tristate is 'off', otherwise True ('on' or 'auto'). Signed-off-by: Markus Lehtonen <[email protected]> --- gbp/tristate.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/gbp/tristate.py b/gbp/tristate.py index 9361060..0a800ec 100644 --- a/gbp/tristate.py +++ b/gbp/tristate.py @@ -66,6 +66,17 @@ class Tristate(object): else: return 'off' + def __nonzero__(self): + """ + >>> Tristate('on').__nonzero__() + True + >>> Tristate('auto').__nonzero__() + True + >>> Tristate('off').__nonzero__() + False + """ + return self._state is not self.OFF + @property def state(self): """Get current state""" -- 1.8.4.5
>From 9cb255378d8bd959dd0c49ed20435c7a84e2f1c5 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen <[email protected]> Date: Tue, 3 Feb 2015 15:56:58 +0200 Subject: [PATCH 05/14] gbp.format: fix one typo and indentation Signed-off-by: Markus Lehtonen <[email protected]> --- gbp/format.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gbp/format.py b/gbp/format.py index 2a4af15..9c37bb1 100644 --- a/gbp/format.py +++ b/gbp/format.py @@ -20,7 +20,7 @@ from gbp.errors import GbpError def format_msg(msg, args): """ - Format a strin with the given dict. Be a bit more verbose than + Format a string with the given dict. Be a bit more verbose than default python about the error cause. >>> format_msg("%(foo)", {}) @@ -39,6 +39,7 @@ def format_msg(msg, args): except ValueError as e: raise GbpError("Failed to format %s with %s: %s" % (msg, args, e)) except KeyError as e: - raise GbpError("Failed to format %s: Missing value %s in %s" % (msg, e, args)) + raise GbpError("Failed to format %s: Missing value %s in %s" % + (msg, e, args)) -- 1.8.4.5
>From 388e1350eca2b1761b4f6445954a2b1387893124 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen <[email protected]> Date: Tue, 3 Feb 2015 16:00:24 +0200 Subject: [PATCH 06/14] gbp.format: rename format_msg() to format_str() The new name more precisely describes it as it can be used to format any strings (e.g. tag names) and not only messages. Signed-off-by: Markus Lehtonen <[email protected]> --- gbp/deb/git.py | 4 ++-- gbp/format.py | 8 ++++---- gbp/scripts/buildpackage.py | 4 ++-- gbp/scripts/import_orig.py | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/gbp/deb/git.py b/gbp/deb/git.py index 2a848d4..7b0b181 100644 --- a/gbp/deb/git.py +++ b/gbp/deb/git.py @@ -19,7 +19,7 @@ import re from gbp.git import GitRepository, GitRepositoryError from gbp.deb.pristinetar import DebianPristineTar -from gbp.format import format_msg +from gbp.format import format_str class DebianGitRepository(GitRepository): """A git repository that holds the source of a Debian package""" @@ -105,7 +105,7 @@ class DebianGitRepository(GitRepository): >>> DebianGitRepository.version_to_tag("debian/%(version)s", "0:0~0") 'debian/0%0_0' """ - return format_msg(format, dict(version=DebianGitRepository._sanitize_version(version))) + return format_str(format, dict(version=DebianGitRepository._sanitize_version(version))) @staticmethod def _sanitize_version(version): diff --git a/gbp/format.py b/gbp/format.py index 9c37bb1..c367dc5 100644 --- a/gbp/format.py +++ b/gbp/format.py @@ -18,20 +18,20 @@ from gbp.errors import GbpError -def format_msg(msg, args): +def format_str(msg, args): """ Format a string with the given dict. Be a bit more verbose than default python about the error cause. - >>> format_msg("%(foo)", {}) + >>> format_str("%(foo)", {}) Traceback (most recent call last): ... GbpError: Failed to format %(foo): Missing value 'foo' in {} - >>> format_msg("%(foo)", {'foo': 'bar'}) + >>> format_str("%(foo)", {'foo': 'bar'}) Traceback (most recent call last): ... GbpError: Failed to format %(foo) with {'foo': 'bar'}: incomplete format - >>> format_msg("A %(foo)s is a %(bar)s", {'foo': 'dog', 'bar': 'mamal'}) + >>> format_str("A %(foo)s is a %(bar)s", {'foo': 'dog', 'bar': 'mamal'}) 'A dog is a mamal' """ try: diff --git a/gbp/scripts/buildpackage.py b/gbp/scripts/buildpackage.py index 87d1710..e1ddeee 100755 --- a/gbp/scripts/buildpackage.py +++ b/gbp/scripts/buildpackage.py @@ -29,7 +29,7 @@ from gbp.command_wrappers import (Command, from gbp.config import (GbpOptionParserDebian, GbpOptionGroup) from gbp.deb.git import (GitRepositoryError, DebianGitRepository) from gbp.deb.source import DebianSource, DebianSourceError -from gbp.format import format_msg +from gbp.format import format_str from gbp.git.vfs import GitVfs from gbp.deb.upstreamsource import DebianUpstreamSource from gbp.errors import GbpError @@ -605,7 +605,7 @@ def main(argv): gbp.log.info("Tagging %s as %s" % (source.changelog.version, tag)) if options.retag and repo.has_tag(tag): repo.delete_tag(tag) - tag_msg = format_msg(options.debian_tag_msg, + tag_msg = format_str(options.debian_tag_msg, dict(pkg=source.sourcepkg, version=source.changelog.version)) repo.create_tag(name=tag, diff --git a/gbp/scripts/import_orig.py b/gbp/scripts/import_orig.py index f73dea1..b2d285d 100644 --- a/gbp/scripts/import_orig.py +++ b/gbp/scripts/import_orig.py @@ -29,7 +29,7 @@ from gbp.deb.changelog import ChangeLog, NoChangeLogError from gbp.deb.git import (GitRepositoryError, DebianGitRepository) from gbp.config import GbpOptionParserDebian, GbpOptionGroup, no_upstream_branch_msg from gbp.errors import GbpError -from gbp.format import format_msg +from gbp.format import format_str import gbp.log from gbp.scripts.common.import_orig import (orig_needs_repack, cleanup_tmp_tree, ask_package_name, ask_package_version, @@ -398,7 +398,7 @@ def main(argv): epoch = '%s:' % cp.epoch info = { 'version': "%s%s-1" % (epoch, version) } env = { 'GBP_BRANCH': options.debian_branch } - gbpc.Command(format_msg(options.postimport, info), extra_env=env, shell=True)() + gbpc.Command(format_str(options.postimport, info), extra_env=env, shell=True)() # Update working copy and index if we've possibly updated the # checked out branch current_branch = repo.get_branch() -- 1.8.4.5
>From a510bff0241116c068d874ad355ed4cfec779abe Mon Sep 17 00:00:00 2001 From: Markus Lehtonen <[email protected]> Date: Wed, 4 Feb 2015 14:09:39 +0200 Subject: [PATCH 07/14] Rewrite gbp.tmpfile This commit changes the tmpfile module to only contain minimal init_tmpdir() and del_tmpdir() functions, intended to be called at the beginning and end of the scipts. The init function creates a base tempdir and configures the Python tempfile module to use that as the (default) directory for temporary files and directories. Correspondingly, the del function removes the base tempdir and restores the configuration of the tempfile module. The new logic ensures that all temporary files/directories will be created in the desired base tempdir (assuming that no 'dir' argument is given to the tempfile functions). The base tempdir is not removed by del_tmpdir() if a non-empty value of GBP_TMPFILE_NOCLEAN is found in environment. This can be useful e.g. in debugging. Signed-off-by: Markus Lehtonen <[email protected]> --- gbp/scripts/import_srpm.py | 22 +++++++++------------- gbp/scripts/pq_rpm.py | 19 ++++++++----------- gbp/tmpfile.py | 37 ++++++++++++++++++++++++++----------- 3 files changed, 43 insertions(+), 35 deletions(-) diff --git a/gbp/scripts/import_srpm.py b/gbp/scripts/import_srpm.py index d2392c0..abf6217 100755 --- a/gbp/scripts/import_srpm.py +++ b/gbp/scripts/import_srpm.py @@ -27,8 +27,8 @@ import shutil import errno import urllib2 -import gbp.tmpfile as tempfile import gbp.command_wrappers as gbpc +from gbp.tmpfile import init_tmpdir, del_tmpdir, tempfile from gbp.rpm import (parse_srpm, guess_spec, SpecFile, NoSpecError, RpmUpstreamSource, compose_version_str) from gbp.rpm.git import (RpmGitRepository, GitRepositoryError) @@ -63,14 +63,14 @@ def download_file(target_dir, url): raise GbpError("Download failed: %s" % err.reason) return local_fn -def download_source(pkg, dirs): +def download_source(pkg): """Download package from a remote location""" if re.match(r'[a-z]{1,5}://', pkg): mode = 'python urllib2' else: mode = 'yumdownloader' - tmpdir = tempfile.mkdtemp(dir=dirs['tmp_base'], prefix='download_') + tmpdir = tempfile.mkdtemp(prefix='download_') gbp.log.info("Trying to download '%s' using '%s'..." % (pkg, mode)) if mode == 'yumdownloader': gbpc.RunAtCommand('yumdownloader', @@ -201,23 +201,21 @@ def main(argv): gbp.log.err("Need to give exactly one package to import. Try --help.") return 1 try: - dirs['tmp_base'] = tempfile.mkdtemp(dir=options.tmp_dir, - prefix='import-srpm') + dirs['tmp_base'] = init_tmpdir(options.tmp_dir, 'import-srpm_') except GbpError as err: gbp.log.err(err) return 1 try: srpm = args[0] if options.download: - srpm = download_source(srpm, dirs) + srpm = download_source(srpm) # Real srpm, we need to unpack, first true_srcrpm = False if not os.path.isdir(srpm) and not srpm.endswith(".spec"): src = parse_srpm(srpm) true_srcrpm = True - dirs['pkgextract'] = tempfile.mkdtemp(dir=dirs['tmp_base'], - prefix='pkgextract_') + dirs['pkgextract'] = tempfile.mkdtemp(prefix='pkgextract_') gbp.log.info("Extracting src rpm to '%s'" % dirs['pkgextract']) src.unpack(dirs['pkgextract']) preferred_spec = src.name + '.spec' @@ -259,10 +257,8 @@ def main(argv): set_bare_repo_options(options) # Create more tempdirs - dirs['origsrc'] = tempfile.mkdtemp(dir=dirs['tmp_base'], - prefix='origsrc_') - dirs['packaging_base'] = tempfile.mkdtemp(dir=dirs['tmp_base'], - prefix='packaging_') + dirs['origsrc'] = tempfile.mkdtemp(prefix='origsrc_') + dirs['packaging_base'] = tempfile.mkdtemp(prefix='packaging_') dirs['packaging'] = os.path.join(dirs['packaging_base'], options.packaging_dir) try: @@ -450,7 +446,7 @@ def main(argv): skipped = True finally: os.chdir(dirs['top']) - gbpc.RemoveTree(dirs['tmp_base'])() + del_tmpdir() if not ret and not skipped: gbp.log.info("Version '%s' imported under '%s'" % (ver_str, spec.name)) diff --git a/gbp/scripts/pq_rpm.py b/gbp/scripts/pq_rpm.py index 3d1c4bc..4dc0f17 100755 --- a/gbp/scripts/pq_rpm.py +++ b/gbp/scripts/pq_rpm.py @@ -28,7 +28,7 @@ import shutil import sys import gbp.log -import gbp.tmpfile as tempfile +from gbp.tmpfile import init_tmpdir, del_tmpdir, tempfile from gbp.config import GbpOptionParserRpm from gbp.rpm.git import GitRepositoryError, RpmGitRepository from gbp.git.modifier import GitModifier @@ -207,18 +207,16 @@ def export_patches(repo, options): GitCommand('status')(['--', spec.specdir]) -def safe_patches(queue, tmpdir_base): +def safe_patches(queue): """ Safe the current patches in a temporary directory - below 'tmpdir_base'. Also, uncompress compressed patches here. @param queue: an existing patch queue - @param tmpdir_base: base under which to create tmpdir - @return: tmpdir and a safed queue (with patches in tmpdir) + @return: safed queue (with patches in tmpdir) @rtype: tuple """ - tmpdir = tempfile.mkdtemp(dir=tmpdir_base, prefix='patchimport_') + tmpdir = tempfile.mkdtemp(prefix='patchimport_') safequeue = PatchSeries() if len(queue) > 0: @@ -300,13 +298,13 @@ def import_spec_patches(repo, options): # Put patches in a safe place if spec_treeish: - packaging_tmp = tempfile.mkdtemp(prefix='dump_', dir=options.tmp_dir) + packaging_tmp = tempfile.mkdtemp(prefix='dump_') packaging_tree = '%s:%s' % (spec_treeish, options.packaging_dir) dump_tree(repo, packaging_tmp, packaging_tree, with_submodules=False, recursive=False) spec.specdir = packaging_tmp in_queue = spec.patchseries() - queue = safe_patches(in_queue, options.tmp_dir) + queue = safe_patches(in_queue) # Do import try: gbp.log.info("Switching to branch '%s'" % pq_branch) @@ -429,8 +427,7 @@ def main(argv): try: # Create base temporary directory for this run - options.tmp_dir = tempfile.mkdtemp(dir=options.tmp_dir, - prefix='gbp-pq-rpm_') + init_tmpdir(options.tmp_dir, prefix='pq-rpm_') current = repo.get_branch() if action == "export": export_patches(repo, options) @@ -455,7 +452,7 @@ def main(argv): gbp.log.err(err) retval = 1 finally: - shutil.rmtree(options.tmp_dir, ignore_errors=True) + del_tmpdir() return retval diff --git a/gbp/tmpfile.py b/gbp/tmpfile.py index e1ad308..b90527f 100644 --- a/gbp/tmpfile.py +++ b/gbp/tmpfile.py @@ -1,6 +1,6 @@ # vim: set fileencoding=utf-8 : # -# (C) 2012 Intel Corporation <[email protected]> +# (C) 2012, 2015 Intel Corporation <[email protected]> # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -17,22 +17,37 @@ """Temporary directory handling""" import os +import shutil import tempfile from gbp.errors import GbpError -def mkdtemp(dir, **kwargs): - """Create temporary directory""" - try: - if not os.path.exists(dir): - os.makedirs(dir) - except OSError as (dummy_e, msg): - raise GbpError, "Unable to create dir %s (%s)" % (dir, msg) +_old_tempdirs = [] + +def init_tmpdir(path, prefix): + """Initialize a temporary directory structure""" try: - return os.path.abspath(tempfile.mkdtemp(dir=dir, **kwargs)) - except OSError as (dummy_e, msg): - raise GbpError, "Unable to create tmpdir under %s (%s)" % (dir, msg) + if not os.path.exists(path): + os.makedirs(path) + except OSError as err: + raise GbpError("Unable to create tmpdir %s (%s)" % (path, err)) + + tmpdir = tempfile.mkdtemp(dir=path, prefix=prefix) + + # Set newly created dir as the default value for all further tempfile + # calls + _old_tempdirs.append(tempfile.tempdir) + tempfile.tempdir = tmpdir + return tmpdir + +def del_tmpdir(): + """Remove tempdir and restore tempfile module""" + if _old_tempdirs: + if os.path.exists(tempfile.tempdir) and \ + not os.getenv('GBP_TMPFILE_NOCLEAN'): + shutil.rmtree(tempfile.tempdir) + tempfile.tempdir = _old_tempdirs.pop() # vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·: -- 1.8.4.5
>From 86f5bf9d4e64bc815f4779fb0eb0d13c701c75fe Mon Sep 17 00:00:00 2001 From: Markus Lehtonen <[email protected]> Date: Wed, 4 Feb 2015 15:41:01 +0200 Subject: [PATCH 08/14] tests.testutils: add ls_dir(), ls_tar() and ls_zip() New helper functions for listing the content of a directory, tarball and a zip archive. Signed-off-by: Markus Lehtonen <[email protected]> --- tests/testutils/__init__.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py index 6df30e2..43ecf80 100644 --- a/tests/testutils/__init__.py +++ b/tests/testutils/__init__.py @@ -5,7 +5,9 @@ from .. import context import os import shutil import subprocess +import tarfile import tempfile +import zipfile import gbp.log import gbp.errors @@ -15,7 +17,8 @@ from gbplogtester import GbpLogTester from debiangittestrepo import DebianGitTestRepo __all__ = ['GbpLogTester, DebianGitTestRepo', 'OsReleaseFile', - 'MockedChangeLog', 'get_dch_default_urgency' ] + 'MockedChangeLog', 'get_dch_default_urgency', + 'ls_dir', 'ls_tar', 'ls_zip'] class OsReleaseFile(object): """Repesents a simple file with key-value pairs""" @@ -84,3 +87,36 @@ def get_dch_default_urgency(): shutil.rmtree(tempdir) return urgency + +def ls_dir(directory, directories=True): + """List the contents of directory, recurse to subdirectories""" + contents = set() + for root, dirs, files in os.walk(directory): + prefix = '' + if root != directory: + prefix = os.path.relpath(root, directory) + '/' + contents.update(['%s%s' % (prefix, fname) for fname in files]) + if directories: + contents.update(['%s%s' % (prefix, dname) for dname in dirs]) + return contents + +def ls_tar(tarball, directories=True): + """List the contents of tar archive""" + tmpdir = tempfile.mkdtemp() + try: + tarobj = tarfile.open(tarball, 'r') + tarobj.extractall(tmpdir) + return ls_dir(tmpdir, directories) + finally: + shutil.rmtree(tmpdir) + +def ls_zip(archive, directories=True): + """List the contents of zip file""" + tmpdir = tempfile.mkdtemp() + try: + zipobj = zipfile.ZipFile(archive, 'r') + zipobj.extractall(tmpdir) + return ls_dir(tmpdir, directories) + finally: + shutil.rmtree(tmpdir) + -- 1.8.4.5
>From 1894e6c3d21a6776961e0531af8d22a84899503b Mon Sep 17 00:00:00 2001 From: Markus Lehtonen <[email protected]> Date: Thu, 26 Jun 2014 10:05:01 +0300 Subject: [PATCH 09/14] ComponentTestBase: add ls_tree() method Signed-off-by: Markus Lehtonen <[email protected]> --- tests/component/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/component/__init__.py b/tests/component/__init__.py index f6b1224..d142d6e 100644 --- a/tests/component/__init__.py +++ b/tests/component/__init__.py @@ -134,6 +134,13 @@ class ComponentTestBase(GbpLogTester): (list(extra), list(missing)) assert not extra and not missing, assert_msg + @staticmethod + def ls_tree(repo, treeish): + """List contents (blobs) in a git treeish""" + objs = repo.list_tree(treeish, True) + blobs = [obj[3] for obj in objs if obj[1] == 'blob'] + return set(blobs) + @classmethod def _check_repo_state(cls, repo, current_branch, branches, files=None, dirs=None): -- 1.8.4.5
>From 128d40e296af3521537e5596d69796eecb746150 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen <[email protected]> Date: Thu, 26 Jun 2014 13:35:19 +0300 Subject: [PATCH 10/14] ComponentTestBase: ignore system and user config User might have a system and/or user specific gbp config files. Disable these config files so that they don't affect the component tests. Signed-off-by: Markus Lehtonen <[email protected]> --- tests/component/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/component/__init__.py b/tests/component/__init__.py index d142d6e..91f4061 100644 --- a/tests/component/__init__.py +++ b/tests/component/__init__.py @@ -89,6 +89,9 @@ class ComponentTestBase(GbpLogTester): # Create a top-level tmpdir for the test cls._tmproot = tempfile.mkdtemp(prefix='gbp_%s_' % cls.__name__, dir='.') + # Prevent local config files from messing up the tests + os.environ['GBP_CONF_FILES'] = '%(top_dir)s/.gbp.conf:' \ + '%(top_dir)s/debian/gbp.conf:%(git_dir)s/gbp.conf' @classmethod def teardown_class(cls): -- 1.8.4.5
>From 5e13a3b8feb83e610d135f455e50db4a5589e69b Mon Sep 17 00:00:00 2001 From: Markus Lehtonen <[email protected]> Date: Thu, 5 Feb 2015 17:13:11 +0200 Subject: [PATCH 11/14] Command: redirect stderr to stdout Makes it possible to get all command output from stdout. Signed-off-by: Markus Lehtonen <[email protected]> --- gbp/command_wrappers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gbp/command_wrappers.py b/gbp/command_wrappers.py index f1312e4..3691151 100644 --- a/gbp/command_wrappers.py +++ b/gbp/command_wrappers.py @@ -63,7 +63,8 @@ class Command(object): log.debug("%s %s %s" % (self.cmd, self.args, args)) self.stderr = '' - stderr_arg = subprocess.PIPE if self.capture_stderr else None + stderr_arg = subprocess.PIPE if self.capture_stderr else \ + subprocess.STDOUT cmd = [ self.cmd ] + self.args + args if self.shell: # subprocess.call only cares about the first argument if shell=True -- 1.8.4.5
>From 5bf9e9cacb5fb9479fb7543faf94b05aac540028 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen <[email protected]> Date: Thu, 5 Feb 2015 17:31:32 +0200 Subject: [PATCH 12/14] Command: pipe stdout to sys.stdout Pipe stdout of the (child) command to sys.stdout of the caller. This makes Python nose to correctly capture the output of the child command, too, which in turn suppresses a lot of spurious output when running nosetests. Signed-off-by: Markus Lehtonen <[email protected]> --- gbp/command_wrappers.py | 13 +++++++++++++ tests/test_PristineTar.py | 2 ++ 2 files changed, 15 insertions(+) diff --git a/gbp/command_wrappers.py b/gbp/command_wrappers.py index 3691151..de67a1f 100644 --- a/gbp/command_wrappers.py +++ b/gbp/command_wrappers.py @@ -23,6 +23,9 @@ import subprocess import os import os.path import signal +import sys +import tempfile + import gbp.log as log class CommandExecFailed(Exception): @@ -65,6 +68,11 @@ class Command(object): self.stderr = '' stderr_arg = subprocess.PIPE if self.capture_stderr else \ subprocess.STDOUT + # Pipe stdout of the child command to sys.stdout so that std* capture + # in nosetests catches the output of the command, too. We need to use + # a temporary file if sys.stdout is not a real file. + stdout_arg = sys.stdout if hasattr(sys.stdout, 'fileno') else \ + tempfile.TemporaryFile() cmd = [ self.cmd ] + self.args + args if self.shell: # subprocess.call only cares about the first argument if shell=True @@ -74,8 +82,13 @@ class Command(object): shell=self.shell, env=self.env, preexec_fn=default_sigpipe, + stdout=stdout_arg, stderr=stderr_arg) (dummy, stderr) = popen.communicate() + # Write stdout content back to sys.stdout if the tempfile hack was used + if stdout_arg != sys.stdout: + stdout_arg.seek(0) + sys.stdout.write(stdout_arg.read()) self.stderr = stderr return popen.returncode diff --git a/tests/test_PristineTar.py b/tests/test_PristineTar.py index e837ef8..71f13fb 100644 --- a/tests/test_PristineTar.py +++ b/tests/test_PristineTar.py @@ -86,6 +86,7 @@ def test_pristine_tar_commit(): >>> import gbp.deb.git >>> repo = gbp.deb.git.DebianGitRepository(repo_dir) >>> repo.pristine_tar.commit('../upstream_1.0.orig.tar.gz', 'upstream') + pristine-tar: committed upstream_1.0.orig.tar.gz.delta to branch pristine-tar """ def test_pristine_has_commit(): @@ -120,6 +121,7 @@ def test_pristine_tar_checkout(): >>> import gbp.deb.git >>> repo = gbp.deb.git.DebianGitRepository(repo_dir) >>> repo.pristine_tar.checkout('upstream', '1.0', 'gzip', '..') + pristine-tar: successfully generated ../upstream_1.0.orig.tar.gz """ -- 1.8.4.5
>From 54a035c27174c539da5ceebdf95321a79dd0b76e Mon Sep 17 00:00:00 2001 From: Markus Lehtonen <[email protected]> Date: Wed, 4 Feb 2015 17:36:47 +0200 Subject: [PATCH 13/14] gbp.rpm: use format_str for composing tag name Signed-off-by: Markus Lehtonen <[email protected]> --- gbp/rpm/git.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/gbp/rpm/git.py b/gbp/rpm/git.py index c7cc023..bc9e020 100644 --- a/gbp/rpm/git.py +++ b/gbp/rpm/git.py @@ -17,6 +17,8 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import re +from gbp.format import format_str +from gbp.errors import GbpError from gbp.git import GitRepository, GitRepositoryError from gbp.pkg.pristinetar import PristineTar from gbp.rpm import compose_version_str @@ -42,7 +44,7 @@ class RpmGitRepository(GitRepository): """ try: tag = self.version_to_tag(format, str_fields) - except KeyError: + except GbpError: return None if self.has_tag(tag): # new tags are injective # dereference to a commit object @@ -65,8 +67,9 @@ class RpmGitRepository(GitRepository): >>> RpmGitRepository.version_to_tag("%(vendor)s/v%(version)s", dict(upstreamversion='1.0', release='2', vendor="myvendor")) 'myvendor/v1.0-2' """ - version_tag = format % dict(str_fields, - version=compose_version_str(str_fields)) + version_tag = format_str(format, + dict(str_fields, + version=compose_version_str(str_fields))) return RpmGitRepository._sanitize_tag(version_tag) @staticmethod -- 1.8.4.5
>From f8cc4a1616e9180b881297fc54b4311691a91f0d Mon Sep 17 00:00:00 2001 From: Markus Lehtonen <[email protected]> Date: Thu, 12 Jan 2012 15:39:12 +0200 Subject: [PATCH 14/14] Introduce gbp-buildpackage-rpm tool Initial version of gbp-buildpackage-rpm - tool for building RPM packages from git. Bigger differences to the (Debian) gbp-buildpackage are that buildpackage-rpm always does the build under a separate build directory (../rpmbuild by default) and that packaging files of a non-native package may be maintained in an orphan branch, without any sourcecode. Also, whether the package is native or not is determined by the existence of an upstream branch or from an explicit config setting. Signed-off-by: Markus Lehtonen <[email protected]> Signed-off-by: Ed Bartosh <[email protected]> Signed-off-by: Zhang Qiang <[email protected]> --- debian/git-buildpackage-rpm.install | 1 + gbp/config.py | 21 +- gbp/scripts/buildpackage_rpm.py | 580 +++++++++++++++++++++++++ gbp/scripts/pq_rpm.py | 2 +- tests/component/rpm/__init__.py | 21 +- tests/component/rpm/data | 2 +- tests/component/rpm/test_buildpackage_rpm.py | 611 +++++++++++++++++++++++++++ 7 files changed, 1228 insertions(+), 10 deletions(-) create mode 100755 gbp/scripts/buildpackage_rpm.py create mode 100644 tests/component/rpm/test_buildpackage_rpm.py diff --git a/debian/git-buildpackage-rpm.install b/debian/git-buildpackage-rpm.install index 2568b4c..5178cb3 100644 --- a/debian/git-buildpackage-rpm.install +++ b/debian/git-buildpackage-rpm.install @@ -1,3 +1,4 @@ usr/lib/python2.?/dist-packages/gbp/rpm/ usr/lib/python2.7/dist-packages/gbp/scripts/import_srpm.py usr/lib/python2.7/dist-packages/gbp/scripts/pq_rpm.py +usr/lib/python2.7/dist-packages/gbp/scripts/buildpackage_rpm.py diff --git a/gbp/config.py b/gbp/config.py index 2ca7227..d5acbbe 100644 --- a/gbp/config.py +++ b/gbp/config.py @@ -102,6 +102,7 @@ class GbpOptionParser(OptionParser): 'sign-tags' : 'False', 'force-create' : 'False', 'no-create-orig' : 'False', + 'cleaner' : '/bin/true', 'keyid' : '', 'posttag' : '', 'postbuild' : '', @@ -561,7 +562,6 @@ class GbpOptionParserDebian(GbpOptionParser): defaults = dict(GbpOptionParser.defaults) defaults.update( { 'builder' : 'debuild -i -I', - 'cleaner' : '/bin/true', } ) @@ -576,7 +576,14 @@ class GbpOptionParserRpm(GbpOptionParser): 'packaging-branch' : 'master', 'packaging-dir' : '', 'packaging-tag' : 'packaging/%(version)s', + 'packaging-tag-msg' : '%(pkg)s %(vendor)s release '\ + '%(version)s', 'spec-file' : '', + 'export-dir' : '../rpmbuild', + 'native' : 'auto', + 'builder' : 'rpmbuild', + 'export-specdir' : 'SPECS', + 'export-sourcedir' : 'SOURCES', }) help = dict(GbpOptionParser.help) @@ -596,9 +603,21 @@ class GbpOptionParserRpm(GbpOptionParser): 'packaging-tag': "Format string for packaging tags, RPM counterpart of the " "'debian-tag' option, default is '%(packaging-tag)s'", + 'packaging-tag-msg': + ("Format string for packaging tag messages, " + "default is '%(packaging-tag-msg)s'"), 'spec-file': "Spec file to use, causes the packaging-dir option to be " "ignored, default is '%(spec-file)s'", + 'native': + "Treat this package as native, default is '%(native)s'", + 'export-specdir': + "Subdir (under EXPORT_DIR) where package spec file is " + "exported default is '%(export-specdir)s'", + 'export-sourcedir': + "Subdir (under EXPORT_DIR) where packaging sources (other than " + "the spec file) are exported, default is " + "'%(export-sourcedir)s'", }) # vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·: diff --git a/gbp/scripts/buildpackage_rpm.py b/gbp/scripts/buildpackage_rpm.py new file mode 100755 index 0000000..6e80a04 --- /dev/null +++ b/gbp/scripts/buildpackage_rpm.py @@ -0,0 +1,580 @@ +# vim: set fileencoding=utf-8 : +# +# (C) 2006-2011 Guido Guenther <[email protected]> +# (C) 2012-2015 Intel Corporation <[email protected]> +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +"""Build an RPM package out of a Git repository""" + +import ConfigParser +import os +import shutil +import sys + +import gbp.log +import gbp.notifications +import gbp.rpm as rpm +from gbp.command_wrappers import Command, RunAtCommand, CommandExecFailed +from gbp.config import GbpOptionParserRpm, GbpOptionGroup +from gbp.errors import GbpError +from gbp.format import format_str +from gbp.pkg import compressor_opts +from gbp.rpm.git import GitRepositoryError, RpmGitRepository +from gbp.rpm.policy import RpmPkgPolicy +from gbp.tmpfile import init_tmpdir, del_tmpdir, tempfile +from gbp.scripts.common.buildpackage import (index_name, wc_name, + git_archive_submodules, + git_archive_single, dump_tree, + write_wc, drop_index) +from gbp.scripts.pq_rpm import parse_spec + + +def makedir(path): + """Create directory""" + try: + if not os.path.exists(path): + os.makedirs(path) + except OSError as err: + raise GbpError("Cannot create dir %s: %s" % (path, err)) + return path + + +def git_archive(repo, spec, output_dir, treeish, comp_level, + with_submodules): + "Create a compressed orig tarball in output_dir using git_archive" + comp_opts = '' + if spec.orig_src['compression']: + comp_opts = compressor_opts[spec.orig_src['compression']][0] + + output = os.path.join(output_dir, spec.orig_src['filename']) + + try: + if repo.has_submodules(treeish) and with_submodules: + repo.update_submodules() + git_archive_submodules(repo, treeish, output, + spec.orig_src['prefix'], + spec.orig_src['compression'], + comp_level, comp_opts, + spec.orig_src['archive_fmt']) + + else: + git_archive_single(treeish, output, spec.orig_src['prefix'], + spec.orig_src['compression'], comp_level, + comp_opts, spec.orig_src['archive_fmt']) + except (GitRepositoryError, CommandExecFailed): + gbp.log.err("Error generating submodules' archives") + return False + return True + + +def prepare_upstream_tarball(repo, spec, options, output_dir): + """Make sure we have an upstream tarball""" + # look in tarball_dir first, if found force a symlink to it + orig_file = spec.orig_src['filename'] + if options.tarball_dir: + gbp.log.debug("Looking for orig tarball '%s' at '%s'" % + (orig_file, options.tarball_dir)) + if not RpmPkgPolicy.symlink_orig(orig_file, options.tarball_dir, + output_dir, force=True): + gbp.log.info("Orig tarball '%s' not found at '%s'" % + (orig_file, options.tarball_dir)) + else: + gbp.log.info("Orig tarball '%s' found at '%s'" % + (orig_file, options.tarball_dir)) + + # build an orig unless the user forbids it, always build (and overwrite + # pre-existing) if user forces it + if options.force_create or (not options.no_create_orig and not + RpmPkgPolicy.has_orig(orig_file, output_dir)): + if not pristine_tar_build_orig(repo, orig_file, output_dir, options): + upstream_tree = git_archive_build_orig(repo, spec, output_dir, + options) + if options.pristine_tar_commit: + if repo.pristine_tar.has_commit(orig_file): + gbp.log.debug("%s already on pristine tar branch" % + orig_file) + else: + archive = os.path.join(output_dir, orig_file) + gbp.log.debug("Adding %s to pristine-tar branch" % + archive) + repo.pristine_tar.commit(archive, upstream_tree) + + +def pristine_tar_build_orig(repo, orig_file, output_dir, options): + """Build orig using pristine-tar""" + if options.pristine_tar: + if not repo.has_branch(repo.pristine_tar_branch): + gbp.log.warn('Pristine-tar branch "%s" not found' % + repo.pristine_tar.branch) + try: + repo.pristine_tar.checkout(os.path.join(output_dir, orig_file)) + return True + except CommandExecFailed: + if options.pristine_tar_commit: + gbp.log.debug("pristine-tar checkout failed, " + "will commit tarball due to " + "'--pristine-tar-commit'") + elif not options.force_create: + raise + return False + +def get_upstream_tree(repo, spec, options): + """Determine the upstream tree from the given options""" + if options.upstream_tree.upper() == 'TAG': + tag_str_fields = {'upstreamversion': spec.upstreamversion, + 'vendor': 'Upstream'} + upstream_tree = repo.version_to_tag(options.upstream_tag, + tag_str_fields) + elif options.upstream_tree.upper() == 'BRANCH': + if not repo.has_branch(options.upstream_branch): + raise GbpError("%s is not a valid branch" % options.upstream_branch) + upstream_tree = options.upstream_branch + else: + upstream_tree = get_tree(repo, options.upstream_tree) + if not repo.has_treeish(upstream_tree): + raise GbpError('Invalid upstream treeish %s' % upstream_tree) + return upstream_tree + + +def get_tree(repo, tree_name): + """ + Get/create a tree-ish to be used for exporting and diffing. Accepts + special keywords for git index and working copies. + """ + try: + if tree_name == index_name: + # Write a tree of the index + tree = repo.write_tree() + elif tree_name == wc_name: + # Write a tree of the working copy + tree = write_wc(repo) + else: + tree = tree_name + except GitRepositoryError as err: + raise GbpError(err) + if not repo.has_treeish(tree): + raise GbpError('Invalid treeish object %s' % tree) + + return tree + + +def get_current_branch(repo): + """Get the currently checked-out branch""" + try: + branch = repo.get_branch() + except GitRepositoryError: + branch = None + return branch + + +def git_archive_build_orig(repo, spec, output_dir, options): + """ + Build orig tarball using git-archive + + @param repo: our git repository + @type repo: L{RpmGitRepository} + @param spec: spec file of the package + @type spec: L{SpecFile} + @param output_dir: where to put the tarball + @type output_dir: C{Str} + @param options: the parsed options + @type options: C{dict} of options + @return: the tree we built the tarball from + @rtype: C{str} + """ + upstream_tree = get_upstream_tree(repo, spec, options) + gbp.log.info("%s does not exist, creating from '%s'" % \ + (spec.orig_src['filename'], upstream_tree)) + if spec.orig_src['compression']: + gbp.log.debug("Building upstream source archive with compression "\ + "'%s -%s'" % (spec.orig_src['compression'], + options.comp_level)) + if not git_archive(repo, spec, output_dir, upstream_tree, + options.comp_level, options.with_submodules): + raise GbpError("Cannot create upstream tarball at '%s'" % \ + output_dir) + return upstream_tree + + +def is_native(repo, options): + """Determine whether a package is native or non-native""" + if options.native.is_auto(): + if repo.has_branch(options.upstream_branch): + return False + # Check remotes, too + for remote_branch in repo.get_remote_branches(): + remote, branch = remote_branch.split('/', 1) + if branch == options.upstream_branch: + gbp.log.debug("Found upstream branch '%s' from remote '%s'" % + (remote, branch)) + return False + return True + + return options.native.is_on() + + +def setup_builder(options, builder_args): + """Setup args and options for builder script""" + if options.builder == 'rpmbuild': + if len(builder_args) == 0: + builder_args.append('-ba') + builder_args.extend([ + '--define "_topdir %s"' % os.path.abspath(options.export_dir), + '--define "_specdir %%_topdir/%s"' % options.export_specdir, + '--define "_sourcedir %%_topdir/%s"' % options.export_sourcedir]) + + +def create_packaging_tag(repo, commit, spec, options): + """Create a packaging/release Git tag""" + version_dict = dict(spec.version, + version=rpm.compose_version_str(spec.version)) + + # Compose tag name and message + tag_name_fields = dict(version_dict, vendor=options.vendor) + tag_name = repo.version_to_tag(options.packaging_tag, tag_name_fields) + + tag_msg = format_str(options.packaging_tag_msg, + dict(version_dict, pkg=spec.name, + vendor=options.vendor)) + + # (Re-)create Git tag + if options.retag and repo.has_tag(tag_name): + repo.delete_tag(tag_name) + repo.create_tag(name=tag_name, msg=tag_msg, sign=options.sign_tags, + keyid=options.keyid, commit=commit) + return tag_name + + +def disable_hooks(options): + """Disable all hooks (except for builder)""" + for hook in ['cleaner', 'postexport', 'prebuild', 'postbuild', 'posttag']: + if getattr(options, hook): + gbp.log.info("Disabling '%s' hook" % hook) + setattr(options, hook, '') + + +def build_parser(name, prefix=None): + """Construct config/option parser""" + try: + parser = GbpOptionParserRpm(command=os.path.basename(name), + prefix=prefix) + except ConfigParser.ParsingError as err: + gbp.log.err(err) + return None + + tag_group = GbpOptionGroup(parser, "tag options", + "options related to git tag creation") + branch_group = GbpOptionGroup(parser, "branch options", + "branch layout options") + cmd_group = GbpOptionGroup(parser, "external command options", + "how and when to invoke external commands and hooks") + orig_group = GbpOptionGroup(parser, "orig tarball options", + "options related to the creation of the orig tarball") + export_group = GbpOptionGroup(parser, "export build-tree options", + "alternative build tree related options") + parser.add_option_group(tag_group) + parser.add_option_group(orig_group) + parser.add_option_group(branch_group) + parser.add_option_group(cmd_group) + parser.add_option_group(export_group) + + parser.add_boolean_config_file_option(option_name="ignore-new", + dest="ignore_new") + parser.add_option("--git-verbose", action="store_true", dest="verbose", + default=False, help="verbose command execution") + parser.add_config_file_option(option_name="tmp-dir", dest="tmp_dir") + parser.add_config_file_option(option_name="color", dest="color", + type='tristate') + parser.add_config_file_option(option_name="color-scheme", + dest="color_scheme") + parser.add_config_file_option(option_name="notify", dest="notify", + type='tristate') + parser.add_config_file_option(option_name="vendor", action="store", + dest="vendor") + parser.add_config_file_option(option_name="native", dest="native", + type='tristate') + tag_group.add_option("--git-tag", action="store_true", dest="tag", + default=False, + help="create a tag after a successful build") + tag_group.add_option("--git-tag-only", action="store_true", dest="tag_only", + default=False, + help="don't build, only tag and run the posttag hook") + tag_group.add_option("--git-retag", action="store_true", dest="retag", + default=False, help="don't fail if the tag already exists") + tag_group.add_boolean_config_file_option(option_name="sign-tags", + dest="sign_tags") + tag_group.add_config_file_option(option_name="keyid", dest="keyid") + tag_group.add_config_file_option(option_name="packaging-tag", + dest="packaging_tag") + tag_group.add_config_file_option(option_name="packaging-tag-msg", + dest="packaging_tag_msg") + tag_group.add_config_file_option(option_name="upstream-tag", + dest="upstream_tag") + orig_group.add_config_file_option(option_name="upstream-tree", + dest="upstream_tree") + orig_group.add_boolean_config_file_option(option_name="pristine-tar", + dest="pristine_tar") + orig_group.add_boolean_config_file_option(option_name="pristine-tar-commit", + dest="pristine_tar_commit") + orig_group.add_config_file_option(option_name="force-create", + dest="force_create", action="store_true", + help="force creation of upstream source tarball") + orig_group.add_config_file_option(option_name="no-create-orig", + dest="no_create_orig", action="store_true", + help="don't create upstream source tarball") + orig_group.add_config_file_option(option_name="tarball-dir", + dest="tarball_dir", type="path", + help="location to look for external tarballs") + orig_group.add_config_file_option(option_name="compression-level", + dest="comp_level", + help="Compression level, default is " + "'%(compression-level)s'") + branch_group.add_config_file_option(option_name="upstream-branch", + dest="upstream_branch") + branch_group.add_config_file_option(option_name="packaging-branch", + dest="packaging_branch") + branch_group.add_boolean_config_file_option(option_name = "ignore-branch", + dest="ignore_branch") + branch_group.add_boolean_config_file_option(option_name = "submodules", + dest="with_submodules") + cmd_group.add_config_file_option(option_name="builder", dest="builder", + help="command to build the package, default is " + "'%(builder)s'") + cmd_group.add_config_file_option(option_name="cleaner", dest="cleaner", + help="command to clean the working copy, default is " + "'%(cleaner)s'") + cmd_group.add_config_file_option(option_name="prebuild", dest="prebuild", + help="command to run before a build, default is " + "'%(prebuild)s'") + cmd_group.add_config_file_option(option_name="postexport", + dest="postexport", + help="command to run after exporting the source tree, " + "default is '%(postexport)s'") + cmd_group.add_config_file_option(option_name="postbuild", dest="postbuild", + help="hook run after a successful build, default is " + "'%(postbuild)s'") + cmd_group.add_config_file_option(option_name="posttag", dest="posttag", + help="hook run after a successful tag operation, default " + "is '%(posttag)s'") + cmd_group.add_boolean_config_file_option(option_name="hooks", dest="hooks") + export_group.add_config_file_option(option_name="export-dir", + dest="export_dir", type="path", + help="Build topdir, also export the sources under " + "EXPORT_DIR, default is '%(export-dir)s'") + export_group.add_config_file_option(option_name="export-specdir", + dest="export_specdir", type="path") + export_group.add_config_file_option(option_name="export-sourcedir", + dest="export_sourcedir", type="path") + export_group.add_config_file_option("export", dest="export", + metavar="TREEISH", + help="export treeish object TREEISH, default is " + "'%(export)s'") + export_group.add_config_file_option(option_name="packaging-dir", + dest="packaging_dir") + export_group.add_config_file_option(option_name="spec-file", + dest="spec_file") + return parser + + +def parse_args(argv, prefix): + """Parse config and command line arguments""" + args = [arg for arg in argv[1:] if arg.find('--%s' % prefix) == 0] + builder_args = [arg for arg in argv[1:] if arg.find('--%s' % prefix) == -1] + + # We handle these although they don't have a --git- prefix + for arg in [ "--help", "-h", "--version" ]: + if arg in builder_args: + args.append(arg) + + parser = build_parser(argv[0], prefix=prefix) + if not parser: + return None, None, None + options, args = parser.parse_args(args) + + gbp.log.setup(options.color, options.verbose, options.color_scheme) + if not options.hooks: + disable_hooks(options) + if options.retag: + if not options.tag and not options.tag_only: + gbp.log.err("'--%sretag' needs either '--%stag' or '--%stag-only'" % + (prefix, prefix, prefix)) + return None, None, None + + return options, args, builder_args + + +def main(argv): + """Entry point for gbp-buildpackage-rpm""" + retval = 0 + prefix = "git-" + spec = None + + options, gbp_args, builder_args = parse_args(argv, prefix) + + if not options: + return 1 + + try: + repo = RpmGitRepository(os.path.curdir) + except GitRepositoryError: + gbp.log.err("%s is not a git repository" % (os.path.abspath('.'))) + return 1 + + branch = get_current_branch(repo) + + try: + init_tmpdir(options.tmp_dir, prefix='buildpackage-rpm') + + tree = get_tree(repo, options.export) + spec = parse_spec(options, repo, treeish=tree) + + Command(options.cleaner, shell=True)() + if not options.ignore_new: + ret, out = repo.is_clean() + if not ret: + gbp.log.err("You have uncommitted changes in your source tree:") + gbp.log.err(out) + raise GbpError("Use --git-ignore-new to ignore.") + + if not options.ignore_new and not options.ignore_branch: + if branch != options.packaging_branch: + gbp.log.err("You are not on branch '%s' but on '%s'" % + (options.packaging_branch, branch)) + raise GbpError("Use --git-ignore-branch to ignore or " + "--git-packaging-branch to set the branch name.") + + # Dump from git to a temporary directory: + packaging_tree = '%s:%s' % (tree, options.packaging_dir) + dump_dir = tempfile.mkdtemp(prefix='packaging_') + gbp.log.debug("Dumping packaging files to '%s'" % dump_dir) + if not dump_tree(repo, dump_dir, packaging_tree, False, False): + raise GbpError + # Re-parse spec from dump dir to get version etc. + spec = rpm.SpecFile(os.path.join(dump_dir, spec.specfile)) + + if not options.tag_only: + # Setup builder opts + setup_builder(options, builder_args) + + # Prepare final export dirs + export_dir = makedir(options.export_dir) + source_dir = makedir(os.path.join(export_dir, + options.export_sourcedir)) + spec_dir = makedir(os.path.join(export_dir, options.export_specdir)) + + # Move packaging files to final export dir + gbp.log.debug("Exporting packaging files from '%s' to '%s'" % + (dump_dir, export_dir)) + for fname in os.listdir(dump_dir): + src = os.path.join(dump_dir, fname) + if fname == spec.specfile: + dst = os.path.join(spec_dir, fname) + else: + dst = os.path.join(source_dir, fname) + try: + shutil.copy2(src, dst) + except IOError as err: + raise GbpError("Error exporting packaging files: %s" % err) + spec.specdir = os.path.abspath(spec_dir) + + # Get/build the orig tarball + if is_native(repo, options): + if spec.orig_src: + # Just build source archive from the exported tree + gbp.log.info("Creating (native) source archive %s from '%s'" + % (spec.orig_src['filename'], tree)) + if spec.orig_src['compression']: + gbp.log.debug("Building source archive with " + "compression '%s -%s'" % + (spec.orig_src['compression'], + options.comp_level)) + if not git_archive(repo, spec, source_dir, tree, + options.comp_level, + options.with_submodules): + raise GbpError("Cannot create source tarball at '%s'" % + source_dir) + # Non-native packages: create orig tarball from upstream + elif spec.orig_src: + prepare_upstream_tarball(repo, spec, options, source_dir) + + # Run postexport hook + if options.postexport: + RunAtCommand(options.postexport, shell=True, + extra_env={'GBP_GIT_DIR': repo.git_dir, + 'GBP_TMP_DIR': export_dir} + )(dir=export_dir) + # Do actual build + if not options.tag_only: + if options.prebuild: + RunAtCommand(options.prebuild, shell=True, + extra_env={'GBP_GIT_DIR': repo.git_dir, + 'GBP_BUILD_DIR': export_dir} + )(dir=export_dir) + + # Finally build the package: + if options.builder.startswith("rpmbuild"): + builder_args.append(os.path.join(spec.specdir, + spec.specfile)) + else: + builder_args.append(spec.specfile) + RunAtCommand(options.builder, builder_args, shell=True, + extra_env={'GBP_BUILD_DIR': export_dir} + )(dir=export_dir) + if options.postbuild: + changes = os.path.abspath("%s/%s.changes" % (source_dir, + spec.name)) + gbp.log.debug("Looking for changes file %s" % changes) + Command(options.postbuild, shell=True, + extra_env={'GBP_CHANGES_FILE': changes, + 'GBP_BUILD_DIR': export_dir})() + + # Tag (note: tags the exported version) + if options.tag or options.tag_only: + gbp.log.info("Tagging %s" % rpm.compose_version_str(spec.version)) + tag = create_packaging_tag(repo, tree, spec, options) + if options.posttag: + sha = repo.rev_parse("%s^{}" % tag) + Command(options.posttag, shell=True, + extra_env={'GBP_TAG': tag, + 'GBP_BRANCH': branch, + 'GBP_SHA1': sha})() + + except CommandExecFailed: + retval = 1 + except GitRepositoryError as err: + gbp.log.err("Git command failed: %s" % err) + retval = 1 + except GbpError, err: + if len(err.__str__()): + gbp.log.err(err) + retval = 1 + finally: + drop_index() + del_tmpdir() + + if not options.tag_only: + if spec and options.notify: + summary = "Gbp-rpm %s" % ["failed", "successful"][not retval] + message = ("Build of %s %s %s" % (spec.name, + rpm.compose_version_str(spec.version), + ["failed", "succeeded"][not retval])) + if not gbp.notifications.notify(summary, message, options.notify): + gbp.log.err("Failed to send notification") + retval = 1 + + return retval + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/gbp/scripts/pq_rpm.py b/gbp/scripts/pq_rpm.py index 4dc0f17..2359bcf 100755 --- a/gbp/scripts/pq_rpm.py +++ b/gbp/scripts/pq_rpm.py @@ -157,7 +157,6 @@ def parse_spec(options, repo, treeish=None): """ try: if options.spec_file: - options.packaging_dir = os.path.dirname(options.spec_file) if not treeish: spec = SpecFile(options.spec_file) else: @@ -173,6 +172,7 @@ def parse_spec(options, repo, treeish=None): raise GbpError("Can't parse spec: %s" % err) relpath = spec.specpath if treeish else os.path.relpath(spec.specpath, repo.path) + options.packaging_dir = os.path.dirname(relpath) gbp.log.debug("Using '%s' from '%s'" % (relpath, treeish or 'working copy')) return spec diff --git a/tests/component/rpm/__init__.py b/tests/component/rpm/__init__.py index b5be3e7..f8a699d 100644 --- a/tests/component/rpm/__init__.py +++ b/tests/component/rpm/__init__.py @@ -42,10 +42,9 @@ class RepoManifest(object): def projects_iter(self): """Return an iterator over projects""" for prj_e in self._doc.getElementsByTagName('project'): - branches = {} + branches = [] for br_e in prj_e.getElementsByTagName('branch'): - rev = br_e.getAttribute('revision') - branches[br_e.getAttribute('name')] = rev + branches.append(dict(br_e.attributes.items())) yield prj_e.getAttribute('name'), branches @@ -63,7 +62,7 @@ class RpmRepoTestBase(ComponentTestBase): """Baseclass for tests run in a Git repository with packaging data""" @classmethod - def setup_class(cls): + def setup_class(cls, mangle_branch_names=True): """Initializations only made once per test run""" super(RpmRepoTestBase, cls).setup_class() cls.manifest = RepoManifest(os.path.join(RPM_TEST_DATA_DIR, @@ -89,9 +88,17 @@ class RpmRepoTestBase(ComponentTestBase): # Fetch all remote refs of the orig repo, too repo.fetch('origin', tags=True, refspec='refs/remotes/*:refs/upstream/*') - for branch, rev in brs.iteritems(): - repo.create_branch(branch, rev) - repo.force_head('master', hard=True) + master_branch = 'master' + for branch in brs: + if mangle_branch_names: + branch_name = branch['name'] + else: + branch_name = branch['orig_name'] + if branch['name'] == 'master': + master_branch = branch_name + repo.create_branch(branch_name, branch['revision']) + repo.set_branch(master_branch) + repo.force_head('HEAD', hard=True) cls.orig_repos[prj] = repo @classmethod diff --git a/tests/component/rpm/data b/tests/component/rpm/data index bae44dd..ff090c1 160000 --- a/tests/component/rpm/data +++ b/tests/component/rpm/data @@ -1 +1 @@ -Subproject commit bae44ddc98ae0ed15ae078cb7c2fc597dee48da5 +Subproject commit ff090c1cf946e3df67795ce3c1437ec6246dbf37 diff --git a/tests/component/rpm/test_buildpackage_rpm.py b/tests/component/rpm/test_buildpackage_rpm.py new file mode 100644 index 0000000..ad8c8e4 --- /dev/null +++ b/tests/component/rpm/test_buildpackage_rpm.py @@ -0,0 +1,611 @@ +# vim: set fileencoding=utf-8 : +# +# (C) 2013-2015 Intel Corporation <[email protected]> +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +"""Unit tests for the gbp-buildpackage-rpm tool""" + +import glob +import mock +import os +import re +import shutil +import stat +import subprocess + +from nose.tools import assert_raises, eq_, ok_ # pylint: disable=E0611 + +from gbp.git import GitRepository +from gbp.scripts.buildpackage_rpm import main as gbp_rpm +from tests.component.rpm import RpmRepoTestBase, RPM_TEST_DATA_DIR +from tests.testutils import ls_tar, ls_zip + +# Disable "Method could be a function warning" +# pylint: disable=R0201 +# Disable "Too many public methods" +# pylint: disable=R0904 + + +DATA_DIR = os.path.join(RPM_TEST_DATA_DIR, 'rpm') +ORIG_DATA_DIR = os.path.join(RPM_TEST_DATA_DIR, 'orig') + +MOCK_NOTIFICATIONS = [] + + +def mock_gbp(args): + """Wrapper for gbp-buildpackage-rpm""" + return gbp_rpm(['arg0', '--git-notify=off'] + args + + ['-ba', '--clean', '--target=noarch', '--nodeps']) + +def mock_notify(summary, message, notify_opt): + """Mock notification system""" + # Auto will succeed + if notify_opt.is_auto(): + MOCK_NOTIFICATIONS.append((summary, message)) + return True + # Otherwise fail + return False + + +class TestGbpRpm(RpmRepoTestBase): + """Basic tests for git-rpm-ch""" + + @staticmethod + def ls_rpm(rpm): + """List the contents of an rpm package""" + args = ['rpm', '-q', '--qf', + '[%{FILEDIGESTS %{FILEMODES} %{FILENAMES}\n]', '-p'] + popen = subprocess.Popen(args + [rpm], stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = popen.communicate() + if popen.returncode: + raise Exception("Failed to get file metadata for %s: %s" % + (rpm, stderr)) + return sorted([(nam, mod, dig) for dig, mod, nam in + [lin.split(None, 2) for lin in stdout.splitlines()]]) + + @staticmethod + def check_rpms(directory): + """Check build results""" + # Only check files, at least for now + files = glob.glob(directory + '/*rpm') + assert files, "No rpms (%s)found in %s" % (files, directory) + for path in files: + ref_file = os.path.join(DATA_DIR, os.path.basename(path)) + eq_(TestGbpRpm.ls_rpm(path), TestGbpRpm.ls_rpm(ref_file)) + + @staticmethod + def check_and_rm_file(filepath, content): + """Check file content and remove it""" + with open(filepath) as fobj: + eq_(fobj.read(), content) + os.unlink(filepath) + + @classmethod + def setup_class(cls, **kwargs): + """Setup unit tests""" + # Don't mangle branch names so that we're able to build the packages + super(TestGbpRpm, cls).setup_class(mangle_branch_names=False, **kwargs) + + def test_invalid_args(self): + """Check graceful exit when called with invalid args""" + GitRepository.create('.') + with assert_raises(SystemExit): + mock_gbp(['--git-invalid-arg']) + + def test_outside_repo(self): + """Run outside a git repository""" + eq_(mock_gbp([]), 1) + self._check_log(0, 'gbp:error: %s is not a git repository' % + os.path.abspath('.')) + + def test_invalid_config_file(self): + """Test invalid config file""" + # Create and commit dummy invalid config file + repo = GitRepository.create('.') + with open('.gbp.conf', 'w') as conffd: + conffd.write('foobar\n') + repo.add_files('.gbp.conf') + repo.commit_all('Add conf') + eq_(mock_gbp([]), 1) + self._check_log(0, 'gbp:error: File contains no section headers.') + + def test_native_build(self): + """Basic test of native pkg""" + self.init_test_repo('gbp-test-native') + eq_(mock_gbp([]), 0) + self.check_rpms('../rpmbuild/RPMS/*') + shutil.rmtree('../rpmbuild') + + eq_(mock_gbp(['--git-native=off']), 1) + self._check_log(0, 'gbp:error: Invalid upstream treeish upstream/') + + def test_native_build2(self): + """Basic test of another native pkg""" + self.init_test_repo('gbp-test-native2') + eq_(mock_gbp([]), 0) + self.check_rpms('../rpmbuild/RPMS/*') + + def test_non_native_build(self): + """Basic test of non-native pkg""" + repo = self.init_test_repo('gbp-test') + eq_(mock_gbp([]), 0) + self.check_rpms('../rpmbuild/RPMS/*') + + # Test nativity guessing from remote branches by creating a dummy + # remote branch + repo.update_ref('refs/remotes/fooremote/foobranch', + 'srcdata/gbp-test/upstream') + eq_(mock_gbp(['--git-upstream-branch=foobranch']), 0) + + def test_option_native(self): + """Test the --git-native option""" + self.init_test_repo('gbp-test2') + eq_(mock_gbp([]), 0) + self.check_rpms('../rpmbuild/RPMS/*') + shutil.rmtree('../rpmbuild') + + # Building this pkg should succeed, but no patches generated, + # only one "manually maintained" patch + eq_(mock_gbp(['--git-native=on']), 0) + self.check_rpms('../rpmbuild/RPMS/*') + eq_(len(glob.glob('../rpmbuild/SOURCES/*patch')), 1) + + def test_options_ignore(self): + """Test the --git-ignore-[new|untracked] options""" + self.init_test_repo('gbp-test-native') + + # Create an untracked file + with open('untracked-file', 'w') as fobj: + fobj.write('this file is not tracked\n') + + # Modify tracked file + with open('README', 'a') as fobj: + fobj.write('new stuff\n') + + eq_(mock_gbp([]), 1) + eq_(mock_gbp(['--git-ignore-new']), 0) + + @mock.patch('gbp.notifications.notify', mock_notify) + def test_option_notify(self): + """Test the --git-notify option""" + self.init_test_repo('gbp-test-native') + + eq_(mock_gbp(['--git-notify=auto']), 0) + summary, message = MOCK_NOTIFICATIONS.pop() + ok_(re.match(r'Gbp-rpm successful', summary), summary) + ok_(re.match(r'Build of \S+ \S+ succeeded', message), message) + + # Mock-notification will fail with "on" setting + eq_(mock_gbp(['--git-notify=on']), 1) + self._check_log(-1, "gbp:error: Failed to send notification") + + # No notification when "off" + eq_(mock_gbp(['--git-notify=off']), 0) + eq_(len(MOCK_NOTIFICATIONS), 0) + + def test_option_tmp_dir(self): + """Test the --git-tmp-dir option""" + self.init_test_repo('gbp-test-native') + + eq_(mock_gbp(['--git-tmp-dir=../gbptmp', '--git-builder=true']), 0) + ok_(os.path.isdir('../gbptmp')) + + # Check tmpdir access/creation error + os.chmod('../gbptmp', 0) + try: + eq_(mock_gbp(['--git-tmp-dir=../gbptmp/foo', '--git-builder=true']), 1) + finally: + os.chmod('../gbptmp', stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + + def test_tagging(self): + """Test tagging options""" + repo = self.init_test_repo('gbp-test-native') + + # Build and tag + eq_(mock_gbp(['--git-tag', '--git-packaging-tag=rel-tag']), 0) + self.check_rpms('../rpmbuild/RPMS/*') + ok_(repo.has_tag('rel-tag')) + sha = repo.rev_parse('HEAD') + eq_(sha, repo.rev_parse('rel-tag^0')) + self.check_rpms('../rpmbuild/RPMS/*') + + # Should fail if the tag already exists + eq_(mock_gbp(['--git-tag', '--git-packaging-tag=rel-tag']), 1) + + # Re-tag + eq_(mock_gbp(['--git-retag', '--git-packaging-tag=rel-tag']), 1) + self._check_log(-1, "gbp:error: '--git-retag' needs either '--git-tag'") + + eq_(mock_gbp(['--git-tag', '--git-packaging-tag=rel-tag', + '--git-retag', '--git-export=HEAD^']), 0) + ok_(repo.has_tag('rel-tag')) + sha2 = repo.rev_parse('HEAD^') + ok_(sha2 != sha) + eq_(sha2, repo.rev_parse('rel-tag^0')) + + # Tag-only + shutil.rmtree('../rpmbuild') + eq_(mock_gbp(['--git-tag-only', '--git-packaging-tag=rel-tag2']), 0) + ok_(not os.path.exists('../rpmbuild')) + ok_(repo.has_tag('rel-tag2')) + + # Valid tag format string keys + tag_keys = ['upstreamversion', 'release', 'version', 'vendor'] + # Should fail if the fag format has invalid keys (foo here) + tag_fmt = '_'.join(['%(' + key + ')s' for key in tag_keys + ['foo']]) + eq_(mock_gbp(['--git-tag', '--git-packaging-tag=%(foo)s']), 1) + # Remove 'foo' and should succeed + tag_fmt = '_'.join(['%(' + key + ')s' for key in tag_keys]) + eq_(mock_gbp(['--git-tag-only', '--git-packaging-tag=%s' % tag_fmt]), 0) + + def test_option_upstream_tree(self): + """Test the --git-upstream-tree option""" + repo = self.init_test_repo('gbp-test') + + # Dummy update to upstream branch + pkg_branch = repo.get_branch() + upstr_branch = 'srcdata/gbp-test/upstream' + orig_files = ['gbp-test/' + path for \ + path in self.ls_tree(repo, upstr_branch)] + ['gbp-test'] + repo.set_branch(upstr_branch) + with open('new-file', 'w') as fobj: + fobj.write('New file\n') + with open('new-file2', 'w') as fobj: + fobj.write('New file 2\n') + repo.add_files(['new-file', 'new-file2']) + repo.commit_files('new-file', 'New content') + repo.commit_files('new-file2', 'New content 2') + repo.set_branch(pkg_branch) + + # TAG (default) does not contain the new files + eq_(mock_gbp([]), 0) + tar_files = ls_tar('../rpmbuild/SOURCES/gbp-test-1.1.tar.bz2') + self.check_files(orig_files, tar_files) + shutil.rmtree('../rpmbuild') + + # Branch contains them both + eq_(mock_gbp(['--git-upstream-tree=BRANCH']), 0) + tar_files = ls_tar('../rpmbuild/SOURCES/gbp-test-1.1.tar.bz2') + self.check_files(orig_files + + ['gbp-test/new-file', 'gbp-test/new-file2'], tar_files) + shutil.rmtree('../rpmbuild') + + # The first "extra-commit" in upstream contains only one new file + eq_(mock_gbp(['--git-upstream-tree=%s^' % upstr_branch]), 0) + tar_files = ls_tar('../rpmbuild/SOURCES/gbp-test-1.1.tar.bz2') + self.check_files(orig_files + ['gbp-test/new-file'], tar_files) + shutil.rmtree('../rpmbuild') + + # Test invalid upstream treeish + eq_(mock_gbp(['--git-upstream-tree=TAG', + '--git-upstream-tag=invalid-tag']), 1) + self._check_log(-1, ".*Invalid upstream treeish invalid-tag") + eq_(mock_gbp(['--git-upstream-tree=BRANCH', '--git-native=no', + '--git-upstream-branch=invalid-branch']), 1) + self._check_log(-1, ".*invalid-branch is not a valid branch") + eq_(mock_gbp(['--git-upstream-tree=invalid-tree']), 1) + self._check_log(-1, ".*Invalid treeish object") + + def test_pristine_tar(self): + """Test pristine-tar""" + repo = self.init_test_repo('gbp-test') + + # Pristine-tar checkout fails when no pristine-tar branch + eq_(mock_gbp(['--git-pristine-tar', + '--git-export=srcdata/gbp-test/release/1.1-2']), 1) + self._check_log(-1, ".*Couldn't checkout") + + # Create pristine-tar branch and try again + repo.create_branch('pristine-tar', 'srcdata/gbp-test/pristine_tar') + eq_(mock_gbp(['--git-pristine-tar', + '--git-export=srcdata/gbp-test/release/1.1-2']), 0) + self.check_rpms('../rpmbuild/RPMS/*') + + def test_pristine_tar_commit(self): + """Test committing upstream tarball to pristine-tar""" + repo = self.init_test_repo('gbp-test') + + eq_(repo.has_branch('pristine-tar'), False) + eq_(mock_gbp(['--git-pristine-tar-commit', + '--git-export=srcdata/gbp-test/release/1.0-1']), 0) + eq_(len(repo.get_commits(until='pristine-tar')), 1) + shutil.rmtree('../rpmbuild') + + # Using --git-pristine-tar and --git-pristine-tar-commit should be ok + eq_(mock_gbp(['--git-pristine-tar', '--git-pristine-tar-commit']), 0) + eq_(len(repo.get_commits(until='pristine-tar')), 2) + shutil.rmtree('../rpmbuild') + + # Second time no pristine-tar should not be commited + eq_(mock_gbp(['--git-pristine-tar-commit']), 0) + eq_(len(repo.get_commits(until='pristine-tar')), 2) + + def test_tarball_dir(self): + """Test a separate tarball cache""" + self.init_test_repo('gbp-test') + + # Create and populate tarball cache + tarball_dir = '../tarballs' + os.mkdir(tarball_dir) + shutil.copy2(os.path.join(ORIG_DATA_DIR, 'gbp-test-1.0.tar.bz2'), + tarball_dir) + + # Test build when tarball is found from cache + eq_(mock_gbp(['--git-export=srcdata/gbp-test/release/1.0-1', + '--git-tarball-dir=%s' % tarball_dir]), 0) + ok_(os.path.islink(os.path.join('..', 'rpmbuild', 'SOURCES', + 'gbp-test-1.0.tar.bz2'))) + + # Test build when tarball is not found from cache + eq_(mock_gbp(['--git-export=srcdata/gbp-test/release/1.1-2', + '--git-tarball-dir=%s' % tarball_dir]), 0) + ok_(os.path.isfile(os.path.join('..', 'rpmbuild', 'SOURCES', + 'gbp-test-1.1.tar.bz2'))) + + def test_packaging_branch_options(self): + """Test the --packaging-branch and --ignore-branch cmdline options""" + repo = self.init_test_repo('gbp-test-native') + + eq_(mock_gbp(['--git-packaging-branch=foo']), 1) + self._check_log(-2, "gbp:error: You are not on branch 'foo'") + + eq_(mock_gbp(['--git-packaging-branch=foo', '--git-ignore-branch']), 0) + + # Test building when not on any branch + repo.set_branch(repo.rev_parse('HEAD')) + eq_(mock_gbp(['--git-builder=true']), 1) + eq_(mock_gbp(['--git-ignore-branch', '--git-builder=true']), 0) + + def test_option_submodules(self): + """Test the --git-submodules option""" + repo = self.init_test_repo('gbp-test') + + # Create submodule to upstream branch + sub_repo = self.orig_repos['gbp-test-native'] + pkg_branch = repo.get_branch() + upstr_branch = 'srcdata/gbp-test/upstream' + repo.set_branch(upstr_branch) + repo.add_submodule(sub_repo.path) + repo.commit_all('Add submodule') + repo.set_branch(pkg_branch) + + sub_files = self.ls_tree(sub_repo, 'HEAD') + upstr_files = ['gbp-test/' + path for + path in self.ls_tree(repo, upstr_branch)] + + # Test the "no" option + eq_(mock_gbp(['--git-no-submodules', '--git-upstream-tree=%s' % + upstr_branch, '--git-ignore-new']), 0) + tar_files = ls_tar('../rpmbuild/SOURCES/gbp-test-1.1.tar.bz2', False) + self.check_files(upstr_files, tar_files) + shutil.rmtree('../rpmbuild') + + # Test the "yes" option + eq_(mock_gbp(['--git-submodules', '--git-upstream-tree=%s' % + upstr_branch, '--git-ignore-new']), 0) + tar_files = ls_tar('../rpmbuild/SOURCES/gbp-test-1.1.tar.bz2', False) + ref_files = upstr_files + ['gbp-test/gbp-test-native.repo/' + path for + path in sub_files] + self.check_files(ref_files, tar_files) + shutil.rmtree('../rpmbuild') + + # Test submodule failure + shutil.rmtree('gbp-test-native.repo') + repo.create('gbp-test-native.repo') + eq_(mock_gbp(['--git-submodules', '--git-upstream-tree=%s' % + upstr_branch, '--git-ignore-new']), 1) + + def test_option_submodules_native(self): + """Test the --git-submodules option for native packages""" + repo = self.init_test_repo('gbp-test-native') + + # Create submodule + sub_repo = self.orig_repos['gbp-test-native2'] + repo.add_submodule(sub_repo.path) + repo.commit_all('Add submodule') + + sub_files = self.ls_tree(sub_repo, 'HEAD') + master_files = ['gbp-test-native-1.0/' + path for + path in self.ls_tree(repo, 'HEAD')] + + # Test + eq_(mock_gbp(['--git-submodules']), 0) + zip_files = ls_zip('../rpmbuild/SOURCES/gbp-test-native-1.0.zip', False) + ref_files = master_files + \ + ['gbp-test-native-1.0/gbp-test-native2.repo/' + path for + path in sub_files] + self.check_files(ref_files, zip_files) + + # Test submodule failure + shutil.rmtree('gbp-test-native2.repo') + repo.create('gbp-test-native2.repo') + eq_(mock_gbp(['--git-submodules', '--git-ignore-new']), 1) + + def test_option_builder(self): + """Test --git-builder option and it's args""" + self.init_test_repo('gbp-test-native') + base_args = ['arg0', '--git-notify=off'] + + # Try rpmbuild with default args + eq_(gbp_rpm(base_args + ['--git-builder=rpmbuild', '--nodeps']), 0) + + # Build without builder args + builder_script = 'echo -n $* > builder_args.txt' + eq_(gbp_rpm(base_args + ['--git-builder=%s' % builder_script]), 0) + with open('../rpmbuild/builder_args.txt') as fobj: + args = fobj.read() + eq_(args, 'gbp-test-native.spec') + + # Build with builder args + eq_(gbp_rpm(base_args + ['--git-builder=%s' % builder_script, + '--arg1', '--arg2']), 0) + with open('../rpmbuild/builder_args.txt') as fobj: + args = fobj.read() + eq_(args, '--arg1 --arg2 gbp-test-native.spec') + + def test_option_cleaner(self): + """Test --git-cleaner option""" + self.init_test_repo('gbp-test-native') + + # Make repo dirty + with open('untracked-file', 'w') as fobj: + fobj.write('this file is not tracked\n') + + # Build on dirty repo should fail + eq_(mock_gbp([]), 1) + + # Build should succeed with cleaner + eq_(mock_gbp(['--git-cleaner=rm untracked-file']), 0) + + def test_hook_options(self): + """Test different hook options""" + self.init_test_repo('gbp-test-native') + + cleaner = 'echo -n cleaner >> ../hooks' + postexport = 'echo -n postexport >> $GBP_TMP_DIR/../hooks' + prebuild = 'echo -n prebuild >> $GBP_BUILD_DIR/../hooks' + postbuild = 'echo -n postbuild >> $GBP_BUILD_DIR/../hooks' + posttag = 'echo -n posttag >> ../hooks' + args = ['--git-cleaner=%s' % cleaner, + '--git-postexport=%s' % postexport, + '--git-prebuild=%s' % prebuild, + '--git-postbuild=%s' % postbuild, + '--git-posttag=%s' % posttag] + + # Only cleaner and posttag is run when tagging + eq_(mock_gbp(args + ['--git-tag-only', '--git-packaging-tag=tag1']), 0) + self.check_and_rm_file('../hooks', 'cleanerposttag') + + # Export and build scripts are run when not tagging + eq_(mock_gbp(args), 0) + self.check_and_rm_file('../hooks', 'cleanerpostexportprebuildpostbuild') + shutil.rmtree('../rpmbuild') + + # All hooks are run when building + eq_(mock_gbp(args + ['--git-tag', '--git-packaging-tag=tag2']), 0) + self.check_and_rm_file('../hooks', + 'cleanerpostexportprebuildpostbuildposttag') + shutil.rmtree('../rpmbuild') + + # Run with hooks disabled + eq_(mock_gbp(args + ['--git-no-hooks']), 0) + ok_(not os.path.exists('../hooks')) + + def test_builddir_options(self): + """Test the options related to different build directories""" + self.init_test_repo('gbp-test-native') + + eq_(mock_gbp(['--git-export-dir=../foo', + '--git-export-sourcedir=source', + '--git-export-specdir=spec']), 0) + + # Check all directories + eq_(set(os.listdir('../foo')), + set(['BUILD', 'BUILDROOT', 'RPMS', 'source', 'spec', 'SRPMS'])) + + def test_export_failure(self): + """Test export dir permission problems""" + self.init_test_repo('gbp-test-native') + s_rwx = stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC + + # Pre-create all files + eq_(mock_gbp(['--git-builder=true']), 0) + + # Error in exporting packaging files + os.chmod('../rpmbuild/SOURCES', 0) + try: + eq_(mock_gbp(['--git-builder=true']), 1) + finally: + os.chmod('../rpmbuild/SOURCES', s_rwx) + self._check_log(-1, ".*Error exporting packaging files") + + # Error in creating archive + os.chmod('../rpmbuild/SOURCES/gbp-test-native-1.0.zip', 0) + try: + eq_(mock_gbp(['--git-builder=true']), 1) + finally: + os.chmod('../rpmbuild/SOURCES/gbp-test-native-1.0.zip', s_rwx) + self._check_log(-1, ".*Error creating ../rpmbuild/SOURCES/.*.zip") + + def test_option_export(self): + """Test the --git-export-option""" + repo = self.init_test_repo('gbp-test') + + # Test exporting of some other commit than HEAD + eq_(mock_gbp(['--git-export=srcdata/gbp-test/release/1.0-1']), 0) + eq_(os.listdir('../rpmbuild/RPMS/noarch'), + ['gbp-test-1.0-1.noarch.rpm']) + self.check_rpms('../rpmbuild/RPMS/*') + + # Modify one tracked file, create one untracked and one ignored file + with open('foo.txt', 'a') as fobj: + fobj.write('staged') + fobj.flush() + repo.add_files('foo.txt') + fobj.write('unstaged') + with open('untracked', 'w') as fobj: + fobj.write('untracked') + with open('ignored.tmp', 'w') as fobj: + fobj.write('ignored') + + base_args = ['--git-ignore-new', '--git-builder=true'] + # Test exporting of git index + foo_txt_index = repo.show('HEAD:foo.txt') + 'staged' + eq_(mock_gbp(base_args + ['--git-export=INDEX']), 0) + self.check_and_rm_file('../rpmbuild/SOURCES/foo.txt', foo_txt_index) + ok_(not os.path.exists('../rpmbuild/SOURCES/untracked')) + ok_(not os.path.exists('../rpmbuild/SOURCES/ignored.tmp')) + shutil.rmtree('../rpmbuild') + + # Test exporting of working copy (include all files) + eq_(mock_gbp(base_args + ['--git-export=WC']), 0) + foo_txt_wc = repo.show('HEAD:foo.txt') + 'staged' + 'unstaged' + self.check_and_rm_file('../rpmbuild/SOURCES/foo.txt', foo_txt_wc) + self.check_and_rm_file('../rpmbuild/SOURCES/untracked', 'untracked') + self.check_and_rm_file('../rpmbuild/SOURCES/ignored.tmp', 'ignored') + shutil.rmtree('../rpmbuild') + + # Test exporting an invalid treeish + eq_(mock_gbp(base_args + ['--git-export=invalid-treeish']), 1) + self._check_log(-1, "gbp:error: Invalid treeish object invalid-treeish") + + def test_option_spec_file(self): + """Test the --git-spec-file cmdline option""" + repo = self.init_test_repo('gbp-test2') + + eq_(mock_gbp(['--git-spec-file=foo.spec']), 1) + self._check_log(-1, "gbp:error: Can't parse spec: Git error") + + eq_(mock_gbp(['--git-spec-file=']), 1) + self._check_log(-1, "gbp:error: Can't parse spec: Multiple spec files") + + eq_(mock_gbp(['--git-spec-file=packaging/gbp-test2.spec']), 0) + + # No spec file found error + repo.set_branch('srcdata/gbp-test2/upstream') + eq_(mock_gbp([]), 1) + self._check_log(-1, ".*Can't parse spec: No spec file found") + + def test_option_packaging_dir(self): + """Test the --git-packaging-dir cmdline option""" + self.init_test_repo('gbp-test-native') + + eq_(mock_gbp(['--git-packaging-dir=foo']), 1) + self._check_log(-1, "gbp:error: Can't parse spec: No spec file found") + + # Packaging dir should be taken from spec file if it is defined + eq_(mock_gbp(['--git-packaging-dir=foo', + '--git-spec-file=packaging/gbp-test-native.spec']), 0) + -- 1.8.4.5
