[gentoo-portage-dev] [PATCH] Support different (de)compressors for binary packages
This patch allows to set the compressor for binary packages via a BINPKG_COMPRESSION variable. BINPKG_COMPRESSION_ARGS allows to specify command-line arguments for that compressor. --- bin/binpkg-helper.py | 91 +++ bin/misc-functions.sh | 9 +++- bin/quickpkg | 67 +++--- man/make.conf.5 | 25 ++ pym/_emerge/BinpkgExtractorAsync.py | 43 +++-- pym/portage/dbapi/bintree.py | 8 +-- pym/portage/util/compression_probe.py | 45 ++--- 7 files changed, 253 insertions(+), 35 deletions(-) create mode 100755 bin/binpkg-helper.py diff --git a/bin/binpkg-helper.py b/bin/binpkg-helper.py new file mode 100755 index 0..b603747cf --- /dev/null +++ b/bin/binpkg-helper.py @@ -0,0 +1,91 @@ +#!/usr/bin/python -b +# Copyright 2009-2017 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import argparse +import sys + +import portage +portage._internal_caller = True +from portage import os +from portage import cpv_getkey +from portage.process import find_binary +from portage.util import ( + shlex_split, + varexpand, + ) +from portage.util.compression_probe import _compressors + +def command_compressioncmd(args): + settings = portage.settings + usage = "usage: compressioncmd ${CATEGORY}/${P}\n" + + if len(args) != 1: + sys.stderr.write(usage) + sys.stderr.write("One argument is required, got %s\n" % len(args)) + return 1 + + cpv = args[0] + binpkg_compression = settings.get("BINPKG_COMPRESSION", "bzip2") + try: + compression = _compressors[binpkg_compression] + except KeyError as e: + sys.stderr.write("Invalid or unsupported compression method: %s" % e.args[0]) + return 1 + # Fallback to bzip2 for the package providing the decompressor + # This solves bootstrapping a client without the decompressor + + if cpv_getkey(cpv) is None: + sys.stderr.write("The argument must be in the ${CATEGORY}/${PN}-${PV} format. Provided argument was: %s\n" % cpv) + return 1 + + if cpv_getkey(cpv) == compression["package"]: + compression = _compressors["bzip2"] + try: + compression_binary = shlex_split(varexpand(compression["compress"], mydict=settings))[0] + except IndexError as e: + sys.stderr.write("Invalid or unsupported compression method: %s" % e.args[0]) + return 1 + if find_binary(compression_binary) is None: + missing_package = compression["package"] + sys.stderr.write("File compression unsupported %s. Missing package: %s" % (binpkg_compression, missing_package)) + return 1 + cmd = [varexpand(x, mydict=settings) for x in shlex_split(compression["compress"])] + # Filter empty elements + cmd = [x for x in cmd if x != ""] + + print(' '.join(cmd)) + return os.EX_OK + +def main(argv): + + if argv and isinstance(argv[0], bytes): + for i, x in enumerate(argv): + argv[i] = portage._unicode_decode(x, errors='strict') + + valid_commands = ('compressioncmd',) + description = "Returns the compression command" + usage = "usage: %s COMMAND [args]" % \ + os.path.basename(argv[0]) + + parser = argparse.ArgumentParser(description=description, usage=usage) + options, args = parser.parse_known_args(argv[1:]) + + if not args: + parser.error("missing command argument") + + command = args[0] + + if command not in valid_commands: + parser.error("invalid command: '%s'" % command) + + if command == 'compressioncmd': + rval = command_compressioncmd(args[1:]) + else: + raise AssertionError("invalid command: '%s'" % command) + + return rval + +if __name__ == "__main__": + rval = main(sys.argv[:]) + sys.exit(rval) diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index 58755a1e1..0ba0db226 100755 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -453,7 +453,7 @@ __dyn_package() { # Make sure $PWD is not ${D} so that we don't leave gmon.out files # in there in case any tools were built with -pg in CFLAGS. - cd "${T}" + cd "${T}" || die if [[ -n ${PKG_INSTALL_MASK} ]] ; then PROOT=${T}/packaging/ @@ -478,8 +478,13 @@ __dyn_package() { [ -z "${PORTAGE_BINPKG_TMPFILE}" ] && \ die "PORTAGE_BINPKG_TMPFILE is unset" mkdir -p "${PORTAGE_BINPKG_TMPFILE%/*}" || die "mkdir failed" + COMPRESSION_COMMAND=$(PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ + "${PORTAGE_PYTHON:-/usr/bin/python}"
[gentoo-portage-dev] [PATCH] emerge: Add head commit per repo to --info
This adds the following to emerge --info output for git and rsync based repositories: Head commit of repository gentoo: 0518b330edac963f54f98df33391b8e7b9eaee4c --- pym/_emerge/actions.py | 10 ++ pym/portage/sync/modules/git/__init__.py | 3 ++- pym/portage/sync/modules/git/git.py| 12 pym/portage/sync/modules/rsync/__init__.py | 3 ++- pym/portage/sync/modules/rsync/rsync.py| 12 pym/portage/sync/syncbase.py | 5 - 6 files changed, 42 insertions(+), 3 deletions(-) diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py index c8a62fb01..3c6c265f7 100644 --- a/pym/_emerge/actions.py +++ b/pym/_emerge/actions.py @@ -1644,8 +1644,18 @@ def action_info(settings, trees, myopts, myfiles): for repo in repos: last_sync = portage.grabfile(os.path.join(repo.location, "metadata", "timestamp.chk")) + head_commit = None if last_sync: append("Timestamp of repository %s: %s" % (repo.name, last_sync[0])) + if repo.sync_type: + sync = portage.sync.module_controller.get_class(repo.sync_type)() + options = { 'repo': repo } + try: + head_commit = sync.retrieve_head(options=options) + except NotImplementedError: + head_commit = (1, False) + if head_commit and head_commit[0] == os.EX_OK: + append("Head commit of repository %s: %s" % (repo.name, head_commit[1])) # Searching contents for the /bin/sh provider is somewhat # slow. Therefore, use the basename of the symlink target diff --git a/pym/portage/sync/modules/git/__init__.py b/pym/portage/sync/modules/git/__init__.py index e7206e12d..2f1d35226 100644 --- a/pym/portage/sync/modules/git/__init__.py +++ b/pym/portage/sync/modules/git/__init__.py @@ -43,12 +43,13 @@ def _check_depth(self, attr): 'sourcefile': "git", 'class': "GitSync", 'description': doc, - 'functions': ['sync', 'new', 'exists'], + 'functions': ['sync', 'new', 'exists', 'retrieve_head'], 'func_desc': { 'sync': 'Performs a git pull on the repository', 'new': 'Creates the new repository at the specified location', 'exists': 'Returns a boolean of whether the specified dir ' + 'exists and is a valid Git repository', + 'retrieve_head': 'Returns the head commit hash', }, 'validate_config': CheckGitConfig, 'module_specific_options': ( diff --git a/pym/portage/sync/modules/git/git.py b/pym/portage/sync/modules/git/git.py index bea79c7e7..8df9ca612 100644 --- a/pym/portage/sync/modules/git/git.py +++ b/pym/portage/sync/modules/git/git.py @@ -130,3 +130,15 @@ def update(self): cwd=portage._unicode_encode(self.repo.location)) return (os.EX_OK, current_rev != previous_rev) + + def retrieve_head(self, **kwargs): + '''Get information about the head commit''' + if kwargs: + self._kwargs(kwargs) + rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"] + try: + ret = (os.EX_OK, subprocess.check_output(rev_cmd, + cwd=portage._unicode_encode(self.repo.location))) + except CalledProcessError: + ret = (1, False) + return ret diff --git a/pym/portage/sync/modules/rsync/__init__.py b/pym/portage/sync/modules/rsync/__init__.py index 7ebb5476c..c2fdc4188 100644 --- a/pym/portage/sync/modules/rsync/__init__.py +++ b/pym/portage/sync/modules/rsync/__init__.py @@ -17,11 +17,12 @@ 'sourcefile': "rsync", 'class': "RsyncSync", 'description': doc, - 'functions': ['sync', 'new', 'exists'], + 'functions': ['sync', 'new', 'exists', 'retrieve_head'], 'func_desc': { 'sync': 'Performs rsync transfers on the repository', 'new': 'Creates the new repository at the specified location', 'exists': 'Returns a boolean if the specified directory exists', + 'retrieve_head': 'Returns the head commit based on metadata/timestamp.commit', }, 'validate_config': CheckSyncConfig, 'module_specific_options': ( diff --git a/pym/