commit: 6b3d262e6316073a2a3be81086c05891d970ae2a Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Tue Jan 6 02:36:11 2015 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Fri Jan 16 17:07:11 2015 +0000 URL: http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=6b3d262e
BinpkgExtractorAsync: xz and gzip decompression (142579) This adds support for using a binary package's compression header to determine the compression type, providing forward-compatibility for xz and gzip decompression. The file name extension is disregared, so that it will be possible to use a compression-independent file naming scheme in the future (see bug #150031 for discussion about proposed file naming schemes). Currently, only decompression is supported. It's useful to provide forward-compatibility now, so that binhost clients will be prepared to handle future binhost servers that use xz or gzip compression. X-Gentoo-Bug: 142579 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=142579 Acked-by: Brian Dolbec <dolsen <AT> gentoo.org> --- pym/_emerge/BinpkgExtractorAsync.py | 25 ++++++++++--- pym/portage/util/compression_probe.py | 68 +++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 4 deletions(-) diff --git a/pym/_emerge/BinpkgExtractorAsync.py b/pym/_emerge/BinpkgExtractorAsync.py index be74c2f..6aaa448 100644 --- a/pym/_emerge/BinpkgExtractorAsync.py +++ b/pym/_emerge/BinpkgExtractorAsync.py @@ -1,8 +1,13 @@ # Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import logging + from _emerge.SpawnProcess import SpawnProcess import portage +from portage.localization import _ +from portage.util.compression_probe import (compression_probe, + _decompressors) import signal import subprocess @@ -20,19 +25,31 @@ class BinpkgExtractorAsync(SpawnProcess): if b"--xattrs" in output: tar_options = "--xattrs" - # Add -q to bzip2 opts, in order to avoid "trailing garbage after - # EOF ignored" warning messages due to xpak trailer. + decomp_cmd = _decompressors.get( + compression_probe(self.pkg_path)) + if decomp_cmd is None: + self.scheduler.output("!!! %s\n" % + _("File compression header unrecognized: %s") % + self.pkg_path, log_path=self.logfile, + background=self.background, level=logging.ERROR) + self.returncode = 1 + self._async_wait() + return + + # Add -q to decomp_cmd opts, in order to avoid "trailing garbage + # after EOF ignored" warning messages due to xpak trailer. # SIGPIPE handling (128 + SIGPIPE) should be compatible with # assert_sigpipe_ok() that's used by the ebuild unpack() helper. self.args = [self._shell_binary, "-c", - ("${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -cq -- %s | tar -xp %s -C %s -f - ; " + \ + ("%s -cq -- %s | tar -xp %s -C %s -f - ; " + \ "p=(${PIPESTATUS[@]}) ; " + \ "if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; then " % (128 + signal.SIGPIPE) + \ "echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \ "if [ ${p[1]} != 0 ] ; then " + \ "echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \ "exit 0 ;") % \ - (portage._shell_quote(self.pkg_path), + (decomp_cmd, + portage._shell_quote(self.pkg_path), tar_options, portage._shell_quote(self.image_dir))] diff --git a/pym/portage/util/compression_probe.py b/pym/portage/util/compression_probe.py new file mode 100644 index 0000000..1dc3547 --- /dev/null +++ b/pym/portage/util/compression_probe.py @@ -0,0 +1,68 @@ +# Copyright 2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import errno +import re +import sys + +if sys.hexversion >= 0x3000000: + basestring = str + +from portage import _encodings, _unicode_encode +from portage.exception import FileNotFound, PermissionDenied + +_decompressors = { + "bzip2": "${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d}", + "gzip": "gzip -d", + "xz": "xz -d", +} + +_compression_re = re.compile(b'^(' + + b'(?P<bzip2>\x42\x5a\x68\x39)|' + + b'(?P<gzip>\x1f\x8b)|' + + b'(?P<xz>\xfd\x37\x7a\x58\x5a\x00))') + +def compression_probe(f): + """ + Identify the compression type of a file. Returns one of the + following identifier strings: + + bzip2 + gzip + xz + + @param f: a file path, or file-like object + @type f: str or file + @return: a string identifying the compression type, or None if the + compression type is unrecognized + @rtype str or None + """ + + open_file = isinstance(f, basestring) + if open_file: + try: + f = open(_unicode_encode(f, + encoding=_encodings['fs'], errors='strict'), mode='rb') + except IOError as e: + if e.errno == PermissionDenied.errno: + raise PermissionDenied(f) + elif e.errno in (errno.ENOENT, errno.ESTALE): + raise FileNotFound(f) + else: + raise + + try: + return _compression_probe_file(f) + finally: + if open_file: + f.close() + +def _compression_probe_file(f): + + m = _compression_re.match(f.read(6)) + if m is not None: + for k, v in m.groupdict().items(): + if v is not None: + return k + + return None
