On Thu, Aug 11, 2011 at 12:29 PM, Arfrever Frehtes Taifersar Arahesis
<arfrever....@gmail.com> wrote:
> 2011-08-11 10:12:04 Rafael Goncalves Martins napisał(a):
>> + except
>> subprocess.CalledProcessError, e:
>
> Please test your patch with all Python versions supported by Portage (at
> least using runtests.sh).
>
Fixed. Tests ok.
New patch attached.
Thanks.
--
Rafael Goncalves Martins
Gentoo Linux developer
http://rafaelmartins.eng.br/
From 749c931c69c531f5a5ac41608c4e797c1f83bcd7 Mon Sep 17 00:00:00 2001
From: "Rafael G. Martins" <rafaelmart...@gentoo.org>
Date: Thu, 11 Aug 2011 04:25:57 -0300
Subject: [PATCH] distpatch: basic implementation of Distfile Patching
Support.
This patch allows Portage to call the tools from app-portage/distpatch
to reconstruct distfiles from binary deltas and validate them.
---
pym/portage/const.py | 6 +-
pym/portage/dbapi/porttree.py | 24 ++++++-
pym/portage/package/ebuild/doebuild.py | 3 +
pym/portage/package/ebuild/fetch.py | 116 +++++++++++++++++++++++++++-----
4 files changed, 128 insertions(+), 21 deletions(-)
diff --git a/pym/portage/const.py b/pym/portage/const.py
index ecaa8f1..6b211f8 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -71,6 +71,8 @@ PRELINK_BINARY = "/usr/sbin/prelink"
INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env"
REPO_NAME_FILE = "repo_name"
REPO_NAME_LOC = "profiles" + "/" + REPO_NAME_FILE
+DELTADB_FILE = "delta.db"
+DELTAS_DIR = "deltas"
PORTAGE_PACKAGE_ATOM = "sys-apps/portage"
LIBC_PACKAGE_ATOM = "virtual/libc"
@@ -89,8 +91,8 @@ SUPPORTED_FEATURES = frozenset([
"allow-missing-manifests",
"assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy",
"ccache", "chflags", "collision-protect", "compress-build-logs",
- "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot",
- "fail-clean", "fixpackages", "force-mirror", "getbinpkg",
+ "digest", "distcc", "distcc-pump", "distlocks", "distpatch", "ebuild-locks",
+ "fakeroot", "fail-clean", "fixpackages", "force-mirror", "getbinpkg",
"installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror",
"metadata-transfer", "mirror", "multilib-strict", "news",
"noauto", "noclean", "nodoc", "noinfo", "noman",
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index bf8ecd9..75a5c8f 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -15,11 +15,13 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level',
'portage.util.listdir:listdir',
'portage.versions:best,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp',
+ 'subprocess',
)
from portage.cache import metadata_overlay, volatile
from portage.cache.cache_errors import CacheError
from portage.cache.mappings import Mapping
+from portage.const import DELTADB_FILE, DELTAS_DIR
from portage.dbapi import dbapi
from portage.exception import PortageException, \
FileNotFound, InvalidAtom, InvalidDependString, InvalidPackageName
@@ -588,7 +590,27 @@ class portdbapi(dbapi):
# into account? check checksums?
for myfile in myfiles:
try:
- fetch_size = int(checksums[myfile]["size"])
+ if "distpatch" in self.settings.features:
+ try:
+ remaining_size = subprocess.check_output([
+ "distpatchq",
+ "delta_fetch_size",
+ os.path.join(self.settings["DISTDIR"], DELTADB_FILE),
+ myfile,
+ self.settings["DISTDIR"],
+ os.path.join(self.settings["DISTDIR"], DELTAS_DIR),
+ ], env=self.settings.environ())
+ except subprocess.CalledProcessError:
+ fetch_size = int(checksums[myfile]["size"])
+ else:
+ try:
+ filesdict[myfile] = int(remaining_size)
+ except ValueError:
+ fetch_size = int(checksums[myfile]["size"])
+ else:
+ continue
+ else:
+ fetch_size = int(checksums[myfile]["size"])
except (KeyError, ValueError):
if debug:
writemsg(_("[bad digest]: missing %(file)s for %(pkg)s\n") % {"file":myfile, "pkg":mypkg})
diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py
index a710e09..19963bb 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -1037,6 +1037,9 @@ def _prepare_fake_distdir(settings, alist):
for x in alist:
symlink_path = os.path.join(edpath, x)
target = os.path.join(orig_distdir, x)
+ reconstructed_target = os.path.join(orig_distdir, "delta-reconstructed", x)
+ if "distpatch" in settings.features and os.path.exists(reconstructed_target):
+ target = reconstructed_target
try:
link_target = os.readlink(symlink_path)
except OSError:
diff --git a/pym/portage/package/ebuild/fetch.py b/pym/portage/package/ebuild/fetch.py
index 5cbbf87..c86317c 100644
--- a/pym/portage/package/ebuild/fetch.py
+++ b/pym/portage/package/ebuild/fetch.py
@@ -22,13 +22,14 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.package.ebuild.doebuild:doebuild_environment,' + \
'_doebuild_spawn',
'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs',
+ 'subprocess'
)
from portage import OrderedDict, os, selinux, _encodings, \
_shell_quote, _unicode_encode
from portage.checksum import hashfunc_map, perform_md5, verify_all
from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \
- GLOBAL_CONFIG_PATH
+ GLOBAL_CONFIG_PATH, DELTADB_FILE, DELTAS_DIR
from portage.data import portage_gid, portage_uid, secpass, userpriv_groups
from portage.exception import FileNotFound, OperationNotPermitted, \
PortageException, TryAgain
@@ -96,6 +97,33 @@ def _spawn_fetch(settings, args, **kwargs):
return rval
+def _spawn_distpatch(settings, myfile, **kwargs):
+ phase_backup = settings.get('EBUILD_PHASE')
+ settings['EBUILD_PHASE'] = 'fetch'
+
+ # mangle distpatcher args
+ args = [
+ 'distpatcher',
+ '--db', os.path.join(settings['DISTDIR'], DELTADB_FILE),
+ '--output', settings['DISTDIR'],
+ '--input', os.path.join(settings['DISTDIR'], DELTAS_DIR),
+ '--distfile'
+ ]
+ if not settings.get('PORTAGE_QUIET', False):
+ args.append('--verbose')
+ args.append(myfile)
+
+ # run distpatcher
+ try:
+ rval = spawn(args, env=settings.environ(), **kwargs)
+ finally:
+ if phase_backup is None:
+ settings.pop('EBUILD_PHASE', None)
+ else:
+ settings['EBUILD_PHASE'] = phase_backup
+
+ return rval
+
_userpriv_test_write_file_cache = {}
_userpriv_test_write_cmd_script = ">> %(file_path)s 2>/dev/null ; rval=$? ; " + \
"rm -f %(file_path)s ; exit $rval"
@@ -572,6 +600,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
pruned_digests["size"] = size
myfile_path = os.path.join(mysettings["DISTDIR"], myfile)
+ myfile_reconstructed_path = os.path.join(mysettings["DISTDIR"], "delta-reconstructed", myfile)
has_space = True
has_space_superuser = True
file_lock = None
@@ -739,13 +768,30 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
raise
del e
+ have_myfile = False
+ have_distpatch = False
try:
mystat = os.stat(myfile_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
+ else:
+ if 'distpatch' in mysettings.features:
+ try:
+ mystat = os.stat(myfile_reconstructed_path)
+ except OSError as _e:
+ if e.errno not in (errno.ENOENT, errno.ESTALE):
+ raise
+ del _e
+ else:
+ myfile_path = myfile_reconstructed_path
+ have_myfile = True
+ have_distpatch = True
del e
else:
+ have_myfile = True
+
+ if have_myfile:
try:
apply_secpass_permissions(
myfile_path, gid=portage_gid, mode=0o664, mask=0o2,
@@ -784,23 +830,38 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
verified_ok, reason = verify_all(
myfile_path, mydigests[myfile])
if not verified_ok:
- writemsg(_("!!! Previously fetched"
- " file: '%s'\n") % myfile, noiselevel=-1)
- writemsg(_("!!! Reason: %s\n") % reason[0],
- noiselevel=-1)
- writemsg(_("!!! Got: %s\n"
- "!!! Expected: %s\n") % \
- (reason[1], reason[2]), noiselevel=-1)
- if reason[0] == _("Insufficient data for checksum verification"):
- return 0
- if distdir_writable:
- temp_filename = \
- _checksum_failure_temp_file(
- mysettings["DISTDIR"], myfile)
- writemsg_stdout(_("Refetching... "
- "File renamed to '%s'\n\n") % \
- temp_filename, noiselevel=-1)
- else:
+ if not have_distpatch:
+ writemsg(_("!!! Previously fetched"
+ " file: '%s'\n") % myfile, noiselevel=-1)
+ writemsg(_("!!! Reason: %s\n") % reason[0],
+ noiselevel=-1)
+ writemsg(_("!!! Got: %s\n"
+ "!!! Expected: %s\n") % \
+ (reason[1], reason[2]), noiselevel=-1)
+ if reason[0] == _("Insufficient data for checksum verification"):
+ return 0
+ if distdir_writable:
+ temp_filename = \
+ _checksum_failure_temp_file(
+ mysettings["DISTDIR"], myfile)
+ writemsg_stdout(_("Refetching... "
+ "File renamed to '%s'\n\n") % \
+ temp_filename, noiselevel=-1)
+ else:
+ eout = EOutput()
+ eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1"
+ eout.ebegin("%s reconstructed file, verifying checksums using distpatch" % myfile)
+ myret = subprocess.call([
+ "distpatchq",
+ "delta_verify_checksums",
+ os.path.join(mysettings["DISTDIR"], DELTADB_FILE),
+ myfile,
+ mysettings["DISTDIR"],
+ ], env=mysettings.environ())
+ eout.eend(myret)
+ if myret == 0:
+ continue
+ if verified_ok:
eout = EOutput()
eout.quiet = \
mysettings.get("PORTAGE_QUIET", None) == "1"
@@ -937,6 +998,25 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
command_var = resumecommand_var
else:
#normal mode:
+ if "distpatch" in mysettings.features:
+ myret = _spawn_distpatch(mysettings, myfile)
+ if myret == 0:
+ reconstructed_dir = os.path.join(mysettings["DISTDIR"], "delta-reconstructed")
+ if os.path.isdir(reconstructed_dir) and myfile in os.listdir(reconstructed_dir):
+ eout = EOutput()
+ eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1"
+ eout.ebegin("%s reconstructed file, verifying checksums using distpatch" % myfile)
+ myret = subprocess.call([
+ "distpatchq",
+ "delta_verify_checksums",
+ os.path.join(mysettings["DISTDIR"], DELTADB_FILE),
+ myfile,
+ mysettings["DISTDIR"],
+ ], env=mysettings.environ())
+ eout.eend(myret)
+ if myret == 0:
+ fetched = 2
+ break
locfetch=fetchcommand
command_var = fetchcommand_var
writemsg_stdout(_(">>> Downloading '%s'\n") % \
--
1.7.6
