On Sun, Aug 21, 2011 at 3:42 PM, Rafael Goncalves Martins
<rafaelmart...@gentoo.org> wrote:
> Patch reworked, after Zac's feedback on #-portage.
>
> Attached.
>

Re-sending. I forgot a print statement in the code. :(

Sorry.

-- 
Rafael Goncalves Martins
Gentoo Linux developer
http://rafaelmartins.eng.br/
From 3fe23a65638160c19abe991d9db6f023818c5395 Mon Sep 17 00:00:00 2001
From: "Rafael G. Martins" <rafaelmart...@gentoo.org>
Date: Sun, 21 Aug 2011 20:32:02 -0300
Subject: [PATCH] distpatch: basic implementation of distfile patching
 support.

This patch allows Portage to call the tools from app-portage/distpatch
to reconstruct distfiles from binary deltas and validate them.
---
 cnf/make.globals                       |    6 ++
 man/make.conf.5                        |   15 ++++
 pym/portage/const.py                   |    6 +-
 pym/portage/dbapi/porttree.py          |   24 ++++++-
 pym/portage/package/ebuild/doebuild.py |    3 +
 pym/portage/package/ebuild/fetch.py    |  116 +++++++++++++++++++++++++++-----
 6 files changed, 149 insertions(+), 21 deletions(-)

diff --git a/cnf/make.globals b/cnf/make.globals
index 2892d50..33b93fc 100644
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -126,6 +126,12 @@ PORTAGE_ELOG_MAILFROM="portage@localhost"
 # Signing command used by repoman
 PORTAGE_GPG_SIGNING_COMMAND="gpg --sign --clearsign --yes --default-key \"\${PORTAGE_GPG_KEY}\" --homedir \"\${PORTAGE_GPG_DIR}\" \"\${FILE}\""
 
+# URL for distpatch deltas root url
+PORTAGE_DELTAS_ROOT_URL="http://soc.dev.gentoo.org/~rafaelmartins/";
+
+# distpatch command, to fetch deltas (depends on app-portage/distpatch)
+PORTAGE_DISTPATCH_COMMAND="distpatcher --db \"\${DELTADB}\" --root-url \"\${ROOT_URL}\" --output \"\${OUTPUT_DIR}\" --input \"\${INPUT_DIR}\" --distfile \${VERBOSE} \"\${FILE}\""
+
 #            *****************************
 #            **  DO NOT EDIT THIS FILE  **
 # ***************************************************
diff --git a/man/make.conf.5 b/man/make.conf.5
index d83258c..b19b38c 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -272,6 +272,11 @@ strangely configured Samba server (oplocks off, NFS re\-export). A tool
 /usr/lib/portage/bin/clean_locks exists to help handle lock issues
 when a problem arises (normally due to a crash or disconnect).
 .TP
+.B distpatch
+Enable experimental support to Distfile Patching. It depends on
+\fBapp-portage/distpatch\fR. Partially based on GLEP 25.
+See: \fIhttp://www.gentoo.org/proj/en/infrastructure/distpatch/\fR
+.TP
 .B ebuild\-locks
 Use locks to ensure that unsandboxed ebuild phases never execute
 concurrently. Also see \fIparallel\-install\fR.
@@ -652,6 +657,16 @@ matching files are excluded when the \fBPORTAGE_COMPRESS\fR command is
 called. Regular expressions are supported and the match is performed only
 against the portion of the file name which follows the last period character.
 .TP
+\fBPORTAGE_DELTAS_ROOT_URL\fR = \fI"URL"\fR
+This variable contains the URL used to fetch deltas to be used by the
+\fBdistpatch\fR feature.
+.TP
+.B PORTAGE_DISTPATCH_COMMAND
+This variable contains the command used for reconstruct distfiles from old
+distfiles and deltas. It must contain the executable as well as the
+place\-holders \\${DELTADB}, \\${ROOT_URL}, \\${OUTPUT_DIR}, \\${INPUT_DIR},
+\\${VERBOSE} and \\${FILE}.
+.TP
 .B PORTAGE_ELOG_CLASSES
 .TP
 .B PORTAGE_ELOG_SYSTEM
diff --git a/pym/portage/const.py b/pym/portage/const.py
index ecaa8f1..6b211f8 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -71,6 +71,8 @@ PRELINK_BINARY           = "/usr/sbin/prelink"
 INVALID_ENV_FILE         = "/etc/spork/is/not/valid/profile.env"
 REPO_NAME_FILE           = "repo_name"
 REPO_NAME_LOC            = "profiles" + "/" + REPO_NAME_FILE
+DELTADB_FILE             = "delta.db"
+DELTAS_DIR               = "deltas"
 
 PORTAGE_PACKAGE_ATOM     = "sys-apps/portage"
 LIBC_PACKAGE_ATOM        = "virtual/libc"
@@ -89,8 +91,8 @@ SUPPORTED_FEATURES       = frozenset([
                            "allow-missing-manifests",
                            "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy",
                            "ccache", "chflags", "collision-protect", "compress-build-logs",
-                           "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot",
-                           "fail-clean", "fixpackages", "force-mirror", "getbinpkg",
+                           "digest", "distcc", "distcc-pump", "distlocks", "distpatch", "ebuild-locks",
+                           "fakeroot", "fail-clean", "fixpackages", "force-mirror", "getbinpkg",
                            "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror",
                            "metadata-transfer", "mirror", "multilib-strict", "news",
                            "noauto", "noclean", "nodoc", "noinfo", "noman",
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index bf8ecd9..75a5c8f 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -15,11 +15,13 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level',
 	'portage.util.listdir:listdir',
 	'portage.versions:best,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp',
+	'subprocess',
 )
 
 from portage.cache import metadata_overlay, volatile
 from portage.cache.cache_errors import CacheError
 from portage.cache.mappings import Mapping
+from portage.const import DELTADB_FILE, DELTAS_DIR
 from portage.dbapi import dbapi
 from portage.exception import PortageException, \
 	FileNotFound, InvalidAtom, InvalidDependString, InvalidPackageName
@@ -588,7 +590,27 @@ class portdbapi(dbapi):
 		# into account? check checksums?
 		for myfile in myfiles:
 			try:
-				fetch_size = int(checksums[myfile]["size"])
+				if "distpatch" in self.settings.features:
+					try:
+						remaining_size = subprocess.check_output([
+							"distpatchq",
+							"delta_fetch_size",
+							os.path.join(self.settings["DISTDIR"], DELTADB_FILE),
+							myfile,
+							self.settings["DISTDIR"],
+							os.path.join(self.settings["DISTDIR"], DELTAS_DIR),
+						], env=self.settings.environ())
+					except subprocess.CalledProcessError:
+						fetch_size = int(checksums[myfile]["size"])
+					else:
+						try:
+							filesdict[myfile] = int(remaining_size)
+						except ValueError:
+							fetch_size = int(checksums[myfile]["size"])
+						else:
+							continue
+				else:
+					fetch_size = int(checksums[myfile]["size"])
 			except (KeyError, ValueError):
 				if debug:
 					writemsg(_("[bad digest]: missing %(file)s for %(pkg)s\n") % {"file":myfile, "pkg":mypkg})
diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py
index a710e09..19963bb 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -1037,6 +1037,9 @@ def _prepare_fake_distdir(settings, alist):
 	for x in alist:
 		symlink_path = os.path.join(edpath, x)
 		target = os.path.join(orig_distdir, x)
+		reconstructed_target = os.path.join(orig_distdir, "delta-reconstructed", x)
+		if "distpatch" in settings.features and os.path.exists(reconstructed_target):
+			target = reconstructed_target
 		try:
 			link_target = os.readlink(symlink_path)
 		except OSError:
diff --git a/pym/portage/package/ebuild/fetch.py b/pym/portage/package/ebuild/fetch.py
index 5cbbf87..1948926 100644
--- a/pym/portage/package/ebuild/fetch.py
+++ b/pym/portage/package/ebuild/fetch.py
@@ -22,13 +22,14 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.package.ebuild.doebuild:doebuild_environment,' + \
 		'_doebuild_spawn',
 	'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs',
+	'subprocess'
 )
 
 from portage import OrderedDict, os, selinux, _encodings, \
 	_shell_quote, _unicode_encode
 from portage.checksum import hashfunc_map, perform_md5, verify_all
 from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \
-	GLOBAL_CONFIG_PATH
+	GLOBAL_CONFIG_PATH, DELTADB_FILE, DELTAS_DIR
 from portage.data import portage_gid, portage_uid, secpass, userpriv_groups
 from portage.exception import FileNotFound, OperationNotPermitted, \
 	PortageException, TryAgain
@@ -96,6 +97,33 @@ def _spawn_fetch(settings, args, **kwargs):
 
 	return rval
 
+def _spawn_distpatch(settings, myfile, **kwargs):
+	phase_backup = settings.get('EBUILD_PHASE')
+	settings['EBUILD_PHASE'] = 'fetch'
+
+	# mangle distpatcher args
+	variables = {
+		"DELTADB": os.path.join(settings['DISTDIR'], DELTADB_FILE),
+		"ROOT_URL": settings.get('PORTAGE_DELTAS_ROOT_URL', 'mirror://gentoo/'),
+		"OUTPUT_DIR": settings['DISTDIR'],
+		"INPUT_DIR": os.path.join(settings['DISTDIR'], DELTAS_DIR),
+		"VERBOSE": "" if settings.get('PORTAGE_QUIET', False) else "--verbose",
+		"FILE": myfile,
+	}
+	args = shlex_split(settings['PORTAGE_DISTPATCH_COMMAND'])
+	args = [varexpand(x, mydict=variables) for x in args]
+
+	# run distpatcher
+	try:
+		rval = spawn(args, env=settings.environ(), **kwargs)
+	finally:
+		if phase_backup is None:
+			settings.pop('EBUILD_PHASE', None)
+		else:
+			settings['EBUILD_PHASE'] = phase_backup
+
+	return rval
+
 _userpriv_test_write_file_cache = {}
 _userpriv_test_write_cmd_script = ">> %(file_path)s 2>/dev/null ; rval=$? ; " + \
 	"rm -f  %(file_path)s ; exit $rval"
@@ -572,6 +600,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
 				pruned_digests["size"] = size
 
 		myfile_path = os.path.join(mysettings["DISTDIR"], myfile)
+		myfile_reconstructed_path = os.path.join(mysettings["DISTDIR"], "delta-reconstructed", myfile)
 		has_space = True
 		has_space_superuser = True
 		file_lock = None
@@ -739,13 +768,30 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
 								raise
 							del e
 
+				have_myfile = False
+				have_distpatch = False
 				try:
 					mystat = os.stat(myfile_path)
 				except OSError as e:
 					if e.errno not in (errno.ENOENT, errno.ESTALE):
 						raise
+					else:
+						if 'distpatch' in mysettings.features:
+							try:
+								mystat = os.stat(myfile_reconstructed_path)
+							except OSError as _e:
+								if e.errno not in (errno.ENOENT, errno.ESTALE):
+									raise
+								del _e
+							else:
+								myfile_path = myfile_reconstructed_path
+								have_myfile = True
+								have_distpatch = True
 					del e
 				else:
+					have_myfile = True
+
+				if have_myfile:
 					try:
 						apply_secpass_permissions(
 							myfile_path, gid=portage_gid, mode=0o664, mask=0o2,
@@ -784,23 +830,38 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
 							verified_ok, reason = verify_all(
 								myfile_path, mydigests[myfile])
 							if not verified_ok:
-								writemsg(_("!!! Previously fetched"
-									" file: '%s'\n") % myfile, noiselevel=-1)
-								writemsg(_("!!! Reason: %s\n") % reason[0],
-									noiselevel=-1)
-								writemsg(_("!!! Got:      %s\n"
-									"!!! Expected: %s\n") % \
-									(reason[1], reason[2]), noiselevel=-1)
-								if reason[0] == _("Insufficient data for checksum verification"):
-									return 0
-								if distdir_writable:
-									temp_filename = \
-										_checksum_failure_temp_file(
-										mysettings["DISTDIR"], myfile)
-									writemsg_stdout(_("Refetching... "
-										"File renamed to '%s'\n\n") % \
-										temp_filename, noiselevel=-1)
-							else:
+								if not have_distpatch:
+									writemsg(_("!!! Previously fetched"
+										" file: '%s'\n") % myfile, noiselevel=-1)
+									writemsg(_("!!! Reason: %s\n") % reason[0],
+										noiselevel=-1)
+									writemsg(_("!!! Got:      %s\n"
+										"!!! Expected: %s\n") % \
+										(reason[1], reason[2]), noiselevel=-1)
+									if reason[0] == _("Insufficient data for checksum verification"):
+										return 0
+									if distdir_writable:
+										temp_filename = \
+											_checksum_failure_temp_file(
+											mysettings["DISTDIR"], myfile)
+										writemsg_stdout(_("Refetching... "
+											"File renamed to '%s'\n\n") % \
+											temp_filename, noiselevel=-1)
+								else:
+									eout = EOutput()
+									eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1"
+									eout.ebegin("%s reconstructed file, verifying checksums using distpatch" % myfile)
+									myret = subprocess.call([
+										"distpatchq",
+										"delta_verify_checksums",
+										os.path.join(mysettings["DISTDIR"], DELTADB_FILE),
+										myfile,
+										mysettings["DISTDIR"],
+									], env=mysettings.environ())
+									eout.eend(myret)
+									if myret == 0:
+										continue
+							if verified_ok:
 								eout = EOutput()
 								eout.quiet = \
 									mysettings.get("PORTAGE_QUIET", None) == "1"
@@ -937,6 +998,25 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
 						command_var = resumecommand_var
 					else:
 						#normal mode:
+						if "distpatch" in mysettings.features:
+							myret = _spawn_distpatch(mysettings, myfile)
+							if myret == 0:
+								reconstructed_dir = os.path.join(mysettings["DISTDIR"], "delta-reconstructed")
+								if os.path.isdir(reconstructed_dir) and myfile in os.listdir(reconstructed_dir):
+									eout = EOutput()
+									eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1"
+									eout.ebegin("%s reconstructed file, verifying checksums using distpatch" % myfile)
+									myret = subprocess.call([
+										"distpatchq",
+										"delta_verify_checksums",
+										os.path.join(mysettings["DISTDIR"], DELTADB_FILE),
+										myfile,
+										mysettings["DISTDIR"],
+									], env=mysettings.environ())
+									eout.eend(myret)
+									if myret == 0:
+										fetched = 2
+										break
 						locfetch=fetchcommand
 						command_var = fetchcommand_var
 					writemsg_stdout(_(">>> Downloading '%s'\n") % \
-- 
1.7.6

Reply via email to