Re: [gentoo-portage-dev] [PATCH] Distfile Patching Support for Portage

2011-08-21 Thread Rafael Goncalves Martins 
On Sun, Aug 21, 2011 at 3:42 PM, Rafael Goncalves Martins
 wrote:
> Patch reworked, after Zac's feedback on #-portage.
>
> Attached.
>

Re-sending. I forgot a print statement in the code. :(

Sorry.

-- 
Rafael Goncalves Martins
Gentoo Linux developer
http://rafaelmartins.eng.br/
From 3fe23a65638160c19abe991d9db6f023818c5395 Mon Sep 17 00:00:00 2001
From: "Rafael G. Martins" 
Date: Sun, 21 Aug 2011 20:32:02 -0300
Subject: [PATCH] distpatch: basic implementation of distfile patching
 support.

This patch allows Portage to call the tools from app-portage/distpatch
to reconstruct distfiles from binary deltas and validate them.
---
 cnf/make.globals   |6 ++
 man/make.conf.5|   15 
 pym/portage/const.py   |6 +-
 pym/portage/dbapi/porttree.py  |   24 ++-
 pym/portage/package/ebuild/doebuild.py |3 +
 pym/portage/package/ebuild/fetch.py|  116 +++-
 6 files changed, 149 insertions(+), 21 deletions(-)

diff --git a/cnf/make.globals b/cnf/make.globals
index 2892d50..33b93fc 100644
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -126,6 +126,12 @@ PORTAGE_ELOG_MAILFROM="portage@localhost"
 # Signing command used by repoman
 PORTAGE_GPG_SIGNING_COMMAND="gpg --sign --clearsign --yes --default-key \"\${PORTAGE_GPG_KEY}\" --homedir \"\${PORTAGE_GPG_DIR}\" \"\${FILE}\""
 
+# URL for distpatch deltas root url
+PORTAGE_DELTAS_ROOT_URL="http://soc.dev.gentoo.org/~rafaelmartins/";
+
+# distpatch command, to fetch deltas (depends on app-portage/distpatch)
+PORTAGE_DISTPATCH_COMMAND="distpatcher --db \"\${DELTADB}\" --root-url \"\${ROOT_URL}\" --output \"\${OUTPUT_DIR}\" --input \"\${INPUT_DIR}\" --distfile \${VERBOSE} \"\${FILE}\""
+
 #*
 #**  DO NOT EDIT THIS FILE  **
 # ***
diff --git a/man/make.conf.5 b/man/make.conf.5
index d83258c..b19b38c 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -272,6 +272,11 @@ strangely configured Samba server (oplocks off, NFS re\-export). A tool
 /usr/lib/portage/bin/clean_locks exists to help handle lock issues
 when a problem arises (normally due to a crash or disconnect).
 .TP
+.B distpatch
+Enable experimental support to Distfile Patching. It depends on
+\fBapp-portage/distpatch\fR. Partially based on GLEP 25.
+See: \fIhttp://www.gentoo.org/proj/en/infrastructure/distpatch/\fR
+.TP
 .B ebuild\-locks
 Use locks to ensure that unsandboxed ebuild phases never execute
 concurrently. Also see \fIparallel\-install\fR.
@@ -652,6 +657,16 @@ matching files are excluded when the \fBPORTAGE_COMPRESS\fR command is
 called. Regular expressions are supported and the match is performed only
 against the portion of the file name which follows the last period character.
 .TP
+\fBPORTAGE_DELTAS_ROOT_URL\fR = \fI"URL"\fR
+This variable contains the URL used to fetch deltas to be used by the
+\fBdistpatch\fR feature.
+.TP
+.B PORTAGE_DISTPATCH_COMMAND
+This variable contains the command used for reconstruct distfiles from old
+distfiles and deltas. It must contain the executable as well as the
+place\-holders \\${DELTADB}, \\${ROOT_URL}, \\${OUTPUT_DIR}, \\${INPUT_DIR},
+\\${VERBOSE} and \\${FILE}.
+.TP
 .B PORTAGE_ELOG_CLASSES
 .TP
 .B PORTAGE_ELOG_SYSTEM
diff --git a/pym/portage/const.py b/pym/portage/const.py
index ecaa8f1..6b211f8 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -71,6 +71,8 @@ PRELINK_BINARY   = "/usr/sbin/prelink"
 INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env"
 REPO_NAME_FILE   = "repo_name"
 REPO_NAME_LOC= "profiles" + "/" + REPO_NAME_FILE
+DELTADB_FILE = "delta.db"
+DELTAS_DIR   = "deltas"
 
 PORTAGE_PACKAGE_ATOM = "sys-apps/portage"
 LIBC_PACKAGE_ATOM= "virtual/libc"
@@ -89,8 +91,8 @@ SUPPORTED_FEATURES   = frozenset([
"allow-missing-manifests",
"assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy",
"ccache", "chflags", "collision-protect", "compress-build-logs",
-   "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot",
-   "fail-clean", "fixpackages", "force-mirror", "getbinpkg",
+   "digest", "distcc", "distcc-pump", "distlocks", "distpatch", "ebuild-locks",
+   "fakeroot", "fail-clean", "fixpackages", "force-mirror", "getbinpkg",
"installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror",
"metadata-transfer", "mirror", "multilib-strict", "news",
"noauto", "noclean", "nodoc", "noinfo", "noman",
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index bf8ecd9..75a5c8f 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/por

Re: [gentoo-portage-dev] [PATCH] Distfile Patching Support for Portage

2011-08-21 Thread Rafael Goncalves Martins 
Patch reworked, after Zac's feedback on #-portage.

Attached.

-- 
Rafael Goncalves Martins
Gentoo Linux developer
http://rafaelmartins.eng.br/
From a12f837c20d3149f077737983bdfea23c8545abb Mon Sep 17 00:00:00 2001
From: "Rafael G. Martins" 
Date: Sun, 21 Aug 2011 03:42:06 -0300
Subject: [PATCH] distpatch: basic implementation of distfile patching
 support.

This patch allows Portage to call the tools from app-portage/distpatch
to reconstruct distfiles from binary deltas and validate them.
---
 cnf/make.globals   |6 ++
 man/make.conf.5|   15 
 pym/portage/const.py   |6 +-
 pym/portage/dbapi/porttree.py  |   24 ++-
 pym/portage/package/ebuild/doebuild.py |3 +
 pym/portage/package/ebuild/fetch.py|  117 +++-
 6 files changed, 150 insertions(+), 21 deletions(-)

diff --git a/cnf/make.globals b/cnf/make.globals
index 2892d50..33b93fc 100644
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -126,6 +126,12 @@ PORTAGE_ELOG_MAILFROM="portage@localhost"
 # Signing command used by repoman
 PORTAGE_GPG_SIGNING_COMMAND="gpg --sign --clearsign --yes --default-key \"\${PORTAGE_GPG_KEY}\" --homedir \"\${PORTAGE_GPG_DIR}\" \"\${FILE}\""
 
+# URL for distpatch deltas root url
+PORTAGE_DELTAS_ROOT_URL="http://soc.dev.gentoo.org/~rafaelmartins/";
+
+# distpatch command, to fetch deltas (depends on app-portage/distpatch)
+PORTAGE_DISTPATCH_COMMAND="distpatcher --db \"\${DELTADB}\" --root-url \"\${ROOT_URL}\" --output \"\${OUTPUT_DIR}\" --input \"\${INPUT_DIR}\" --distfile \${VERBOSE} \"\${FILE}\""
+
 #*
 #**  DO NOT EDIT THIS FILE  **
 # ***
diff --git a/man/make.conf.5 b/man/make.conf.5
index d83258c..b19b38c 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -272,6 +272,11 @@ strangely configured Samba server (oplocks off, NFS re\-export). A tool
 /usr/lib/portage/bin/clean_locks exists to help handle lock issues
 when a problem arises (normally due to a crash or disconnect).
 .TP
+.B distpatch
+Enable experimental support to Distfile Patching. It depends on
+\fBapp-portage/distpatch\fR. Partially based on GLEP 25.
+See: \fIhttp://www.gentoo.org/proj/en/infrastructure/distpatch/\fR
+.TP
 .B ebuild\-locks
 Use locks to ensure that unsandboxed ebuild phases never execute
 concurrently. Also see \fIparallel\-install\fR.
@@ -652,6 +657,16 @@ matching files are excluded when the \fBPORTAGE_COMPRESS\fR command is
 called. Regular expressions are supported and the match is performed only
 against the portion of the file name which follows the last period character.
 .TP
+\fBPORTAGE_DELTAS_ROOT_URL\fR = \fI"URL"\fR
+This variable contains the URL used to fetch deltas to be used by the
+\fBdistpatch\fR feature.
+.TP
+.B PORTAGE_DISTPATCH_COMMAND
+This variable contains the command used for reconstruct distfiles from old
+distfiles and deltas. It must contain the executable as well as the
+place\-holders \\${DELTADB}, \\${ROOT_URL}, \\${OUTPUT_DIR}, \\${INPUT_DIR},
+\\${VERBOSE} and \\${FILE}.
+.TP
 .B PORTAGE_ELOG_CLASSES
 .TP
 .B PORTAGE_ELOG_SYSTEM
diff --git a/pym/portage/const.py b/pym/portage/const.py
index ecaa8f1..6b211f8 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -71,6 +71,8 @@ PRELINK_BINARY   = "/usr/sbin/prelink"
 INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env"
 REPO_NAME_FILE   = "repo_name"
 REPO_NAME_LOC= "profiles" + "/" + REPO_NAME_FILE
+DELTADB_FILE = "delta.db"
+DELTAS_DIR   = "deltas"
 
 PORTAGE_PACKAGE_ATOM = "sys-apps/portage"
 LIBC_PACKAGE_ATOM= "virtual/libc"
@@ -89,8 +91,8 @@ SUPPORTED_FEATURES   = frozenset([
"allow-missing-manifests",
"assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy",
"ccache", "chflags", "collision-protect", "compress-build-logs",
-   "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot",
-   "fail-clean", "fixpackages", "force-mirror", "getbinpkg",
+   "digest", "distcc", "distcc-pump", "distlocks", "distpatch", "ebuild-locks",
+   "fakeroot", "fail-clean", "fixpackages", "force-mirror", "getbinpkg",
"installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror",
"metadata-transfer", "mirror", "multilib-strict", "news",
"noauto", "noclean", "nodoc", "noinfo", "noman",
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index bf8ecd9..75a5c8f 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -15,11 +15,13 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_lev

Re: [gentoo-portage-dev] [PATCH] Distfile Patching Support for Portage

2011-08-11 Thread Rafael Goncalves Martins 
On Thu, Aug 11, 2011 at 12:29 PM, Arfrever Frehtes Taifersar Arahesis
 wrote:
> 2011-08-11 10:12:04 Rafael Goncalves Martins napisał(a):
>> +                                       except 
>> subprocess.CalledProcessError, e:
>
> Please test your patch with all Python versions supported by Portage (at 
> least using runtests.sh).
>

Fixed. Tests ok.

New patch attached.

Thanks.

-- 
Rafael Goncalves Martins
Gentoo Linux developer
http://rafaelmartins.eng.br/
From 749c931c69c531f5a5ac41608c4e797c1f83bcd7 Mon Sep 17 00:00:00 2001
From: "Rafael G. Martins" 
Date: Thu, 11 Aug 2011 04:25:57 -0300
Subject: [PATCH] distpatch: basic implementation of Distfile Patching
 Support.

This patch allows Portage to call the tools from app-portage/distpatch
to reconstruct distfiles from binary deltas and validate them.
---
 pym/portage/const.py   |6 +-
 pym/portage/dbapi/porttree.py  |   24 ++-
 pym/portage/package/ebuild/doebuild.py |3 +
 pym/portage/package/ebuild/fetch.py|  116 +++-
 4 files changed, 128 insertions(+), 21 deletions(-)

diff --git a/pym/portage/const.py b/pym/portage/const.py
index ecaa8f1..6b211f8 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -71,6 +71,8 @@ PRELINK_BINARY   = "/usr/sbin/prelink"
 INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env"
 REPO_NAME_FILE   = "repo_name"
 REPO_NAME_LOC= "profiles" + "/" + REPO_NAME_FILE
+DELTADB_FILE = "delta.db"
+DELTAS_DIR   = "deltas"
 
 PORTAGE_PACKAGE_ATOM = "sys-apps/portage"
 LIBC_PACKAGE_ATOM= "virtual/libc"
@@ -89,8 +91,8 @@ SUPPORTED_FEATURES   = frozenset([
"allow-missing-manifests",
"assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy",
"ccache", "chflags", "collision-protect", "compress-build-logs",
-   "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot",
-   "fail-clean", "fixpackages", "force-mirror", "getbinpkg",
+   "digest", "distcc", "distcc-pump", "distlocks", "distpatch", "ebuild-locks",
+   "fakeroot", "fail-clean", "fixpackages", "force-mirror", "getbinpkg",
"installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror",
"metadata-transfer", "mirror", "multilib-strict", "news",
"noauto", "noclean", "nodoc", "noinfo", "noman",
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index bf8ecd9..75a5c8f 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -15,11 +15,13 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level',
 	'portage.util.listdir:listdir',
 	'portage.versions:best,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp',
+	'subprocess',
 )
 
 from portage.cache import metadata_overlay, volatile
 from portage.cache.cache_errors import CacheError
 from portage.cache.mappings import Mapping
+from portage.const import DELTADB_FILE, DELTAS_DIR
 from portage.dbapi import dbapi
 from portage.exception import PortageException, \
 	FileNotFound, InvalidAtom, InvalidDependString, InvalidPackageName
@@ -588,7 +590,27 @@ class portdbapi(dbapi):
 		# into account? check checksums?
 		for myfile in myfiles:
 			try:
-fetch_size = int(checksums[myfile]["size"])
+if "distpatch" in self.settings.features:
+	try:
+		remaining_size = subprocess.check_output([
+			"distpatchq",
+			"delta_fetch_size",
+			os.path.join(self.settings["DISTDIR"], DELTADB_FILE),
+			myfile,
+			self.settings["DISTDIR"],
+			os.path.join(self.settings["DISTDIR"], DELTAS_DIR),
+		], env=self.settings.environ())
+	except subprocess.CalledProcessError:
+		fetch_size = int(checksums[myfile]["size"])
+	else:
+		try:
+			filesdict[myfile] = int(remaining_size)
+		except ValueError:
+			fetch_size = int(checksums[myfile]["size"])
+		else:
+			continue
+else:
+	fetch_size = int(checksums[myfile]["size"])
 			except (KeyError, ValueError):
 if debug:
 	writemsg(_("[bad digest]: missing %(file)s for %(pkg)s\n") % {"file":myfile, "pkg":mypkg})
diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py
index a710e09..19963bb 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -1037,6 +1037,9 @@ def _prepare_fake_distdir(settings, alist):
 	for x in alist:
 		symlink_path = os.path.join(edpath, x)
 		target = os.path.join(orig_distdir, x)
+		reconstructed_target = os.path.join(orig_distdir, "delta-reconstructed", x)
+		if "distpatch" in settings.features and os.path.exists(reconstructed_target):
+			target = reconstructed_target
 		try:
 			link_t

Re: [gentoo-portage-dev] [PATCH] Distfile Patching Support for Portage

2011-08-11 Thread Arfrever Frehtes Taifersar Arahesis 
2011-08-11 10:12:04 Rafael Goncalves Martins napisał(a):
> +   except subprocess.CalledProcessError, 
> e:

Please test your patch with all Python versions supported by Portage (at least 
using runtests.sh).

-- 
Arfrever Frehtes Taifersar Arahesis


signature.asc
Description: This is a digitally signed message part.

[gentoo-portage-dev] [PATCH] Distfile Patching Support for Portage

2011-08-11 Thread Rafael Goncalves Martins 
Hi,

I'm sending attached a patch that does the basic implementation of
Distfile Patching Support for Portage. This is part of my GSoC project
[1].

It relies on the tools provided by app-portage/distpatch, commited to
the tree today, and is able to get an old tarball, plus a XZ
compressed binary delta and reconstruct the needed tarball, if a delta
is available.

You may find a few sample deltas and a delta.db in my public page in
soc.dev.gentoo.org [2]. More deltas will be available soon. These ones
are just for tests.

Basic steps to test: (be careful with the commands! :)

# echo 'DELTAS_ROOT_URL="http://soc.dev.gentoo.org/~rafaelmartins/";'
>> /etc/make.conf
# echo 'FEATURES="${FEATURES} distpatch"' >> /etc/make.conf
# wget -O $(portageq distdir)/delta.db
http://soc.dev.gentoo.org/~rafaelmartins/delta.db
# emerge --fetchonly =grep-2.8
# rm $(portageq distdir)/grep-2.9.tar.xz
# emerge -av1 =grep-2.9

You'll see that the fetch size reported is pretty smaller than the
full tarball (80kB X >1MB). If you hit return, a delta will be fetched
and saved to $DISTDIR/deltas, the tarball will be reconstructed, the
checksums will be verified and the package will be installed. The
reconstructed tarball will be saved to $DISTDIR, if the checksums
match with the original tarball, ofrto $DISTDIR/delta-reconstructed,
if the checksums from compressed files unmatched but the checksums for
the uncompressed files matched. This is handled in a secure way by the
distpatch tools and the delta.db file.

Currently the delta.db file is fetched manually and placed in $DISTDIR
because we don't decided yet about how it will be shipped to users.
Probably through rsync, but we can't say this before have a "real"
deployment of the delta generators. This will require a little patch
soon.

This patch isn't obtrusive, and shouldn't affect any users with
FEATURES="-distpatch".

All the variable names can be changed, if wanted. I'm not very good at
choose names :)

That's it.

[1] - http://www.gentoo.org/proj/en/infrastructure/distpatch/
[2] - http://soc.dev.gentoo.org/~rafaelmartins/

Regards,

-- 
Rafael Goncalves Martins
Gentoo Linux developer
http://rafaelmartins.eng.br/
From 32e2626e849e221bace8c4150c82efb4170f6807 Mon Sep 17 00:00:00 2001
From: "Rafael G. Martins" 
Date: Thu, 11 Aug 2011 04:25:57 -0300
Subject: [PATCH] distpatch: basic implementation of Distfile Patching
 Support.

This patch allows Portage to call the tools from app-portage/distpatch
to reconstruct distfiles from binary deltas and validate them.
---
 pym/portage/const.py   |6 +-
 pym/portage/dbapi/porttree.py  |   24 ++-
 pym/portage/package/ebuild/doebuild.py |3 +
 pym/portage/package/ebuild/fetch.py|  116 +++-
 4 files changed, 128 insertions(+), 21 deletions(-)

diff --git a/pym/portage/const.py b/pym/portage/const.py
index ecaa8f1..6b211f8 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -71,6 +71,8 @@ PRELINK_BINARY   = "/usr/sbin/prelink"
 INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env"
 REPO_NAME_FILE   = "repo_name"
 REPO_NAME_LOC= "profiles" + "/" + REPO_NAME_FILE
+DELTADB_FILE = "delta.db"
+DELTAS_DIR   = "deltas"
 
 PORTAGE_PACKAGE_ATOM = "sys-apps/portage"
 LIBC_PACKAGE_ATOM= "virtual/libc"
@@ -89,8 +91,8 @@ SUPPORTED_FEATURES   = frozenset([
"allow-missing-manifests",
"assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy",
"ccache", "chflags", "collision-protect", "compress-build-logs",
-   "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot",
-   "fail-clean", "fixpackages", "force-mirror", "getbinpkg",
+   "digest", "distcc", "distcc-pump", "distlocks", "distpatch", "ebuild-locks",
+   "fakeroot", "fail-clean", "fixpackages", "force-mirror", "getbinpkg",
"installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror",
"metadata-transfer", "mirror", "multilib-strict", "news",
"noauto", "noclean", "nodoc", "noinfo", "noman",
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index bf8ecd9..e19adbf 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -15,11 +15,13 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level',
 	'portage.util.listdir:listdir',
 	'portage.versions:best,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp',
+	'subprocess',
 )
 
 from portage.cache import metadata_overlay, volatile
 from portage.cache.cache_errors import CacheError
 from portage.cache.mappings import Mapping
+from portage.const import DELTADB_FILE, DELTAS_DIR
 from portage.dbapi import dbapi
 from portage.