Add the "presto" bool config option, document it.
Add hooks to downloadPkgs().
---
docs/yum.conf.5 | 12 +++++
yum/__init__.py | 18 ++++++++
yum/config.py | 2 +
yum/presto.py | 135 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 167 insertions(+)
create mode 100644 yum/presto.py
diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index 62b76f8..93cb297 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -372,6 +372,13 @@ default of 5 connections. Note that there are also
implicit per-mirror limits
and the downloader honors these too.
.IP
+\fBpresto\fR
+
+Either `0' or `1'. Set this to `1' to use delta-RPM files, if available.
+This reduces the download size of updates significantly, but local rebuild
+is CPU intensive. Default is `1' (on).
+
+.IP
\fBsslcacert \fR
Path to the directory containing the databases of the certificate authorities
yum should use to verify SSL certificates. Defaults to none - uses system
@@ -930,6 +937,11 @@ repository.
Overrides the \fBip_resolve\fR option from the [main] section for this
repository.
+.IP
+\fBpresto\fR
+
+Overrides the \fBpresto\fR option from the [main] section for this
+repository.
.IP
\fBsslcacert \fR
diff --git a/yum/__init__.py b/yum/__init__.py
index 8766c95..c04b871 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -90,6 +90,7 @@ from packages import YumUrlPackage, YumNotFoundPackage
from constants import *
from yum.rpmtrans import RPMTransaction,SimpleCliCallBack
from yum.i18n import to_unicode, to_str, exception2msg
+from yum.presto import Presto
import string
import StringIO
@@ -2236,6 +2237,7 @@ much more problems).
po.basepath # prefetch now; fails when repos are closed
return False
+ pkgs = []
for po in pkglist:
if hasattr(po, 'pkgtype') and po.pkgtype == 'local':
continue
@@ -2243,8 +2245,21 @@ much more problems).
continue
if errors:
return errors
+ pkgs.append(po)
+
+ # download presto metadata
+ presto = Presto(self, pkgs)
+ for po in pkgs:
+ if presto.to_drpm(po) and verify_local(po):
+ # there's .drpm already, use it
+ presto.rebuild(po, adderror)
+ continue
remote_pkgs.append(po)
remote_size += po.size
+ if presto.deltasize:
+ self.verbose_logger.info(_('Delta RPMs reduced %s of updates to %s
(%d%% saved)'),
+ format_number(presto.rpmsize), format_number(presto.deltasize),
+ 100 - presto.deltasize*100.0/presto.rpmsize)
if downloadonly:
# close DBs, unlock
@@ -2272,6 +2287,9 @@ much more problems).
if hasattr(urlgrabber.progress, 'text_meter_total_size'):
urlgrabber.progress.text_meter_total_size(remote_size,
local_size[0])
+ if po in presto.deltas:
+ presto.rebuild(po, adderror)
+ return
if po.repoid not in done_repos:
done_repos.add(po.repoid)
# Check a single package per. repo. ... to give a hint to
diff --git a/yum/config.py b/yum/config.py
index 3b22e41..d279ab3 100644
--- a/yum/config.py
+++ b/yum/config.py
@@ -791,6 +791,7 @@ class YumConf(StartupConf):
allowed = ('ipv4', 'ipv6', 'whatever'),
mapper = {'4': 'ipv4', '6': 'ipv6'})
max_connections = IntOption(0)
+ presto = BoolOption(True)
http_caching = SelectionOption('all', ('none', 'packages', 'all'))
metadata_expire = SecondsOption(60 * 60 * 6) # Time in seconds (6h).
@@ -949,6 +950,7 @@ class RepoConf(BaseConfig):
throttle = Inherit(YumConf.throttle)
timeout = Inherit(YumConf.timeout)
ip_resolve = Inherit(YumConf.ip_resolve)
+ presto = Inherit(YumConf.presto)
http_caching = Inherit(YumConf.http_caching)
metadata_expire = Inherit(YumConf.metadata_expire)
diff --git a/yum/presto.py b/yum/presto.py
new file mode 100644
index 0000000..08462f5
--- /dev/null
+++ b/yum/presto.py
@@ -0,0 +1,135 @@
+# Integrated delta rpm support
+# Copyright 2013 Zdenek Pavlas
+
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330,
+# Boston, MA 02111-1307 USA
+
+from yum.constants import TS_UPDATE
+from yum.Errors import RepoError
+from yum.i18n import exception2msg, _
+from urlgrabber import grabber
+async = hasattr(grabber, 'parallel_wait')
+from xml.etree.cElementTree import iterparse
+import os, gzip, subprocess
+
+class Presto:
+ def __init__(self, ayum, pkgs):
+ self.verbose_logger = ayum.verbose_logger
+ self.deltas = {}
+ self._rpmsave = {}
+ self.rpmsize = 0
+ self.deltasize = 0
+
+ # calculate update sizes
+ pinfo = {}
+ reposize = {}
+ for po in pkgs:
+ if not po.repo.presto:
+ continue
+ if po.state != TS_UPDATE and po.name not in
ayum.conf.installonlypkgs:
+ continue
+ pinfo.setdefault(po.repo, {})[po.pkgtup] = po
+ reposize[po.repo] = reposize.get(po.repo, 0) + po.size
+
+ # download delta metadata
+ mdpath = {}
+ for repo in reposize:
+ for name in ('prestodelta', 'deltainfo'):
+ try: data = repo.repoXML.getData(name); break
+ except: pass
+ else:
+ self.verbose_logger.warn(_('No Presto metadata available for
%s'), repo)
+ continue
+ path = repo.cachedir +'/'+ os.path.basename(data.location[1])
+ if not os.path.exists(path) and int(data.size) > reposize[repo]:
+ self.verbose_logger.info(_('Not downloading Presto metadata
for %s'), repo)
+ continue
+
+ def failfunc(e, name=name, repo=repo):
+ mdpath.pop(repo, None)
+ if hasattr(e, 'exception'): e = e.exception
+ self.verbose_logger.warn(_('Failed to download %s for
repository %s: %s'),
+ name, repo, exception2msg(e))
+ kwargs = {}
+ if async and repo._async:
+ kwargs['failfunc'] = failfunc
+ kwargs['async'] = True
+ try: mdpath[repo] = repo._retrieveMD(name, **kwargs)
+ except Errors.RepoError, e: failfunc(e)
+ if async:
+ grabber.parallel_wait()
+
+ # parse metadata, populate self.deltas
+ for repo, path in mdpath.items():
+ pinfo_repo = pinfo[repo]
+ if path.endswith('.gz'):
+ path = gzip.open(path)
+ for ev, el in iterparse(path):
+ if el.tag != 'newpackage': continue
+ new = el.get('name'), el.get('arch'), el.get('epoch'),
el.get('version'), el.get('release')
+ po = pinfo_repo.get(new)
+ if po:
+ best = po.size * 0.75 # make this configurable?
+ have = ayum._up.installdict.get(new[:2], [])
+ for el in el.findall('delta'):
+ size = int(el.find('size').text)
+ old = el.get('oldepoch'), el.get('oldversion'),
el.get('oldrelease')
+ if size >= best or old not in have:
+ continue
+ # the old version is installed, seq check should never
fail. kill this?
+ seq = el.find('sequence').text
+ if subprocess.call(['/usr/bin/applydeltarpm', '-C',
'-s', seq]) != 0:
+ self.verbose_logger.warn(_('Deltarpm sequence
check failed for %s'), seq)
+ continue
+
+ best = size
+ csum = el.find('checksum')
+ csum = csum.get('type'), csum.text
+ self.deltas[po] = size, el.find('filename').text, csum
+ if po not in self.deltas:
+ self.verbose_logger.warn(_('No suitable drpm files for
%s'), po)
+ el.clear()
+
+ def to_drpm(self, po):
+ try: size, remote, csum = self.deltas[po]
+ except KeyError: return False
+ self._rpmsave[po] = po.packagesize, po.relativepath, po.localpath
+
+ # update stats
+ self.rpmsize += po.packagesize
+ self.deltasize += size
+
+ # update size/path/checksum to drpm values
+ po.packagesize = size
+ po.relativepath = remote
+ po.localpath = os.path.dirname(po.localpath) +'/'+
os.path.basename(remote)
+ po.returnIdSum = lambda: csum
+ return True
+
+ def rebuild(self, po, adderror):
+ # restore rpm values
+ deltapath = po.localpath
+ po.packagesize, po.relativepath, po.localpath = self._rpmsave[po]
+ del po.returnIdSum
+
+ # rebuild it from drpm
+ if subprocess.call(['/usr/bin/applydeltarpm', deltapath,
po.localpath]) != 0:
+ return adderror(po, _('Delta RPM rebuild failed'))
+ # source drpm was already checksummed.. is this necessary?
+ if not po.verifyLocalPkg():
+ return adderror(po, _('Checksum of the delta-rebuilt RPM failed'))
+ # no need to keep this
+ os.unlink(deltapath)
--
1.7.11.7
_______________________________________________
Yum-devel mailing list
Yum-devel@lists.baseurl.org
http://lists.baseurl.org/mailman/listinfo/yum-devel
