revisor/base.py | 124 +++---------------------------------------------
revisor/misc.py | 69 ++++++++++++++++++++++++++
unity/scripts/respin.sh | 15 +++--
3 files changed, 86 insertions(+), 122 deletions(-)
New commits:
commit 94b8ca4e85c9adf099897355fa2ecd9bc9ed8ca7
Merge: 7cb4f01... 82d392e...
Author: Jeroen van Meeuwen (Fedora Unity) <kana...@fedoraunity.org>
Date: Tue Apr 14 16:34:21 2009 +0200
Merge branch 'master' of ssh://git.fedorahosted.org/git/revisor
commit 7cb4f0160c1365ed8995c6488ad296d554940cf3
Author: Jeroen van Meeuwen (Fedora Unity) <kana...@fedoraunity.org>
Date: Tue Apr 14 16:32:39 2009 +0200
Consolidate routines to download packages
diff --git a/revisor/base.py b/revisor/base.py
index 8718640..622c6fc 100644
--- a/revisor/base.py
+++ b/revisor/base.py
@@ -723,51 +723,8 @@ class RevisorBase:
warnings.append("Cannot find a debuginfo rpm for
%s-%s:%s-%s.%s") % (po.name,po.epoch,po.version,po.release,po.arch)
pbar = self.progress_bar(_("Downloading Debuginfo Packages"))
- total = len(self.debuginfopolist)
- current = 0.0
- pbar.set_fraction(0.0)
- self.prev_download = ""
- self.attempt = 1
- self.current = 1
-
- for pkg in self.debuginfopolist:
- (n,a,e,v,r) = pkg.pkgtup
- packages = self.cfg.yumobj.pkgSack.searchNevra(n,e,v,r,a)
- for download in packages:
- repo = self.cfg.yumobj.repos.getRepo(download.repoid)
- if hasattr(download,"returnSimple"):
- remote = download.returnSimple('relativepath')
- else:
- remote = download.relativepath
- if os.path.exists(download.localPkg()) and
os.path.getsize(download.localPkg()) ==
int(download.returnSimple('packagesize')):
- continue
-
- path = repo.getPackage(download)
- if not os.path.exists(download.localPkg()) and
os.path.exists(path):
- try:
- os.link(path,download.localPkg())
- except:
- if self.cfg.copy_local:
- shutil.copy2(path,download.localPkg())
- else:
- os.symlink(path,download.localPkg())
-
- txt = _("Downloading %s") % remote
- if self.prev_download == "":
- self.prev_download = download
- elif self.prev_download == download:
- self.attempt += 1
- self.current -= 1
- else:
- self.attempt = 1
- self.prev_download = download
-
- if self.attempt > 1:
- txt = "#%d: %s" % (self.attempt,txt)
- self.log.debug(txt, level=1)
- current += 1
- pbar.set_fraction(current/total)
+ revisor.misc.download_packages(self.debuginfopolist, self.log,
self.cfg, pbar)
pbar.destroy()
@@ -775,8 +732,11 @@ class RevisorBase:
def download_source_packages(self):
"""Download source packages using self.polist"""
+
self.enable_source_repositories()
+ self.cfg.plugins.exec_hook("pre_download_source_packages")
+
self.log.debug(_("Creating a list of SRPMs"), level=9)
for po in self.polist:
srpm = po.returnSimple('sourcerpm').split('.src.rpm')[0]
@@ -799,52 +759,13 @@ class RevisorBase:
self.log.error(_("Error: Cannot find a source rpm for %s") %
srpm)
pbar = self.progress_bar(_("Downloading Source Packages"))
- total = len(self.srpmpolist)
- current = 0.0
- pbar.set_fraction(0.0)
- self.prev_download = ""
- self.attempt = 1
- self.current = 1
-
- for pkg in self.srpmpolist:
- (n,a,e,v,r) = pkg.pkgtup
- packages = self.cfg.yumobj.pkgSack.searchNevra(n,e,v,r,a)
- for download in packages:
- repo = self.cfg.yumobj.repos.getRepo(download.repoid)
- if hasattr(download,"returnSimple"):
- remote = download.returnSimple('relativepath')
- else:
- remote = download.relativepath
- if os.path.exists(download.localPkg()) and
os.path.getsize(download.localPkg()) ==
int(download.returnSimple('packagesize')):
- continue
- path = repo.getPackage(download)
- if not os.path.exists(download.localPkg()) and
os.path.exists(path):
- try:
- os.link(path,download.localPkg())
- except:
- if self.cfg.copy_local:
- shutil.copy2(path,download.localPkg())
- else:
- os.symlink(path,download.localPkg())
-
- txt = _("Downloading %s") % remote
- if self.prev_download == "":
- self.prev_download = download
- elif self.prev_download == download:
- self.attempt += 1
- self.current -= 1
- else:
- self.attempt = 1
- self.prev_download = download
- if self.attempt > 1:
- txt = "#%d: %s" % (self.attempt,txt)
- self.log.debug(txt, level=1)
- current += 1
- pbar.set_fraction(current/total)
+ revisor.misc.download_packages(self.srpmpolist, self.log, self.cfg,
pbar)
pbar.destroy()
+ self.cfg.plugins.exec_hook("post_download_source_packages")
+
self.disable_source_repositories()
def download_packages(self):
@@ -859,36 +780,7 @@ class RevisorBase:
pbar = self.progress_bar(_("Downloading Packages"))
- dlCb = revisor.progress.dlcb(pbar, dlpkgs, log=self.log, cfg=self.cfg)
- self.cfg.yumobj.repos.setProgressBar(dlCb)
-
- # Packages already downloaded
- real_dlpkgs = []
- for po in dlpkgs:
- if os.path.exists(po.localPkg()) and
os.path.getsize(po.localPkg()) == int(po.returnSimple('packagesize')):
- self.log.debug(_("Using local copy of %s-%s-%s.%s at %s") %
(po.name, po.version, po.release, po.arch, po.localPkg()), level=9)
- dlCb._do_end(1)
- else:
- real_dlpkgs.append(po)
-
- try:
- probs = self.cfg.yumobj.downloadPkgs(real_dlpkgs, dlCb)
- except yum.Errors.RepoError, errmsg:
- self.log.error(errmsg)
- except IndexError:
- self.log.error(_("Unable to find a suitable mirror."))
-
- self.cfg.yumobj.repos.setProgressBar(None)
-
- if len(probs.keys()) > 0:
- errstr = []
- for key in probs.keys():
- errors = yum.misc.unique(probs[key])
- for error in errors:
- errstr.append("%s: %s" %(key, error))
-
- details_str = string.join(errstr, "\n")
- self.log.error(_("Errors were encountered while downloading
packages: %s") % details_str, recoverable=self.cfg.gui_mode)
+ revisor.misc.download_packages(self.polist, self.log, self.cfg, pbar)
pbar.destroy()
diff --git a/revisor/misc.py b/revisor/misc.py
index f22a63e..58ca5c0 100644
--- a/revisor/misc.py
+++ b/revisor/misc.py
@@ -78,6 +78,55 @@ def get_file(url, working_directory="/var/tmp"):
def download_file(url, file_name, title=None):
urlgrabber.urlgrab(url, file_name, copy_local=1)
+def download_packages(polist, log, cfg, pbar, yumobj=None):
+ """
+ Downloads packages.
+
+ Using a list of Package Objects, determines what packages have already
+ been downloaded and downloads the other packages.
+ """
+
+ if yumobj == None:
+ if hasattr(cfg,"yumobj"):
+ yumobj = cfg.yumobj
+ else:
+ log.error(_("cfg parameter to revisor.misc.download_packages() " +
\
+ "expected to be a Revisor ConfigStore with a YUM " + \
+ "Object, or a YUM Object to be passed separately"),
+ recoverable=False
+ )
+
+ dlCb = revisor.progress.dlcb(pbar, polist, log=log, cfg=cfg)
+ yumobj.repos.setProgressBar(dlCb)
+
+ # Packages already downloaded
+ real_dlpkgs = []
+ for po in polist:
+ if os.path.exists(po.localPkg()) and os.path.getsize(po.localPkg()) ==
int(po.returnSimple('packagesize')):
+ log.debug(_("Using local copy of %s-%s-%s.%s at %s") % (po.name,
po.version, po.release, po.arch, po.localPkg()), level=9)
+ dlCb._do_end(1)
+ else:
+ real_dlpkgs.append(po)
+
+ try:
+ probs = yumobj.downloadPkgs(real_dlpkgs, dlCb)
+ except yum.Errors.RepoError, errmsg:
+ log.error(errmsg)
+ except IndexError:
+ log.error(_("Unable to find a suitable mirror."))
+
+ yumobj.repos.setProgressBar(None)
+
+ if len(probs.keys()) > 0:
+ errstr = []
+ for key in probs.keys():
+ errors = yum.misc.unique(probs[key])
+ for error in errors:
+ errstr.append("%s: %s" %(key, error))
+
+ details_str = string.join(errstr, "\n")
+ log.error(_("Errors were encountered while downloading packages: %s")
% details_str, recoverable=cfg.gui_mode)
+
def check_file(file_name, checksum=None, destroy=False):
"""
Checks if a file exists. Basically returns True if the file exists, unless
the
@@ -183,6 +232,22 @@ def resolve_dependencies_inclusive(yumobj, logger=None,
pbar=None, resolved_deps
return (resolved_deps, final_pkgobjs)
#yumobj.tsInfo.makelists()
+def get_source_package_builddeps(yumobj, po, pbar, logger=None,
resolved_deps={}, final_pkgobjs={}):
+ """
+ Gets the buildrequirements for a package object, and selects those.
+ """
+ pass
+
+def get_source_package_binary_rpms(yumobj, po, pbar, logger=None,
resolved_deps={}, final_pkgobjs={}):
+ """
+ Gets all the binary rpms composed from the source package, and selects
+ those.
+
+ Uses the yum object (yumobj) and the po (binary rpm po) to determine
the
+ source rpm and all binary rpms coming from that source rpm.
+ """
+ pass
+
def get_package_deps(yumobj, po, pbar, logger=None, resolved_deps={},
final_pkgobjs={}):
"""Add the dependencies for a given package to the
transaction info"""
@@ -380,6 +445,10 @@ def link_pkgs(pos, destdir, copy_local=False, pbar=None,
log=None):
i = 0
total = float(len(pos))
+ # Just so that we know. If the hardlink fails once, why try it again?
+ # FIXME: Do something really smart with this
+ hardlink_failed = False
+
for po in pos:
try:
os.link(po.localPkg(), destdir + "/" +
os.path.basename(po.localPkg()))
commit 8cc97671e6222cb119057fdcb84011f317fe7458
Author: Jeroen van Meeuwen (Fedora Unity) <kana...@fedoraunity.org>
Date: Tue Apr 14 04:05:35 2009 +0200
Toggle cleanup default to 1
diff --git a/unity/scripts/respin.sh b/unity/scripts/respin.sh
index a367373..4567fc0 100755
--- a/unity/scripts/respin.sh
+++ b/unity/scripts/respin.sh
@@ -93,7 +93,7 @@ LIVE=0
LIVE_LOCALIZED=0
INSTALL=0
JUST_LIST=0
-cleanup=2
+cleanup=1
##
## Get the options
commit b01e0cc457ac92d36dd909d4d0fc3c0e54a4cf40
Author: Jeroen van Meeuwen (Fedora Unity) <kana...@fedoraunity.org>
Date: Mon Apr 13 16:54:12 2009 +0200
Our user account probably doesn't have permission to write to that directory
diff --git a/unity/scripts/respin.sh b/unity/scripts/respin.sh
index 26f7d52..a367373 100755
--- a/unity/scripts/respin.sh
+++ b/unity/scripts/respin.sh
@@ -332,7 +332,9 @@ for version in ${VERSIONS}; do
if [ ! -z "${isoimage}" ]; then
for pkg in `find ${REVISORDIR}/$datestamp/$spin/os/$arch/
-name "*.rpm"`; do
rpmquery -p --nogpg --qf="%{SIZE}\t%{NAME}.%{ARCH}\n"
$pkg
- done | sort -n -r >
${REVISORDIR}/$datestamp/$spin/log/rpms-$spin.log
+ done | sort -n -r > ${TMPDIR:-/tmp}/rpms-$spin.log
+
+ sudo mv ${TMPDIR:-/tmp}/rpms-$spin.log
${REVISORDIR}/$datestamp/$spin/log/
# Now that we have today's spin, if we have yesterday's
spin, we can compare
#
@@ -343,8 +345,8 @@ for version in ${VERSIONS}; do
rpms_log_history=`find
${REVISORDIR}/$hist_date/$spin/log/ -name "rpms-*.log" 2>/dev/null`
rpms_log_today=`find
${REVISORDIR}/$datestamp/$spin/log/ -name "rpms-*.log" 2>/dev/null`
if [ ! -z "$rpms_log_history" -a ! -z
"$rpms_log_today" ]; then
- `pwd`/unity/scripts/live-respin-size-diff.py
$rpms_log_history $rpms_log_today > rpms-diff-${hist_date}-$datestamp.log && \
- sudo mv rpms-diff-${hist_date}-$datestamp.log
${REVISORDIR}/$datestamp/$spin/log/
+ `pwd`/unity/scripts/live-respin-size-diff.py
$rpms_log_history $rpms_log_today >
${TMPDIR:-/tmp}/rpms-diff-${hist_date}-$datestamp.log && \
+ sudo mv
${TMPDIR:-/tmp}/rpms-diff-${hist_date}-$datestamp.log
${REVISORDIR}/$datestamp/$spin/log/
fi
i=$[ $i + 1 ]
done
commit f7a14f56ad3c5bc36daf505be84c590144f70341
Author: Jeroen van Meeuwen (Fedora Unity) <kana...@fedoraunity.org>
Date: Mon Apr 13 14:51:46 2009 +0200
Add a proxy to the mix so that we can delete the mirrorlist removal
diff --git a/unity/scripts/respin.sh b/unity/scripts/respin.sh
index 8916986..26f7d52 100755
--- a/unity/scripts/respin.sh
+++ b/unity/scripts/respin.sh
@@ -56,6 +56,9 @@ export TORRENTDIR=/data/bittorrent/
# What is the base directory for all revisor products?
export REVISORDIR=/data/revisor/
+# See if we have a proxy. If so, use it.
+[ `host proxy >/dev/null 2>&1; echo $?` -eq 0 -a -z "${HTTP_PROXY}" ] &&
export HTTP_PROXY=proxy:3128
+
function usage() {
echo "$0 [options]"
echo ""
@@ -231,7 +234,6 @@ for version in ${VERSIONS}; do
mock -v -r revisor-$version-$arch install $revisor_deps
echo -en "test -d /revisor && (cd revisor; git pull) || git clone
${GIT_REVISOR}; \\
cd /revisor; \\
- sed -i -e 's/^mirrorlist/#mirrorlist/g'
unity/conf/conf.d/*.conf conf/conf.d/*.conf; \\
autoreconf && ./configure; \\
./switchhere --yes;" | mock -v -r revisor-$version-$arch shell
@@ -283,7 +285,6 @@ for version in ${VERSIONS}; do
echo "find /var/lib/rpm/ -name '__db.*' -delete; \\
cd /revisor; \\
- sed -i -e 's/^mirrorlist/#mirrorlist/g'
unity/conf/conf.d/*.conf conf/conf.d/*.conf; \\
autoreconf && ./configure; \\
./revisor.py --cli --config
unity/conf/${real_version}-live-respin.conf \\
--destination-directory
${REVISORDIR}/$datestamp/ \\
_______________________________________________
revisor-devel mailing list
revisor-devel@lists.fedorahosted.org
https://fedorahosted.org/mailman/listinfo/revisor-devel
