-BEGIN PGP SIGNED MESSAGE-
Hash: SHA1
Hi,
I was playing with the metadata cache stuff this weekend and decided to write a patch
that obsoletes metadata transfers on sync. Simply apply this patch to portage-2.1_preX
and add "metadb" to FEATURES in make.conf. Side note: this feature should
resolve bug 90518.
I also wrote a script that tests consistency between regenerated metadata and the
pre-generated metadata from the rsync mirrors. Actually, writing this script taught me
what I needed to know to write the patch. If you want to try this script, you need to
remove /var/cache/edb/dep/${PORTDIR} in order to force a regen. Also, the script will
not work as designed when FEATURES="metadb" is enabled.
Zac
-BEGIN PGP SIGNATURE-
Version: GnuPG v1.4.2 (GNU/Linux)
iD8DBQFDuI8D/ejvha5XGaMRArFhAJ9/zXBSlbMK84f0EHLjTUHf6Vg6/ACgwOkw
zjV8plh/mv3JwgXRcsxsSoA=
=7CP0
-END PGP SIGNATURE-
Index: portage-2.1_pre3/pym/cache/util.py
===
--- portage-2.1_pre3.orig/pym/cache/util.py
+++ portage-2.1_pre3/pym/cache/util.py
@@ -101,3 +101,13 @@ class non_quiet_mirroring(quiet_mirrorin
def corruption(self,key,*arg): print "corrupt %s:" % key,arg
def eclass_stale(self,key,*arg):print "stale %s:"%key,arg
+def is_cache_valid(emtime, eclasses, mydata):
+ if emtime != long(mydata.get("_mtime_", 0)):
+ return False
+ ec_dict = mydata.get("_eclasses_", [])
+ if len(ec_dict) == 0:
+ return True
+ for eclass, tup in ec_dict.iteritems():
+ if eclass not in eclasses or tuple(tup) != eclasses[eclass]:
+ return False
+ return True
Index: portage-2.1_pre3/pym/portage.py
===
--- portage-2.1_pre3.orig/pym/portage.py
+++ portage-2.1_pre3/pym/portage.py
@@ -29,6 +29,7 @@ try:
from time import sleep
from random import shuffle
from cache.cache_errors import CacheError
+ from cache.util import is_cache_valid
except SystemExit, e:
raise
except Exception, e:
@@ -4721,7 +4722,12 @@ class portdbapi(dbapi):
for x in self.porttrees:
# location, label, auxdbkeys
self.auxdb[x] = self.auxdbmodule(portage_const.DEPCACHE_PATH, x, filtered_auxdbkeys, gid=portage_gid)
-
+
+ if "metadb" in self.mysettings.features:
+ for x in self.porttrees:
+if os.path.exists(x+"/metadata/cache"):
+ self.metadb[x]=self.metadbmodule(x, "metadata/cache", auxdbkeys)
+
def close_caches(self):
for x in self.auxdb.keys():
self.auxdb[x].sync()
@@ -4825,14 +4831,16 @@ class portdbapi(dbapi):
raise KeyError
try:
- mydata = self.auxdb[mylocation][mycpv]
- if emtime != long(mydata.get("_mtime_", 0)):
-doregen = True
- elif len(mydata.get("_eclasses_", [])) > 0:
-doregen = not self.eclassdb.is_eclass_data_valid(mydata["_eclasses_"])
- else:
-doregen = False
-
+ mydata=None
+ doregen=False
+ if self.metadb.has_key(mylocation):
+try:
+ mydata = self.metadb[mylocation][mycpv]
+except (KeyError, CacheError):
+ pass
+ if not mydata or not is_cache_valid(emtime, self.eclassdb.eclasses, mydata):
+mydata = self.auxdb[mylocation][mycpv]
+doregen = not is_cache_valid(emtime, self.eclassdb.eclasses, mydata)
except KeyError:
doregen = True
except CacheError:
Index: portage-2.1_pre3/bin/emerge
===
--- portage-2.1_pre3.orig/bin/emerge
+++ portage-2.1_pre3/bin/emerge
@@ -2768,6 +2768,10 @@ if myaction in ["sync","rsync","metadata
except:
pass
+ if updatecache_flg and "metadb" in portage.features:
+ updatecache_flg=False
+ print "!!! Skipping metadata transfer because metadb is in FEATURES."
+
if os.path.exists(myportdir+"/metadata/cache") and updatecache_flg:
if "--quiet" not in myopts:
print "\n>>> Updating Portage cache: ",
#!/usr/bin/env python
# Copyright 1999-2006 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# $Header: $
#
# Zac Medico <[EMAIL PROTECTED]>
#
import portage
def is_cache_valid(emtime, eclasses, mydata):
if emtime != long(mydata.get("_mtime_", 0)):
return False
ec_dict = mydata.get("_eclasses_", [])
if len(ec_dict) == 0:
return True
for eclass, tup in ec_dict.iteritems():
if eclass not in eclasses or tuple(tup) != eclasses[eclass]:
return False
return True
def aux_get_dict(dbapi, cpv, auxdbkeys):
data_list = dbapi.aux_get(cpv, auxdbkeys)
data_dict = {}
for i in xrange(len(auxdbkeys)):
data_dict[auxdbkeys[i]]=data_list[i]
return data_dict
if __name__ == "__main__":
portdir = portage.settings["PORTDIR"]
rsync_module = portage.settings.load_best_module("portdbapi.metadbmodule")
auxdbkeys = filter(lambda x: not x.startswith("UNUSED_0"), portage.auxdbkeys)
rsync_cache = rsync_module(portdir, "metadata/cache", auxdbkeys)
portdb = portage.db["/"]["porttree"].dbapi
# discard overlay
for portree in portdb.porttrees:
if portree!=portdir:
portdb.portt
