[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

2016-09-19 Thread Zac Medico 
commit: d10bafb2c84be84ee47a2204938df4b3b9f238c0
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Sep 19 16:39:38 2016 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Sep 19 16:41:56 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d10bafb2

cache.fs_template: set 0o644 defaut perms

Fixes 5652bc88514b ("flat_hash: use mkstemp in _setitem)
X-Gentoo-Bug: 594358
X-Gentoo-Bug-URL: https://bugs.gentoo.org/594358

 pym/portage/cache/fs_template.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/cache/fs_template.py b/pym/portage/cache/fs_template.py
index fa44abc..e3c3c12 100644
--- a/pym/portage/cache/fs_template.py
+++ b/pym/portage/cache/fs_template.py
@@ -24,7 +24,7 @@ class FsBased(template.database):
 
def __init__(self, *args, **config):
 
-   for x, y in (("gid", -1), ("perms", -1)):
+   for x, y in (("gid", -1), ("perms", 0o644)):
if x in config:
# Since Python 3.4, chown requires int type (no 
proxies).
setattr(self, "_" + x, int(config[x]))

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

2016-07-24 Thread Zac Medico 
commit: 5652bc88514bdb36b36b544f7fc7e623cf25caae
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jul 24 22:44:31 2016 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Jul 24 23:10:01 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5652bc88

flat_hash: use mkstemp in _setitem

Fix the _setitem method to use mkstemp in order to avoid a
race condition when multiple pid namespaces share the same
cache directory.

Reported-by: Mike Frysinger  chromium.org>
X-Chromium-Bug: 477727
X-Chromium-Bug-url: https://bugs.chromium.org/p/chromium/issues/detail?id=477727

 pym/portage/cache/flat_hash.py | 49 +-
 1 file changed, 24 insertions(+), 25 deletions(-)

diff --git a/pym/portage/cache/flat_hash.py b/pym/portage/cache/flat_hash.py
index 3a899c0..7978324 100644
--- a/pym/portage/cache/flat_hash.py
+++ b/pym/portage/cache/flat_hash.py
@@ -1,4 +1,4 @@
-# Copyright 2005-2014 Gentoo Foundation
+# Copyright 2005-2016 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 # Author(s): Brian Harring (ferri...@gentoo.org)
 
@@ -10,6 +10,7 @@ import errno
 import io
 import stat
 import sys
+import tempfile
 import os as _os
 from portage import os
 from portage import _encodings
@@ -66,27 +67,14 @@ class database(fs_template.FsBased):
raise cache_errors.CacheCorruption(cpv, e)
 
def _setitem(self, cpv, values):
-   s = cpv.rfind("/")
-   fp = os.path.join(self.location,cpv[:s],".update.%i.%s" % 
(os.getpid(), cpv[s+1:]))
try:
-   myf = io.open(_unicode_encode(fp,
-   encoding=_encodings['fs'], errors='strict'),
-   mode='w', encoding=_encodings['repo.content'],
-   errors='backslashreplace')
-   except (IOError, OSError) as e:
-   if errno.ENOENT == e.errno:
-   try:
-   self._ensure_dirs(cpv)
-   myf = io.open(_unicode_encode(fp,
-   encoding=_encodings['fs'], 
errors='strict'),
-   mode='w', 
encoding=_encodings['repo.content'],
-   errors='backslashreplace')
-   except (OSError, IOError) as e:
-   raise cache_errors.CacheCorruption(cpv, 
e)
-   else:
-   raise cache_errors.CacheCorruption(cpv, e)
+   fd, fp = tempfile.mkstemp(dir=self.location)
+   except EnvironmentError as e:
+   raise cache_errors.CacheCorruption(cpv, e)
 
-   try:
+   with io.open(fd, mode='w',
+   encoding=_encodings['repo.content'],
+   errors='backslashreplace') as myf:
for k in self._write_keys:
v = values.get(k)
if not v:
@@ -95,8 +83,7 @@ class database(fs_template.FsBased):
# k and v are coerced to unicode, in order to 
prevent TypeError
# when writing raw bytes to TextIOWrapper with 
Python 2.
myf.write("%s=%s\n" % (k, v))
-   finally:
-   myf.close()
+
self._ensure_access(fp)
 
#update written.  now we move it.
@@ -104,9 +91,21 @@ class database(fs_template.FsBased):
new_fp = os.path.join(self.location,cpv)
try:
os.rename(fp, new_fp)
-   except (OSError, IOError) as e:
-   os.remove(fp)
-   raise cache_errors.CacheCorruption(cpv, e)
+   except EnvironmentError as e:
+   success = False
+   try:
+   if errno.ENOENT == e.errno:
+   try:
+   self._ensure_dirs(cpv)
+   os.rename(fp, new_fp)
+   success = True
+   except EnvironmentError as e:
+   raise 
cache_errors.CacheCorruption(cpv, e)
+   else:
+   raise cache_errors.CacheCorruption(cpv, 
e)
+   finally:
+   if not success:
+   os.remove(fp)
 
def _delitem(self, cpv):
 #  import pdb;pdb.set_trace()

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

2016-07-13 Thread Zac Medico 
commit: 9abbda7d054761ae6c333d3e6d420632b9658b6d
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jul 10 06:11:41 2016 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Jul 13 11:29:34 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9abbda7d

portage.cache: write md5 instead of mtime (bug 568934)

Change cache modules to write md5 in cache entries, instead of mtime.
Since portage-2.2.27, the relevant cache modules have had the ability
to read cache entries containing either md5 or mtime, therefore this
change is backward-compatible with portage-2.2.27 and later.

Also fix the reconstruct_eclasses function to raise CacheCorruption
when the specified chf_type is md5 and the cache entry contains mtime
data, and optimize __getitem__ to skip reconstruct_eclasses calls when
the entry appears to have a different chf_type.

X-Gentoo-Bug: 568934
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=568934
Acked-by: Alexander Berntsen  gentoo.org>

 pym/portage/cache/anydbm.py|  4 ++--
 pym/portage/cache/flat_hash.py |  4 ++--
 pym/portage/cache/sqlite.py|  4 ++--
 pym/portage/cache/template.py  | 36 
 4 files changed, 38 insertions(+), 10 deletions(-)

diff --git a/pym/portage/cache/anydbm.py b/pym/portage/cache/anydbm.py
index 80d24e5..88d85b0 100644
--- a/pym/portage/cache/anydbm.py
+++ b/pym/portage/cache/anydbm.py
@@ -36,8 +36,8 @@ from portage.cache import cache_errors
 
 class database(fs_template.FsBased):
 
-   validation_chf = 'mtime'
-   chf_types = ('mtime', 'md5')
+   validation_chf = 'md5'
+   chf_types = ('md5', 'mtime')
 
autocommits = True
cleanse_keys = True

diff --git a/pym/portage/cache/flat_hash.py b/pym/portage/cache/flat_hash.py
index cca0f10..3a899c0 100644
--- a/pym/portage/cache/flat_hash.py
+++ b/pym/portage/cache/flat_hash.py
@@ -163,5 +163,5 @@ class md5_database(database):
 
 
 class mtime_md5_database(database):
-   validation_chf = 'mtime'
-   chf_types = ('mtime', 'md5')
+   validation_chf = 'md5'
+   chf_types = ('md5', 'mtime')

diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py
index 32e4076..69150f6 100644
--- a/pym/portage/cache/sqlite.py
+++ b/pym/portage/cache/sqlite.py
@@ -18,8 +18,8 @@ if sys.hexversion >= 0x300:
 
 class database(fs_template.FsBased):
 
-   validation_chf = 'mtime'
-   chf_types = ('mtime', 'md5')
+   validation_chf = 'md5'
+   chf_types = ('md5', 'mtime')
 
autocommits = False
synchronous = False

diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py
index a7c6de0..8662d85 100644
--- a/pym/portage/cache/template.py
+++ b/pym/portage/cache/template.py
@@ -54,6 +54,15 @@ class database(object):
 
if self.serialize_eclasses and "_eclasses_" in d:
for chf_type in chf_types:
+   if '_%s_' % chf_type not in d:
+   # Skip the reconstruct_eclasses call, 
since it's
+   # a waste of time if it contains a 
different chf_type
+   # than the current one. In the past, it 
was possible
+   # for reconstruct_eclasses called with 
chf_type='md5'
+   # to "successfully" return invalid data 
here, because
+   # it was unable to distinguish between 
md5 data and
+   # mtime data.
+   continue
try:
d["_eclasses_"] = 
reconstruct_eclasses(cpv, d["_eclasses_"],
chf_type, 
paths=self.store_eclass_paths)
@@ -62,6 +71,9 @@ class database(object):
raise
else:
break
+   else:
+   raise cache_errors.CacheCorruption(cpv,
+   'entry does not contain a recognized 
chf_type')
 
elif "_eclasses_" not in d:
d["_eclasses_"] = {}
@@ -310,6 +322,23 @@ def serialize_eclasses(eclass_dict, chf_type='mtime', 
paths=True):
for k, v in sorted(eclass_dict.items(), key=_keysorter))
 
 
+def _md5_deserializer(md5):
+   """
+   Without this validation, it's possible for reconstruct_eclasses to
+   mistakenly interpret mtime data as md5 data, and return an invalid
+   data structure containing strings where ints are expected.
+   """
+   if len(md5) != 32:
+   raise ValueError('expected 32 hex digits')
+   return md5
+
+
+_chf_deserializers = {
+   'md5': _md5_deserializer,
+   'mtime': long,
+}
+
+
 def

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

2015-12-29 Thread Zac Medico 
commit: 7bef8d93aa79ad7a4001e30c74f7aa267ac95771
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Dec 24 11:19:32 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Dec 29 16:40:02 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7bef8d93

sqlite: enable md5 validation (bug 568934)

Add forward-compatibility for cache entries containing md5 digests
instead of mtimes for validation.

X-Gentoo-Bug: 568934
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=568934
Acked-by: Alexander Berntsen  gentoo.org>

 pym/portage/cache/sqlite.py | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py
index 310ac94..32e4076 100644
--- a/pym/portage/cache/sqlite.py
+++ b/pym/portage/cache/sqlite.py
@@ -18,6 +18,9 @@ if sys.hexversion >= 0x300:
 
 class database(fs_template.FsBased):
 
+   validation_chf = 'mtime'
+   chf_types = ('mtime', 'md5')
+
autocommits = False
synchronous = False
# cache_bytes is used together with page_size (set at sqlite build time)
@@ -28,10 +31,12 @@ class database(fs_template.FsBased):
def __init__(self, *args, **config):
super(database, self).__init__(*args, **config)
self._import_sqlite()
-   self._allowed_keys = ["_mtime_", "_eclasses_"]
+   self._allowed_keys = ["_eclasses_"]
self._allowed_keys.extend(self._known_keys)
-   self._allowed_keys.sort()
+   self._allowed_keys.extend('_%s_' % k for k in self.chf_types)
self._allowed_keys_set = frozenset(self._allowed_keys)
+   self._allowed_keys = sorted(self._allowed_keys_set)
+
self.location = os.path.join(self.location, 
self.label.lstrip(os.path.sep).rstrip(os.path.sep))

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

2015-12-29 Thread Zac Medico 
commit: 6c09ab7d6c6b2ffcf7e2641874167b0bff12ff91
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Dec 24 11:08:54 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Dec 29 16:39:57 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6c09ab7d

template.database.__getitem__: allow missing mtime (bug 568934)

Fix __getitem__ to allow missing mtime when a suitable alternative
(such as md5) is available.

Fixes: 669d11bd8af5 ("flat_hash: enable md5 validation for /var/cache/edb/dep 
(bug 568934)")
Acked-by: Alexander Berntsen  gentoo.org>

 pym/portage/cache/template.py | 36 ++--
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py
index a942b36..a7c6de0 100644
--- a/pym/portage/cache/template.py
+++ b/pym/portage/cache/template.py
@@ -46,12 +46,13 @@ class database(object):
self.commit()
self.updates = 0
d=self._getitem(cpv)
-   if self.serialize_eclasses and "_eclasses_" in d:
-   try:
-   chf_types = self.chf_types
-   except AttributeError:
-   chf_types = (self.validation_chf,)
 
+   try:
+   chf_types = self.chf_types
+   except AttributeError:
+   chf_types = (self.validation_chf,)
+
+   if self.serialize_eclasses and "_eclasses_" in d:
for chf_type in chf_types:
try:
d["_eclasses_"] = 
reconstruct_eclasses(cpv, d["_eclasses_"],
@@ -69,16 +70,23 @@ class database(object):
# to omit it in comparisons between cache entries like
# those that egencache uses to avoid redundant writes.
d.pop("INHERITED", None)
+
+   mtime_required = not any(d.get('_%s_' % x)
+   for x in chf_types if x != 'mtime')
+
mtime = d.get('_mtime_')
-   if mtime is None:
-   raise cache_errors.CacheCorruption(cpv,
-   '_mtime_ field is missing')
-   try:
-   mtime = long(mtime)
-   except ValueError:
-   raise cache_errors.CacheCorruption(cpv,
-   '_mtime_ conversion to long failed: %s' % 
(mtime,))
-   d['_mtime_'] = mtime
+   if not mtime:
+   if mtime_required:
+   raise cache_errors.CacheCorruption(cpv,
+   '_mtime_ field is missing')
+   d.pop('_mtime_', None)
+   else:
+   try:
+   mtime = long(mtime)
+   except ValueError:
+   raise cache_errors.CacheCorruption(cpv,
+   '_mtime_ conversion to long failed: %s' 
% (mtime,))
+   d['_mtime_'] = mtime
return d
 
def _getitem(self, cpv):

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

2015-12-29 Thread Zac Medico 
commit: 7e663905063ca92ad82127a797356e35bfc5bdfa
Author: Zac Medico  gentoo  org>
AuthorDate: Fri Dec 25 05:56:18 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Dec 29 16:40:04 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7e663905

anydbm: enable md5 validation (bug 568934)

Add forward-compatibility for cache entries containing md5 digests
instead of mtimes for validation.

X-Gentoo-Bug: 568934
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=568934
Acked-by: Alexander Berntsen  gentoo.org>

 pym/portage/cache/anydbm.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pym/portage/cache/anydbm.py b/pym/portage/cache/anydbm.py
index 1d56b14..80d24e5 100644
--- a/pym/portage/cache/anydbm.py
+++ b/pym/portage/cache/anydbm.py
@@ -36,6 +36,9 @@ from portage.cache import cache_errors
 
 class database(fs_template.FsBased):
 
+   validation_chf = 'mtime'
+   chf_types = ('mtime', 'md5')
+
autocommits = True
cleanse_keys = True
serialize_eclasses = False

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/, pym/portage/package/ebuild/

2015-12-22 Thread Zac Medico 
commit: 669d11bd8af5a2bd4cca1710a09c94294ad1e4dd
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Dec 21 07:59:55 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Dec 22 17:34:32 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=669d11bd

flat_hash: enable md5 validation for /var/cache/edb/dep (bug 568934)

Since operations like `git reset --hard` (useful to implement shallow
pull) will reset timestamps of all files in the tree, the status quo
of using timestamps for validation of cache in /var/cache/edb/dep
is sub-optimal.

For forward-compatibility, add a flat_hash.mtime_md5_database cache
module which is capable of validating cache entries containing either
mtimes or md5 digests. Update the config class to use this cache
module by default for /var/cache/edb/dep.

X-Gentoo-Bug: 568934
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=568934
Acked-by: Alexander Berntsen  gentoo.org>

 pym/portage/cache/flat_hash.py   |  5 +
 pym/portage/cache/template.py| 35 ++-
 pym/portage/package/ebuild/config.py |  6 +++---
 3 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/pym/portage/cache/flat_hash.py b/pym/portage/cache/flat_hash.py
index 5304296..cca0f10 100644
--- a/pym/portage/cache/flat_hash.py
+++ b/pym/portage/cache/flat_hash.py
@@ -160,3 +160,8 @@ class md5_database(database):
 
validation_chf = 'md5'
store_eclass_paths = False
+
+
+class mtime_md5_database(database):
+   validation_chf = 'mtime'
+   chf_types = ('mtime', 'md5')

diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py
index bc81b86..a942b36 100644
--- a/pym/portage/cache/template.py
+++ b/pym/portage/cache/template.py
@@ -47,8 +47,21 @@ class database(object):
self.updates = 0
d=self._getitem(cpv)
if self.serialize_eclasses and "_eclasses_" in d:
-   d["_eclasses_"] = reconstruct_eclasses(cpv, 
d["_eclasses_"],
-   self.validation_chf, 
paths=self.store_eclass_paths)
+   try:
+   chf_types = self.chf_types
+   except AttributeError:
+   chf_types = (self.validation_chf,)
+
+   for chf_type in chf_types:
+   try:
+   d["_eclasses_"] = 
reconstruct_eclasses(cpv, d["_eclasses_"],
+   chf_type, 
paths=self.store_eclass_paths)
+   except cache_errors.CacheCorruption:
+   if chf_type is chf_types[-1]:
+   raise
+   else:
+   break
+
elif "_eclasses_" not in d:
d["_eclasses_"] = {}
# Never return INHERITED, since portdbapi.aux_get() will
@@ -204,15 +217,27 @@ class database(object):
return x
 
def validate_entry(self, entry, ebuild_hash, eclass_db):
-   hash_key = '_%s_' % self.validation_chf
+   try:
+   chf_types = self.chf_types
+   except AttributeError:
+   chf_types = (self.validation_chf,)
+
+   for chf_type in chf_types:
+   if self._validate_entry(chf_type, entry, ebuild_hash, 
eclass_db):
+   return True
+
+   return False
+
+   def _validate_entry(self, chf_type, entry, ebuild_hash, eclass_db):
+   hash_key = '_%s_' % chf_type
try:
entry_hash = entry[hash_key]
except KeyError:
return False
else:
-   if entry_hash != getattr(ebuild_hash, 
self.validation_chf):
+   if entry_hash != getattr(ebuild_hash, chf_type):
return False
-   update = 
eclass_db.validate_and_rewrite_cache(entry['_eclasses_'], self.validation_chf,
+   update = 
eclass_db.validate_and_rewrite_cache(entry['_eclasses_'], chf_type,
self.store_eclass_paths)
if update is None:
return False

diff --git a/pym/portage/package/ebuild/config.py 
b/pym/portage/package/ebuild/config.py
index d45c2a0..0bae55b 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -160,8 +160,8 @@ class config(object):
'repository', 'RESTRICT', 'LICENSE',)
 
_module_aliases = {
-   "cache.metadata_overlay.database" : 
"portage.cache.flat_hash.database",
-   "portage.cache.metadata_overlay.database" : 
"portage.cache.flat_hash.database",
+

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

2014-11-14 Thread Zac Medico 
commit: 9f6967dbc38ea55c0de5fbf5ad663ca676c54743
Author: Zac Medico zmedico AT gentoo DOT org
AuthorDate: Thu Nov 13 19:35:48 2014 +
Commit: Zac Medico zmedico AT gentoo DOT org
CommitDate: Fri Nov 14 17:30:31 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9f6967db

fs_template._ensure_dirs: handle EEXIST (529120)

There was a race inside fs_template._ensure_dirs which could cause it to
raise EEXIST if a concurrent process created the directory after
os.path.exists returned False. Fix it by using the util.ensure_dirs
function, which already handles EEXIST.

X-Gentoo-Bug: 529120
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=529120
Acked-by: Alexander Berntsen bernalex AT gentoo.org

---
 pym/portage/cache/fs_template.py | 25 ++---
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/pym/portage/cache/fs_template.py b/pym/portage/cache/fs_template.py
index de4fe4b..fa44abc 100644
--- a/pym/portage/cache/fs_template.py
+++ b/pym/portage/cache/fs_template.py
@@ -10,7 +10,7 @@ from portage import os
 from portage.proxy.lazyimport import lazyimport
 lazyimport(globals(),
'portage.exception:PortageException',
-   'portage.util:apply_permissions',
+   'portage.util:apply_permissions,ensure_dirs',
 )
 del lazyimport
 
@@ -61,20 +61,15 @@ class FsBased(template.database):
 
for dir in 
path.lstrip(os.path.sep).rstrip(os.path.sep).split(os.path.sep):
base = os.path.join(base,dir)
-   if not os.path.exists(base):
-   if self._perms != -1:
-   um = os.umask(0)
-   try:
-   perms = self._perms
-   if perms == -1:
-   perms = 0
-   perms |= 0o755
-   os.mkdir(base, perms)
-   if self._gid != -1:
-   os.chown(base, -1, self._gid)
-   finally:
-   if self._perms != -1:
-   os.umask(um)
+   if ensure_dirs(base):
+   # We only call apply_permissions if ensure_dirs 
created
+   # a new directory, so as not to interfere with
+   # permissions of existing directories.
+   mode = self._perms
+   if mode == -1:
+   mode = 0
+   mode |= 0o755
+   apply_permissions(base, mode=mode, 
gid=self._gid)
 
def _prune_empty_dirs(self):
all_dirs = []