-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
Brian Harring wrote:
> On Mon, Jan 30, 2006 at 10:21:22AM -0800, Zac Medico wrote:
>> Zac Medico wrote:
>>> Okay, I've created a file-like class called atomic_ostream and it is now
>>> used for both write_atomic() and writedict().
>> I've been using this patch locally with no problems. Do we have
>> any more feedback or are people satisfied with it? IMO we need
>> something like this, if not for (unsupported) parallel merges, at
>> least to prevent loss of an important file when it is being
>> overwritten and an IO error occurs (see bug 114133).
>
> Meh.... you're not supposed to call me on being a slacker for not
> commenting. :)
>
> No complaints with this going into svn for upcoming 2.1_pre*, but I'd
> like to see the class rewritten actually- the file object lacks a lot
> of capabilities that a normal file object has.
>
> I'd suggest deriving straight from the file class, and just doing
> changes in close and open. Should be a much smaller class also ;)
I've revised the class so that it's actually derived from the builtin file type
now. In addition, I've added some more functionality to the
apply_permissions() function in hope of making it come closer to a generalized
permission application function that can be used throughout portage. Feedback,
on apply_permissions() in particular, would be appreciated.
Zac
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.2 (GNU/Linux)
iD8DBQFD6Uq2/ejvha5XGaMRAlhRAJ4sw9f8Vu3pSPc8e7j7fAZwb9M7nACfSfD1
/7plMOyzhut3hijk1fvzd+Q=
=wMIM
-----END PGP SIGNATURE-----
Index: pym/portage_util.py
===================================================================
--- pym/portage_util.py (revision 2677)
+++ pym/portage_util.py (working copy)
@@ -3,7 +3,7 @@
# $Id: /var/cvsroot/gentoo-src/portage/pym/portage_util.py,v 1.11.2.6 2005/04/23 07:26:04 jstubbs Exp $
-import sys,string,shlex,os.path
+import sys,string,shlex,os,stat
if not hasattr(__builtins__, "set"):
from sets import Set as set
@@ -189,9 +189,8 @@
"""Writes out a dict to a file; writekey=0 mode doesn't write out
the key and assumes all values are strings, not lists."""
myfile = None
- myf2 = "%s.%i" % (myfilename, os.getpid())
try:
- myfile=open(myf2,"w")
+ myfile=atomic_ofstream(myfilename)
if not writekey:
for x in mydict.values():
myfile.write(x+"\n")
@@ -199,11 +198,10 @@
for x in mydict.keys():
myfile.write("%s %s\n" % (x, " ".join(mydict[x])))
myfile.close()
- os.rename(myf2, myfilename)
-
+ myfile=None
except IOError:
if myfile is not None:
- os.unlink(myf2)
+ myfile.close()
return 0
return 1
@@ -451,4 +449,94 @@
if x not in u:
u.append(x)
return u
+
+def apply_permissions(filename, uid=-1, gid=-1, mode=0, mask=-1,
+ stat_desired=None, stat_cached=None):
+ """
+ Required Parameters:
+ filename target file path
+
+ Optional Parameters:
+ uid=-1 desired uid
+ gid=-1 desired gid
+ mode=0 desired mode bits to enable
+ mask=-1 undesired mode bits to disable
+ stat_desired=None stat object storing the desired use uid, gid,
+ and mode (use instead of separate integers)
+ stat_cached=None cached stat object representing initial state
+
+ Apply user, group, and mode bits to a file.
+ The desired bits may either be passed in as
+ seperate integers or as a stat object via the
+ stat_desired parameter.
+ This function will not change bits that are already
+ within the desired constraints. When mask < 0 (default),
+ an exact mode match is forced, otherwise, only the constrained
+ bits need to match"""
+
+ if stat_cached is None:
+ stat_cached = os.stat(filename)
+
+ if stat_desired:
+ uid = stat_desired.st_uid
+ gid = stat_desired.st_gid
+ mode = stat_desired.st_mode
+
+ if (uid != -1 and uid != stat_cached.st_uid) or \
+ (gid != -1 and gid != stat_cached.st_gid):
+ os.chown(filename, uid, gid)
+
+ if mask >= 0:
+ if (mode & stat_cached.st_mode != mode) or \
+ (mask ^ stat_cached.st_mode != stat_cached.st_mode):
+ new_mode = mode | stat_cached.st_mode
+ new_mode = mask ^ new_mode
+ os.chmod(filename, new_mode)
+ elif mode != stat_cached.st_mode:
+ os.chmod(filename, mode)
+
+class atomic_ofstream(file):
+ """Write a file atomically via os.rename(). Atomic replacement prevents
+ interprocess interference and prevents corruption of the target
+ file when the write is interrupted (for example, when an 'out of space'
+ error occurs)."""
+
+ def __init__(self, filename, mode='w', **kargs):
+ """Opens a temporary filename.pid in the same directory as filename."""
+ if mode != 'w':
+ raise IOError("invalid mode: %s" % mode)
+ self._real_name = filename
+ tmp_name = "%s.%i" % (filename, os.getpid())
+ super(atomic_ofstream, self).__init__(tmp_name, mode=mode, **kargs)
+
+ def close(self):
+ """Closes the temporary file, copies permissions (if possible),
+ and performs the atomic replacement via os.rename()."""
+ if not self.closed:
+ try:
+ super(atomic_ofstream, self).close()
+ try:
+ apply_permissions(self.name, stat_desired=os.stat(self._real_name))
+ except OSError, oe:
+ import errno
+ if oe.errno in (errno.ENOENT,errno.EPERM):
+ pass
+ else:
+ raise oe
+ os.rename(self.name, self._real_name)
+ finally:
+ # Make sure we cleanup the temp file
+ # even if an exception is raised.
+ try:
+ os.unlink(self.name)
+ except OSError, oe:
+ pass
+
+ def __del__(self):
+ self.close()
+
+def write_atomic(file_path, content):
+ f = atomic_ofstream(file_path)
+ f.write(content)
+ f.close()
Index: pym/portage.py
===================================================================
--- pym/portage.py (revision 2677)
+++ pym/portage.py (working copy)
@@ -5858,10 +5858,7 @@
os.chown(pdir, 0, portage_gid)
os.chmod(pdir, 02770)
- myworld=open(self.myroot+WORLD_FILE,"w")
- for x in newworldlist:
- myworld.write(x+"\n")
- myworld.close()
+ portage_util.write_atomic(os.path.join(self.myroot,WORLD_FILE),"\n".join(newworldlist))
#do original postrm
if myebuildpath and os.path.exists(myebuildpath):
@@ -6870,10 +6867,7 @@
if processed:
#update our internal mtime since we processed all our directives.
mtimedb["updates"][mykey]=os.stat(mykey)[stat.ST_MTIME]
- myworld=open("/"+WORLD_FILE,"w")
- for x in worldlist:
- myworld.write(x+"\n")
- myworld.close()
+ portage_util.write_atomic(WORLD_FILE,"\n".join(worldlist))
print ""
def commit_mtimedb():
Index: bin/regenworld
===================================================================
--- bin/regenworld (revision 2677)
+++ bin/regenworld (working copy)
@@ -6,7 +6,7 @@
import sys
sys.path.insert(0, "/usr/lib/portage/pym")
-import portage, string, re
+import portage, portage_util, string, re
__candidatematcher__ = re.compile("^[0-9]+: \\*\\*\\* emerge ")
__noncandidatematcher__ = re.compile(" sync( |$)| clean( |$)| search( |$)|--oneshot| unmerge( |$)")
@@ -88,6 +88,4 @@
print "add to world:",myfavkey
worldlist.append(myfavkey)
-myfile=open(portage.WORLD_FILE, "w")
-myfile.write(string.join(worldlist, '\n')+'\n')
-myfile.close()
+portage_util.write_atomic(portage.WORLD_FILE,"\n".join(worldlist))
Index: bin/emerge
===================================================================
--- bin/emerge (revision 2677)
+++ bin/emerge (working copy)
@@ -1916,7 +1916,7 @@
myfavdict[myfavkey]=myfavkey
print ">>> Recording",myfavkey,"in \"world\" favorites file..."
if not "--fetchonly" in myopts:
- portage.writedict(myfavdict,portage.root+portage.WORLD_FILE,writekey=0)
+ portage_util.write_atomic(os.path.join(portage.root,portage.WORLD_FILE),"\n".join(myfavdict.values()))
portage.mtimedb["resume"]["mergelist"]=mymergelist[:]
@@ -2087,7 +2087,7 @@
myfavdict[myfavkey]=myfavkey
print ">>> Recording",myfavkey,"in \"world\" favorites file..."
emergelog(" === ("+str(mergecount)+" of "+str(len(mymergelist))+") Updating world file ("+x[pkgindex]+")")
- portage.writedict(myfavdict,myroot+portage.WORLD_FILE,writekey=0)
+ portage_util.write_atomic(os.path.join(myroot,portage.WORLD_FILE),"\n".join(myfavdict.values()))
if ("noclean" not in portage.features) and (x[0] != "binary"):
short_msg = "emerge: ("+str(mergecount)+" of "+str(len(mymergelist))+") "+x[pkgindex]+" Clean Post"
Index: bin/emaint
===================================================================
--- bin/emaint (revision 2677)
+++ bin/emaint (working copy)
@@ -7,7 +7,7 @@
import re
-import os, portage, portage_const
+import os, portage, portage_const, portage_util
class WorldHandler(object):
def name():
@@ -40,7 +40,7 @@
def fix(self):
errors = []
try:
- open(portage_const.WORLD_FILE, "w").write("\n".join(self.okay))
+ portage_util.write_atomic(portage_const.WORLD_FILE,"\n".join(self.okay))
except OSError:
errors.append(portage_const.WORLD_FILE + " could not be opened for writing")
return errors
