-----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 Brian Harring wrote: > On Mon, Jan 30, 2006 at 10:21:22AM -0800, Zac Medico wrote: >> Zac Medico wrote: >>> Okay, I've created a file-like class called atomic_ostream and it is now >>> used for both write_atomic() and writedict(). >> I've been using this patch locally with no problems. Do we have >> any more feedback or are people satisfied with it? IMO we need >> something like this, if not for (unsupported) parallel merges, at >> least to prevent loss of an important file when it is being >> overwritten and an IO error occurs (see bug 114133). > > Meh.... you're not supposed to call me on being a slacker for not > commenting. :) > > No complaints with this going into svn for upcoming 2.1_pre*, but I'd > like to see the class rewritten actually- the file object lacks a lot > of capabilities that a normal file object has. > > I'd suggest deriving straight from the file class, and just doing > changes in close and open. Should be a much smaller class also ;)
I've revised the class so that it's actually derived from the builtin file type now. In addition, I've added some more functionality to the apply_permissions() function in hope of making it come closer to a generalized permission application function that can be used throughout portage. Feedback, on apply_permissions() in particular, would be appreciated. Zac -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.2 (GNU/Linux) iD8DBQFD6VSw/ejvha5XGaMRAjj7AJ9yDw/mwYr50O6kb6WQrBwpzrRzGgCbBCc6 +maew32bmpMfG0jGT2FWQpo= =FxsW -----END PGP SIGNATURE-----
Index: pym/portage_util.py =================================================================== --- pym/portage_util.py (revision 2677) +++ pym/portage_util.py (working copy) @@ -3,7 +3,7 @@ # $Id: /var/cvsroot/gentoo-src/portage/pym/portage_util.py,v 1.11.2.6 2005/04/23 07:26:04 jstubbs Exp $ -import sys,string,shlex,os.path +import sys,string,shlex,os,stat if not hasattr(__builtins__, "set"): from sets import Set as set @@ -189,9 +189,8 @@ """Writes out a dict to a file; writekey=0 mode doesn't write out the key and assumes all values are strings, not lists.""" myfile = None - myf2 = "%s.%i" % (myfilename, os.getpid()) try: - myfile=open(myf2,"w") + myfile=atomic_ofstream(myfilename) if not writekey: for x in mydict.values(): myfile.write(x+"\n") @@ -199,11 +198,10 @@ for x in mydict.keys(): myfile.write("%s %s\n" % (x, " ".join(mydict[x]))) myfile.close() - os.rename(myf2, myfilename) - + myfile=None except IOError: if myfile is not None: - os.unlink(myf2) + myfile.close() return 0 return 1 @@ -451,4 +449,94 @@ if x not in u: u.append(x) return u + +def apply_permissions(filename, uid=-1, gid=-1, mode=0, mask=-1, + stat_desired=None, stat_cached=None): + """ + Required Parameters: + filename target file path + + Optional Parameters: + uid=-1 desired uid + gid=-1 desired gid + mode=0 desired mode bits to enable + mask=-1 undesired mode bits to disable + stat_desired=None stat object storing the desired use uid, gid, + and mode (use instead of separate integers) + stat_cached=None cached stat object representing initial state + + Apply user, group, and mode bits to a file. + The desired bits may either be passed in as + seperate integers or as a stat object via the + stat_desired parameter. + This function will not change bits that are already + within the desired constraints. When mask < 0 (default), + an exact mode match is forced, otherwise, only the constrained + bits need to match""" + + if stat_cached is None: + stat_cached = os.stat(filename) + + if stat_desired: + uid = stat_desired.st_uid + gid = stat_desired.st_gid + mode = stat_desired.st_mode + + if (uid != -1 and uid != stat_cached.st_uid) or \ + (gid != -1 and gid != stat_cached.st_gid): + os.chown(filename, uid, gid) + + if mask >= 0: + if (mode & stat_cached.st_mode != mode) or \ + (mask ^ stat_cached.st_mode != stat_cached.st_mode): + new_mode = mode | stat_cached.st_mode + new_mode = mask ^ new_mode + os.chmod(filename, new_mode) + elif mode != stat_cached.st_mode: + os.chmod(filename, mode) + +class atomic_ofstream(file): + """Write a file atomically via os.rename(). Atomic replacement prevents + interprocess interference and prevents corruption of the target + file when the write is interrupted (for example, when an 'out of space' + error occurs).""" + + def __init__(self, filename, mode='w', **kargs): + """Opens a temporary filename.pid in the same directory as filename.""" + if mode != 'w': + raise IOError("invalid mode: %s" % mode) + self._real_name = filename + tmp_name = "%s.%i" % (filename, os.getpid()) + super(atomic_ofstream, self).__init__(tmp_name, mode=mode, **kargs) + + def close(self): + """Closes the temporary file, copies permissions (if possible), + and performs the atomic replacement via os.rename().""" + if not self.closed: + try: + super(atomic_ofstream, self).close() + try: + apply_permissions(self.name, stat_desired=os.stat(self._real_name)) + except OSError, oe: + import errno + if oe.errno in (errno.ENOENT,errno.EPERM): + pass + else: + raise oe + os.rename(self.name, self._real_name) + finally: + # Make sure we cleanup the temp file + # even if an exception is raised. + try: + os.unlink(self.name) + except OSError, oe: + pass + + def __del__(self): + self.close() + +def write_atomic(file_path, content): + f = atomic_ofstream(file_path) + f.write(content) + f.close() Index: pym/portage.py =================================================================== --- pym/portage.py (revision 2677) +++ pym/portage.py (working copy) @@ -5858,10 +5858,7 @@ os.chown(pdir, 0, portage_gid) os.chmod(pdir, 02770) - myworld=open(self.myroot+WORLD_FILE,"w") - for x in newworldlist: - myworld.write(x+"\n") - myworld.close() + portage_util.write_atomic(os.path.join(self.myroot,WORLD_FILE),"\n".join(newworldlist)) #do original postrm if myebuildpath and os.path.exists(myebuildpath): @@ -6870,10 +6867,7 @@ if processed: #update our internal mtime since we processed all our directives. mtimedb["updates"][mykey]=os.stat(mykey)[stat.ST_MTIME] - myworld=open("/"+WORLD_FILE,"w") - for x in worldlist: - myworld.write(x+"\n") - myworld.close() + portage_util.write_atomic(WORLD_FILE,"\n".join(worldlist)) print "" def commit_mtimedb(): Index: bin/regenworld =================================================================== --- bin/regenworld (revision 2677) +++ bin/regenworld (working copy) @@ -6,7 +6,7 @@ import sys sys.path.insert(0, "/usr/lib/portage/pym") -import portage, string, re +import portage, portage_util, string, re __candidatematcher__ = re.compile("^[0-9]+: \\*\\*\\* emerge ") __noncandidatematcher__ = re.compile(" sync( |$)| clean( |$)| search( |$)|--oneshot| unmerge( |$)") @@ -88,6 +88,4 @@ print "add to world:",myfavkey worldlist.append(myfavkey) -myfile=open(portage.WORLD_FILE, "w") -myfile.write(string.join(worldlist, '\n')+'\n') -myfile.close() +portage_util.write_atomic(portage.WORLD_FILE,"\n".join(worldlist)) Index: bin/emerge =================================================================== --- bin/emerge (revision 2677) +++ bin/emerge (working copy) @@ -1916,7 +1916,7 @@ myfavdict[myfavkey]=myfavkey print ">>> Recording",myfavkey,"in \"world\" favorites file..." if not "--fetchonly" in myopts: - portage.writedict(myfavdict,portage.root+portage.WORLD_FILE,writekey=0) + portage_util.write_atomic(os.path.join(portage.root,portage.WORLD_FILE),"\n".join(myfavdict.values())) portage.mtimedb["resume"]["mergelist"]=mymergelist[:] @@ -2087,7 +2087,7 @@ myfavdict[myfavkey]=myfavkey print ">>> Recording",myfavkey,"in \"world\" favorites file..." emergelog(" === ("+str(mergecount)+" of "+str(len(mymergelist))+") Updating world file ("+x[pkgindex]+")") - portage.writedict(myfavdict,myroot+portage.WORLD_FILE,writekey=0) + portage_util.write_atomic(os.path.join(myroot,portage.WORLD_FILE),"\n".join(myfavdict.values())) if ("noclean" not in portage.features) and (x[0] != "binary"): short_msg = "emerge: ("+str(mergecount)+" of "+str(len(mymergelist))+") "+x[pkgindex]+" Clean Post" Index: bin/emaint =================================================================== --- bin/emaint (revision 2677) +++ bin/emaint (working copy) @@ -7,7 +7,7 @@ import re -import os, portage, portage_const +import os, portage, portage_const, portage_util class WorldHandler(object): def name(): @@ -40,7 +40,7 @@ def fix(self): errors = [] try: - open(portage_const.WORLD_FILE, "w").write("\n".join(self.okay)) + portage_util.write_atomic(portage_const.WORLD_FILE,"\n".join(self.okay)) except OSError: errors.append(portage_const.WORLD_FILE + " could not be opened for writing") return errors