antoine 2004/03/09 06:14:33
Modified: python/gump/utils sync.py
python/gump/test sync.py
Log:
Make Sync extend Annotable
also add support for copy
Revision Changes Path
1.2 +122 -88 gump/python/gump/utils/sync.py
Index: sync.py
===================================================================
RCS file: /home/cvs/gump/python/gump/utils/sync.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- sync.py 4 Mar 2004 21:38:47 -0000 1.1
+++ sync.py 9 Mar 2004 14:14:32 -0000 1.2
@@ -70,19 +70,28 @@
from gump.utils.work import *
from gump.utils.file import *
from gump.utils.launcher import *
-class Sync:
+from gump.utils.note import *
+class Sync(Annotatable):
"""
this class can be used to sync two directories
x = Sync(sourcedir, targetdir) # construct
x.execute() #run
if targetdir does not exist, it will be created
if sourcedir does not exist, the class will raise an IOError
+ the class can be also used to copy from one directory to another
+ x = Sync(sourcedir, targetdir, 1)
"""
- def __init__(self, sourcedir, targetdir):
+ def __init__(self, sourcedir, targetdir, copyflag = 0):
+ Annotatable.__init__(self)
self.sourcedir = sourcedir
self.targetdir = targetdir
+ self.copyflag = copyflag
def execute(self):
- log.info('Starting sync from [' + self.sourcedir + ']')
+ if self.copyflag:
+ action = 'copy'
+ else:
+ action = 'sync'
+ log.info('Starting %s from [%s]' % (action,self.sourcedir))
log.info(' target dir [' + self.targetdir + ']')
if not os.path.exists(self.sourcedir):
log.error('Exiting sync, source directory does not exist [' +
self.sourcedir + ']')
@@ -97,92 +106,117 @@
os.makedirs(self.targetdir)
except Exception, details:
log.exception('failed on ' + str(details))
- return
- copytree(self.sourcedir, self.targetdir, 1)
+ raise IOError, 'could not create directory [' + self.targetdir + ']'
+ self.copytree(self.sourcedir, self.targetdir, 1)
-def copytree(src, dst, symlinks=0):
- names = os.listdir(src)
- try:
- result = os.stat(dst)
- except Exception, details:
- result = None
- # handle case where result exists but is not a directory
- if result and not S_ISDIR(result[ST_MODE]):
- remove(dst)
- result = None
- if not result:
- os.makedirs(dst)
- if result:
- names2 = os.listdir(dst)
- epurate(src, dst, names, names2)
- for name in names:
- srcname = os.path.join(src, name)
- dstname = os.path.join(dst, name)
- try:
- if symlinks and os.path.islink(srcname):
- linkto = os.readlink(srcname)
- os.symlink(linkto, dstname)
- elif os.path.isdir(srcname):
- copytree(srcname, dstname, symlinks)
- else:
- maybecopy(srcname, dstname)
- except (IOError, os.error), why:
- message = "Can't copy [%s] to [%s]: [%s]" % (`srcname`, `dstname`,
str(why))
- log.exception(message)
- raise IOError, message
-def epurate(sourcedir, destdir, acceptablefiles, existingfiles):
- """
- this routine will delete from a set of existing files
- in a directory the one which are not part of an
- array of acceptablefiles
- somedir = directory where the epuration is to take place
- """
- for afile in existingfiles:
- fullsourcefile = os.path.join(sourcedir, afile)
- fulldestfile = os.path.join(destdir, afile)
- if not afile in acceptablefiles:
- tobedeleted = os.path.join(destdir, afile)
- result = os.stat(tobedeleted)
- if S_ISDIR(result[ST_MODE]):
- log.debug('attempting to remove directory [%s]' % (`tobedeleted`))
- shutil.rmtree(tobedeleted)
- else:
- log.debug('attempting to remove file [%s]' % (`tobedeleted`))
- os.remove(tobedeleted)
- elif os.path.isdir(fullsourcefile) and not os.path.isdir(fulldestfile):
- log.debug('removing file [%s] to be replaced by directory'
- %(`fulldestfile`))
- os.remove(fulldestfile)
- elif os.path.isfile(fullsourcefile) and os.path.isdir(fulldestfile):
- log.debug('removing directory [%s] to be replaced by file'
- %(`fulldestfile`))
- shutil.rmtree(fulldestfile)
-
-def maybecopy(srcname, dstname):
- """
- copy a file from srcname to dstname if
- dstname does not exist
- or srcname and dstname have different dates
- or srcname and dstname have different sizes
- """
- result = os.stat(srcname)
- try:
- result2 = os.stat(dstname)
- except (Exception), details:
- result2 = None
- okcopy = 0
- if not result2:
- okcopy = 1
- elif S_ISDIR(result2[ST_MODE]):
- shutil.rmtree(dstname)
- okcopy = 1
- elif result[ST_SIZE] != result2[ST_SIZE]:
- okcopy = 1
- elif result[ST_MTIME] != result2[ST_MTIME]:
- okcopy = 1
- if okcopy:
- log.debug("Attempting copy from [%s] to [%s]" %(`srcname`, `dstname`))
- shutil.copy2(srcname, dstname)
+ def copytree(self, src, dst, symlinks=0):
+ names = os.listdir(src)
+ try:
+ result = os.stat(dst)
+ except Exception, details:
+ result = None
+ # handle case where result exists but is not a directory
+ if result and not S_ISDIR(result[ST_MODE]):
+ remove(dst)
+ result = None
+ if not result:
+ os.makedirs(dst)
+ if result:
+ names2 = os.listdir(dst)
+ self.removenonmatching(src, dst, names, names2)
+ if not self.copyflag:
+ self.epurate(src, dst, names, names2)
+ for name in names:
+ srcname = os.path.join(src, name)
+ dstname = os.path.join(dst, name)
+ try:
+ if symlinks and os.path.islink(srcname):
+ linkto = os.readlink(srcname)
+ os.symlink(linkto, dstname)
+ elif os.path.isdir(srcname):
+ self.copytree(srcname, dstname, symlinks)
+ else:
+ self.maybecopy(srcname, dstname)
+ except (IOError, os.error), why:
+ message = "Can't copy [%s] to [%s]: [%s]" % (`srcname`, `dstname`,
str(why))
+ log.exception(message)
+ raise IOError, message
+ def epurate(self, sourcedir, destdir, acceptablefiles, existingfiles):
+ """
+ this routine will delete from a set of existing files
+ in a directory the one which are not part of an
+ array of acceptablefiles
+ sourcedir = directory from which the copy is taking place
+ destdir = directory where the epuration is to take place
+ acceptablefiles = array of filenames of files which are accepted at
destination
+ existingfiles = array of filenames which exist at destination
+ """
+ for afile in existingfiles:
+ fullsourcefile = os.path.join(sourcedir, afile)
+ fulldestfile = os.path.join(destdir, afile)
+ if not afile in acceptablefiles:
+ tobedeleted = os.path.join(destdir, afile)
+ result = os.stat(tobedeleted)
+ if S_ISDIR(result[ST_MODE]):
+ log.debug('attempting to remove directory [%s]' %
(`tobedeleted`))
+ shutil.rmtree(tobedeleted)
+ else:
+ log.debug('attempting to remove file [%s]' % (`tobedeleted`))
+ os.remove(tobedeleted)
+ def removenonmatching(self, sourcedir, destdir, acceptablefiles, existingfiles):
+ """
+ this routine will remove from destination
+ the entries which are files at destination and directory at source
+ the entries which are directory at destination and files at source
+ leaves untouched entries which exist both at source and at destination
+ sourcedir = directory from which the copy is taking place
+ destdir = directory where the epuration is to take place
+ acceptablefiles = array of filenames of files which are accepted at
destination
+ existingfiles = array of filenames which exist at destination
+ """
+ removed = []
+ for afile in existingfiles:
+ fullsourcefile = os.path.join(sourcedir, afile)
+ fulldestfile = os.path.join(destdir, afile)
+ if afile in acceptablefiles:
+ if os.path.isdir(fullsourcefile) and not
os.path.isdir(fulldestfile):
+ log.debug('removing file [%s] to be replaced by directory'
+ %(`fulldestfile`))
+ os.remove(fulldestfile)
+ removed.append(afile)
+ elif os.path.isfile(fullsourcefile) and
os.path.isdir(fulldestfile):
+ log.debug('removing directory [%s] to be replaced by file'
+ %(`fulldestfile`))
+ shutil.rmtree(fulldestfile)
+ removed.append(afile)
+ for afile in removed:
+ existingfiles.remove(afile)
+
+ def maybecopy(self, srcname, dstname):
+ """
+ copy a file from srcname to dstname if
+ dstname does not exist
+ or srcname and dstname have different dates
+ or srcname and dstname have different sizes
+ """
+ result = os.stat(srcname)
+ try:
+ result2 = os.stat(dstname)
+ except (Exception), details:
+ result2 = None
+ okcopy = 0
+ if not result2:
+ okcopy = 1
+ elif S_ISDIR(result2[ST_MODE]):
+ shutil.rmtree(dstname)
+ okcopy = 1
+ elif result[ST_SIZE] != result2[ST_SIZE]:
+ okcopy = 1
+ elif result[ST_MTIME] != result2[ST_MTIME]:
+ okcopy = 1
+ if okcopy:
+ log.debug("Attempting copy from [%s] to [%s]" %(`srcname`, `dstname`))
+ shutil.copy2(srcname, dstname)
1.2 +65 -17 gump/python/gump/test/sync.py
Index: sync.py
===================================================================
RCS file: /home/cvs/gump/python/gump/test/sync.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- sync.py 4 Mar 2004 21:38:47 -0000 1.1
+++ sync.py 9 Mar 2004 14:14:33 -0000 1.2
@@ -116,21 +116,21 @@
try:
result = os.stat(self.destination)
except Exception, details:
- raiseIssue(['destination directory was not created', self.destination])
+ self.raiseIssue(['destination directory was not created',
self.destination])
try:
result = os.stat(self.destination_subdir1)
except Exception, details:
- raiseIssue(['destination_subdir1 directory was not created',
self.destination_subdir1])
+ self.raiseIssue(['destination_subdir1 directory was not created',
self.destination_subdir1])
result_source = None
result_destination = None
try:
result_source = os.stat(self.source_alphatxt)
except Exception, details:
- raiseIssue(['file was not created', self.source_alphatxt])
+ self.raiseIssue(['file was not created', self.source_alphatxt])
try:
result_destination = os.stat(self.destination_alphatxt)
except Exception, details:
- raiseIssue(['file was not created', self.destination_alphatxt])
+ self.raiseIssue(['file was not created', self.destination_alphatxt])
log.debug("size of file [%s] is %i" % (`self.destination_alphatxt`,
result_destination[stat.ST_SIZE]))
log.debug("modification date of file [%s] is %s" %
@@ -166,17 +166,17 @@
os.utime(self.destination_alphatxt, (epoch_sometime, epoch_sometime))
mySync.execute()
if os.path.exists(destination_junktxt):
- raiseIssue(['junk file was not deleted', destination_junktxt])
+ self.raiseIssue(['junk file was not deleted', destination_junktxt])
result_source = None
result_destination = None
try:
result_source = os.stat(self.source_alphatxt)
except Exception, details:
- raiseIssue(['file was not created', self.source_alphatxt])
+ self.raiseIssue(['file was not created', self.source_alphatxt])
try:
result_destination = os.stat(self.destination_alphatxt)
except Exception, details:
- raiseIssue(['file was not created', self.destination_alphatxt])
+ self.raiseIssue(['file was not created', self.destination_alphatxt])
log.debug("size of file [%s] is %i" % (`self.destination_alphatxt`,
result_destination[stat.ST_SIZE]))
log.debug("modification date of file [%s] is %s" %
@@ -208,18 +208,18 @@
shutil.copy2(self.source_alphatxt, junk_file2)
mySync.execute()
if os.path.isdir(self.destination_alphatxt):
- raiseIssue(['destination text file remained a directory',
+ self.raiseIssue(['destination text file remained a directory',
self.destination_alphatxt])
result_source = None
result_destination = None
try:
result_source = os.stat(self.source_alphatxt)
except Exception, details:
- raiseIssue(['file was not created', self.source_alphatxt])
+ self.raiseIssue(['file was not created', self.source_alphatxt])
try:
result_destination = os.stat(self.destination_alphatxt)
except Exception, details:
- raiseIssue(['file was not created', self.destination_alphatxt])
+ self.raiseIssue(['file was not created', self.destination_alphatxt])
log.debug("size of file [%s] is %i" % (`self.destination_alphatxt`,
result_destination[stat.ST_SIZE]))
log.debug("modification date of file [%s] is %s" %
@@ -253,18 +253,66 @@
shutil.copy2(junk_source_file1, junk_source_file2)
mySync.execute()
if os.path.isfile(self.destination_alphatxt):
- raiseIssue(['destination text file remained a file',
+ self.raiseIssue(['destination text file remained a file',
self.destination_alphatxt])
self.genericCompare(self.source_alphatxt, self.destination_alphatxt)
- log.debug('finished')
+ def testSymbolicLink(self):
+ """
+ this test only runs on operating systems where os.name will return
+ posix
+ the setUp gets done
+ a symbolic link sl pointing to source_subdir1 gets created
+ the sync gets done
+ we want to check whether sl exists on the destination side
+ as a symbolic link
+ """
+ if os.name == 'posix':
+ dstname = os.path.join(self.source_subdir1, 'myfirstlink')
+ os.symlink('subdir1', dstname)
+ mySync = Sync(self.source, self.destination)
+ mySync.execute()
+ self.genericCompare(self.source, self.destination)
+ def testCopy1(self):
+ """
+ the setUp gets done
+ a sync runs
+ then source_alphatxt is deleted
+ another file source_betatxt is created
+ a sync with the copy flag runs
+ the test will check that :
+ alphatxt remains on the destination side
+ betatxt gets copied
+ """
+ mySync = Sync(self.source, self.destination)
+ mySync.execute()
+ os.remove(self.source_alphatxt)
+ betatxt = "beta.txt"
+ source_betatxt = os.path.join(self.source_subdir1, betatxt)
+ destination_betatxt = os.path.join(self.destination_subdir1, betatxt)
+ myfile = file(source_betatxt, 'w+')
+ myfile.write('Hello World')
+ myfile.close()
+ # Sat, 20 May 2000 12:07:40 +0000
+ sometime=[2000,5,20,12,7,40,5,141,-1]
+ epoch_sometime = time.mktime(sometime)
+ os.utime(source_betatxt, (epoch_sometime, epoch_sometime))
+ mySync = Sync(self.source, self.destination, 1)
+ mySync.execute()
+ # check that alpha.txt was preserved on the destination side
+ if not os.path.exists(self.destination_alphatxt):
+ self.raiseIssue(['file was deleted on the destination side in a copy'
+ + 'operation', self.source_alphatxt])
+ # check that beta.txt was copied
+ self.compareFiles(source_betatxt, destination_betatxt)
+
def genericCompare(self, mysource, mydestination):
"""
compare 2 directories source and destination
"""
if not os.path.isdir(mysource):
- raiseIssue([mysource, ' not a directory'])
+ self.raiseIssue([mysource, ' not a directory'])
if not os.path.isdir(mydestination):
- raiseIssue([mydestination, ' not a directory'])
+ self.raiseIssue([mydestination, ' not a directory'])
names = os.listdir(mysource)
for aname in names:
inode_source = os.path.join(mysource, aname)
@@ -273,7 +321,7 @@
self.compareFiles(inode_source, inode_dest)
elif os.path.islink(inode_source):
if not os.path.islink(inode_dest):
- raiseIssue([inode_dest, ' not a symbolic link'])
+ self.raiseIssue([inode_dest, ' not a symbolic link'])
linkto_source = os.readlink(inode_source)
linkto_dest = os.readlink(inode_dest)
self.assertTrue([inode_dest, ' points to ', linkto_source ],
@@ -291,11 +339,11 @@
try:
result_source = os.stat(inode_source)
except Exception, details:
- raiseIssue(['could not stat ', inode_source])
+ self.raiseIssue(['could not stat ', inode_source])
try:
result_dest = os.stat(inode_dest)
except Exception, details:
- raiseIssue(['could not stat ', inode_dest])
+ self.raiseIssue(['could not stat ', inode_dest])
self.assertTrue("modtime is equal for [%s] compared to [%s]"
%(`inode_source`,`inode_dest`),
result_source[stat.ST_MTIME]==result_dest[stat.ST_MTIME])
---------------------------------------------------------------------
To unsubscribe, e-mail: [EMAIL PROTECTED]
For additional commands, e-mail: [EMAIL PROTECTED]
