Faidon has uploaded a new change for review.
https://gerrit.wikimedia.org/r/62419
Change subject: swiftrepl: add a much faster sync_deletes() method
......................................................................
swiftrepl: add a much faster sync_deletes() method
Instead of looping through all files and issuing HEADs for each of them,
compare container listing pages between source & destination and HEAD
only on the files existing on destination but not on source.
This cuts down runtime from weeks/months to ~4 hours for commons
containers.
Change-Id: Ibcb532a38605a6cf7006564ef6e35686404ce5ff
---
M swiftrepl/swiftrepl.py
1 file changed, 66 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/software
refs/changes/19/62419/1
diff --git a/swiftrepl/swiftrepl.py b/swiftrepl/swiftrepl.py
index aeacf12..c1d8231 100644
--- a/swiftrepl/swiftrepl.py
+++ b/swiftrepl/swiftrepl.py
@@ -337,7 +337,7 @@
print "FINISHED:", srccontainer.name
-def sync_deletes(srccontainer, srcconnpool, dstconnpool):
+def sync_deletes_slow(srccontainer, srcconnpool, dstconnpool):
global NOBJECT
dstconn = dstconnpool.get()
@@ -387,6 +387,71 @@
srcconnpool.put(srccontainer.conn)
srccontainer.conn = None
+def sync_deletes(srccontainer, srcconnpool, dstconnpool):
+ global NOBJECT
+
+ dstconn = dstconnpool.get()
+ try:
+ dstcontainer = dstconn.get_container(srccontainer.name)
+ except cloudfiles.errors.NoSuchContainer as e:
+ # Destination container doesn't exist; nothing to delete
+ return
+ finally:
+ dstconnpool.put(dstconn)
+
+ srclimit = dstlimit = 50*NOBJECT
+ # fetch 20% more of source
+ srclimit = int(srclimit * 1.2)
+
+ last = ''
+ deletes, processed = 0, 0
+ while True:
+
+ dstobjects = get_container_objects(dstcontainer,
limit=dstlimit, marker=last, connpool=dstconnpool)
+
+ # bail-out early on empty containers
+ if len(dstobjects) == 0:
+ break
+
+ srcobjects = get_container_objects(srccontainer,
limit=srclimit, marker=last, connpool=srcconnpool)
+
+ dstset = set([ obj.name for obj in dstobjects ])
+ srcset = set([ obj.name for obj in srcobjects ])
+ diff = dstset - srcset
+
+ for dstname in diff:
+ # do a HEAD to make sure it's gone
+ srccontainer.conn = srcconnpool.get()
+ try:
+ srcobj = srccontainer.get_object(dstname)
# Does a HEAD
+ except cloudfiles.errors.NoSuchObject as e:
+ # File was deleted on the source
+ print "Deleting object",
dstname.encode("ascii", errors="ignore")
+
+ deletes += 1
+ dstcontainer.conn = dstconnpool.get()
+ try:
+ dstcontainer.delete_object(dstname)
+ finally:
+ dstconnpool.put(dstcontainer.conn)
+ dstcontainer.conn = None
+ finally:
+ srcconnpool.put(srccontainer.conn)
+ srccontainer.conn = None
+
+ last = dstobjects[-1].name.encode("utf-8")
+ processed += len(dstobjects)
+
+ pct = lambda x, y: y != 0 and int(float(x) / y * 100) or 0
+ print "STATS: %s processed: %d/%d (%d%%), deleted: %d" %
(srccontainer.name,
+
processed, dstcontainer.object_count,
+
pct(processed, dstcontainer.object_count),
+ deletes)
+
+ if len(dstobjects) < dstlimit:
+ break
+ print "FINISHED:", srccontainer.name
+
def replicator_thread(*args, **kwargs):
while True:
try:
--
To view, visit https://gerrit.wikimedia.org/r/62419
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibcb532a38605a6cf7006564ef6e35686404ce5ff
Gerrit-PatchSet: 1
Gerrit-Project: operations/software
Gerrit-Branch: master
Gerrit-Owner: Faidon <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
