ArielGlenn has uploaded a new change for review.
https://gerrit.wikimedia.org/r/230507
Change subject: dumps: fix link cleanup for stubs/content parallel runs
......................................................................
dumps: fix link cleanup for stubs/content parallel runs
in this case there will be a recombine phase to generate the
final file; we don't want to remove the latest link until that
job completes
Change-Id: If39cd6028304d70dcdccfee3291840cd82304599
---
M xmldumps-backup/worker.py
1 file changed, 8 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/dumps
refs/changes/07/230507/1
diff --git a/xmldumps-backup/worker.py b/xmldumps-backup/worker.py
index 98d0d10..4392c01 100644
--- a/xmldumps-backup/worker.py
+++ b/xmldumps-backup/worker.py
@@ -1999,7 +1999,7 @@
checkpoint =
item.checkpointFile.checkpoint
for d in dumpNames:
-
self.symLinks.removeSymLinksFromOldRuns(self.wiki.date, d, chunk, checkpoint )
+
self.symLinks.removeSymLinksFromOldRuns(self.wiki.date, d, chunk, checkpoint,
onlychunks=item.onlychunks)
self.feeds.cleanupFeeds()
@@ -2071,7 +2071,7 @@
# if the args are False or None, we remove all the old links for all
values of the arg.
# example: if chunk is False or None then we remove all old values for
all chunks
# "old" means "older than the specified datestring".
- def removeSymLinksFromOldRuns(self, dateString, dumpName=None,
chunk=None, checkpoint=None):
+ def removeSymLinksFromOldRuns(self, dateString, dumpName=None,
chunk=None, checkpoint=None, onlychunks=False):
# fixme this needs to do more work if there are chunks or
checkpoint files linked in here from
# earlier dates. checkpoint ranges change, and configuration of
chunks changes too, so maybe
# old files still exist and the links need to be removed
because we have newer files for the
@@ -2090,7 +2090,7 @@
# fixme check that these are ok
if the value is None
if dumpName and
(fileObj.dumpName != dumpName):
continue
- if chunk and (fileObj.chunk !=
chunk):
+ if (chunk or onlychunks) and
(fileObj.chunk != chunk):
continue
if checkpoint and
(fileObj.checkpoint != checkpoint):
continue
@@ -2159,6 +2159,8 @@
# called last by child classes in their constructor, so that
# their functions overriding things like the dumpbName can
# be set up before we use them to set class attributes.)
+ if not hasattr(self, 'onlychunks'):
+ self.onlychunks = False
if not hasattr(self, '_chunksEnabled'):
self._chunksEnabled = False
if not hasattr(self, '_checkpointsEnabled'):
@@ -2817,6 +2819,7 @@
self._chunks = chunks
if self._chunks:
self._chunksEnabled = True
+ self.onlychunks = True
self.historyDumpName = "stub-meta-history"
self.currentDumpName = "stub-meta-current"
self.articlesDumpName = "stub-articles"
@@ -3032,6 +3035,7 @@
self._chunks = chunks
if self._chunks:
self._chunksEnabled = True
+ self.onlychunks = True
self._pageID = {}
self._chunkToDo = chunkToDo
@@ -3801,6 +3805,7 @@
self._chunks = chunks
if self._chunks:
self._chunksEnabled = True
+ self.onlychunks = True
Dump.__init__(self, name, desc)
def getDumpName(self):
--
To view, visit https://gerrit.wikimedia.org/r/230507
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If39cd6028304d70dcdccfee3291840cd82304599
Gerrit-PatchSet: 1
Gerrit-Project: operations/dumps
Gerrit-Branch: ariel
Gerrit-Owner: ArielGlenn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
