ArielGlenn has uploaded a new change for review.
https://gerrit.wikimedia.org/r/234973
Change subject: dump lists for rsync: copy partial or in progress dump if most
recent
......................................................................
dump lists for rsync: copy partial or in progress dump if most recent
this means labs will get copies of dumps in progress, which will
mean files show up as soon as they are available instead of waiting
for all stages of all dumps to complete
Change-Id: I154e1da7b4f3052260687464662eead3070b7107
---
M modules/snapshot/files/dumps/list-last-n-good-dumps.py
1 file changed, 64 insertions(+), 26 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/73/234973/1
diff --git a/modules/snapshot/files/dumps/list-last-n-good-dumps.py
b/modules/snapshot/files/dumps/list-last-n-good-dumps.py
index 275c595..98efd3b 100755
--- a/modules/snapshot/files/dumps/list-last-n-good-dumps.py
+++ b/modules/snapshot/files/dumps/list-last-n-good-dumps.py
@@ -198,6 +198,41 @@
else:
return os.path.join(self.config.publicDir, name)
+ def getDirStatus(self, dirToCheck, day):
+ if isdir(os.path.join(dirToCheck, day)):
+ try:
+ statusFile = os.path.join(dirToCheck, day, "status.html")
+ fd = open(statusFile, "r")
+ text = fd.read()
+ fd.close()
+ except:
+ # if there is no status file, the dir could have any
+ # kind of random junk in it so don't risk it
+ return None
+ return text
+ return None
+
+ def getFirstDir(self, dirs, dirToCheck):
+ if not dirs:
+ return False
+ for day in dirs:
+ text = self.getDirStatus(dirToCheck, day)
+ if text is None:
+ continue
+ if not "failed" in text:
+ return day
+
+ # no dump in there that's not failed. meh.
+ # try again and take the first one we can read
+ for day in dirs:
+ text = self.getDirStatus(dirToCheck, day)
+ if text is None:
+ continue
+ return day
+
+ # no dumps with status we can read. give up
+ return False
+
def listDumpsForProject(self, project):
"""get list of dump directories for a given project
ordered by good dumps first, most recent to oldest, then
@@ -214,33 +249,36 @@
dirs.sort()
dirs.reverse()
- dirsGood = []
dirsFailed = []
- dirsIncomplete = []
+ dirsComplete = []
+ dirsOther = []
+
+ dirsReported = []
+
+ dirFirst = self.getFirstDir(dirs, dirToCheck)
+ if not dirFirst:
+ # never dumped
+ return dirsReported
+
for day in dirs:
- if isdir(os.path.join(dirToCheck, day)):
- try:
- statusFile = os.path.join(dirToCheck, day, "status.html")
- fd = open(statusFile, "r")
- text = fd.read()
- fd.close()
- except:
- # if there is no status file, the dir could have any
- # kind of random junk in it so don't risk it
- continue
- if not "Dump complete" in text:
- dirsIncomplete.append(day)
- elif "failed" in text:
- dirsFailed.append(day)
- else:
- dirsGood.append(day)
- # we list good (complete not failed) dumps first, then failed dumps,
- # then incomplete dumps.
- dirs = []
- dirs.extend(dirsGood)
- dirs.extend(dirsFailed)
- dirs.extend(dirsIncomplete)
- return dirs
+ if day == dirFirst:
+ continue
+ text = self.getDirStatus(dirToCheck, day)
+ if text is None:
+ continue
+
+ if "failed" in text:
+ dirsFailed.append(day)
+ elif "Dump complete" in text:
+ dirsComplete.append(day)
+ else:
+ dirsOther.append(day)
+
+ dirsReported.append(dirFirst)
+ dirsReported.extend(dirsComplete)
+ dirsReported.extend(dirsOther)
+ dirsReported.append(dirsFailed)
+ return dirsReported
def listFileTemplates(self):
"""list the templates for filenames that were provided
@@ -267,7 +305,7 @@
up the files that are part of the public run, not scratch or other
files, and the filenames are either full paths or are relative
to the base directory of the public dumps, depending on
- user-specified options"""
+ user-specified options."""
filesWanted = []
if self.fileListTemplate:
dirContents = os.listdir(dirName)
--
To view, visit https://gerrit.wikimedia.org/r/234973
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I154e1da7b4f3052260687464662eead3070b7107
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
