Hashar has uploaded a new change for review.
https://gerrit.wikimedia.org/r/50371
Change subject: pep8 whitespaces fixing
......................................................................
pep8 whitespaces fixing
W293 blank line contains whitespace
W291 trailing whitespace
E101 indentation contains mixed spaces and tabs
E201 whitespace after '('
E202 whitespace before ')'
Change-Id: I198cb8ac04071f2d461990910e0a9a0ec0872cbf
---
M WikiDump.py
M monitor.py
M worker.py
3 files changed, 86 insertions(+), 86 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/dumps
refs/changes/71/50371/1
diff --git a/WikiDump.py b/WikiDump.py
index 49699b4..ec03431 100644
--- a/WikiDump.py
+++ b/WikiDump.py
@@ -58,14 +58,14 @@
def dumpFile(filename, text):
"""Dump a string to a file, as atomically as possible, via a temporary
file in the same directory."""
-
+
# I'd use os.tempnam() here but it whines about symlinks attacks.
tempFilename = filename + ".tmp"
-
+
file = open(tempFilename, "wt")
file.write(text)
file.close()
-
+
# This may fail across filesystems or on Windows.
# Of course nothing else will work on Windows. ;)
os.rename(tempFilename, filename)
@@ -130,7 +130,7 @@
}
conf = ConfigParser.SafeConfigParser(defaults)
conf.read(files)
-
+
self.dbList = dbList(conf.get("wiki", "dblist"))
self.privateList = dbList(conf.get("wiki", "privatelist"))
biglistFile = conf.get("wiki", "biglist")
@@ -143,32 +143,32 @@
self.flaggedRevsList = dbList(flaggedRevsFile)
else:
self.flaggedRevsList = []
-
+
self.wikiDir = conf.get("wiki", "dir")
self.forceNormal = conf.getint("wiki", "forceNormal")
self.halt = conf.getint("wiki", "halt")
-
+
self.publicDir = conf.get("output", "public")
self.privateDir = conf.get("output", "private")
self.webRoot = conf.get("output", "webroot")
self.index = conf.get("output", "index")
self.templateDir = conf.get("output", "templateDir")
-
+
self.adminMail = conf.get("reporting", "adminmail")
self.mailFrom = conf.get("reporting", "mailfrom")
self.smtpServer = conf.get("reporting", "smtpserver")
self.staleAge = conf.getint("reporting", "staleAge")
-
+
self.dbUser = conf.get("database", "user")
self.dbPassword = conf.get("database", "password")
-
+
self.php = conf.get("tools", "php")
self.bzip2 = conf.get("tools", "bzip2")
self.sevenzip = conf.get("tools", "sevenzip")
self.mysql = conf.get("tools", "mysql")
-
+
self.keep = conf.getint("cleanup", "keep")
-
+
def dbListByAge(self):
"""
Sort wikis in reverse order of last successful dump :
@@ -202,11 +202,11 @@
available.append((dumpFailed, age, db))
available.sort()
return [db for (failed, age, db) in available]
-
+
def readTemplate(self, name):
template = os.path.join(self.templateDir, name)
return readFile(template)
-
+
def mail(self, subject, body):
"""Send out a quickie email."""
message = email.MIMEText.MIMEText(body)
@@ -229,19 +229,19 @@
self.dbName = dbName
self.date = None
self.watchdog = None
-
+
def isPrivate(self):
return self.dbName in self.config.privateList
-
+
def isBig(self):
return self.dbName in self.config.bigList
def hasFlaggedRevs(self):
return self.dbName in self.config.flaggedRevsList
-
+
def isLocked(self):
return os.path.exists(self.lockFile())
-
+
def isStale(self):
if not self.isLocked():
return False
@@ -253,21 +253,21 @@
return False
# Paths and directories...
-
+
def publicDir(self):
if self.isPrivate():
return self.privateDir()
else:
return os.path.join(self.config.publicDir, self.dbName)
-
+
def privateDir(self):
return os.path.join(self.config.privateDir, self.dbName)
-
+
def webDir(self):
return "/".join((self.config.webRoot, self.dbName))
-
+
# Actions!
-
+
def lock(self):
if not os.path.isdir(self.privateDir()):
try:
@@ -279,20 +279,20 @@
f = atomicCreate(self.lockFile(), "w")
f.write("%s %d" % (socket.getfqdn(), os.getpid()))
f.close()
-
+
self.watchdog = LockWatchdog(self.lockFile())
self.watchdog.start()
return True
-
+
def unlock(self):
if self.watchdog:
self.watchdog.stopWatching()
self.watchdog = None
os.remove(self.lockFile())
-
+
def setDate(self, date):
self.date = date
-
+
def cleanupStaleLock(self):
date = self.latestDump()
if date:
@@ -300,15 +300,15 @@
self.writeStatus(self.reportStatusLine(
"<span class=\"failed\">dump aborted</span>"))
self.unlock()
-
+
def writeIndex(self, html):
index = os.path.join(self.publicDir(), self.date, "index.html")
dumpFile(index, html)
-
+
def writeStatus(self, message):
index = os.path.join(self.publicDir(), self.date, "status.html")
dumpFile(index, message)
-
+
def statusLine(self):
date = self.latestDump()
if date:
@@ -319,7 +319,7 @@
return self.reportStatusLine("missing status
record")
else:
return self.reportStatusLine("has not yet been dumped")
-
+
def reportStatusLine(self, status, error=False):
if error:
# No state information, hide the timestamp
@@ -356,39 +356,39 @@
return []
dates.sort()
return dates
-
+
# private....
-
+
def lockFile(self):
return os.path.join(self.privateDir(), "lock")
-
+
def lockAge(self):
return fileAge(self.lockFile())
class LockWatchdog(threading.Thread):
"""Touch the given file every 10 seconds until asked to stop."""
-
+
# For emergency aborts
threads = []
-
+
def __init__(self, lockfile):
threading.Thread.__init__(self)
self.lockfile = lockfile
self.trigger = threading.Event()
self.finished = threading.Event()
-
+
def stopWatching(self):
"""Run me outside..."""
# Ask the thread to stop...
self.trigger.set()
-
+
# Then wait for it, to ensure that the lock file
# doesn't get touched again after we delete it on
# the main thread.
self.finished.wait(10)
self.finished.clear()
-
+
def run(self):
LockWatchdog.threads.append(self)
while not self.trigger.isSet():
@@ -397,7 +397,7 @@
self.trigger.clear()
self.finished.set()
LockWatchdog.threads.remove(self)
-
+
def touchLock(self):
"""Run me inside..."""
os.utime(self.lockfile, None)
diff --git a/monitor.py b/monitor.py
index be2e429..ca4dfe2 100644
--- a/monitor.py
+++ b/monitor.py
@@ -9,32 +9,32 @@
def generateIndex():
running = False
states = []
-
+
for dbName in config.dbListByAge():
wiki = WikiDump.Wiki(config, dbName)
if wiki.isStale():
print dbName + " is stale"
wiki.cleanupStaleLock()
if wiki.isLocked():
- try:
- f = open( wiki.lockFile(), 'r' )
- (host, pid) = f.readline().split(" ")
- f.close()
- print dbName, "is locked by pid", pid, "on", host
- except:
- print dbName, "is locked"
+ try:
+ f = open(wiki.lockFile(), 'r')
+ (host, pid) = f.readline().split(" ")
+ f.close()
+ print dbName, "is locked by pid", pid, "on",
host
+ except:
+ print dbName, "is locked"
running = running or wiki.isLocked()
states.append(wiki.statusLine())
-
+
if running:
status = "Dumps are in progress..."
else:
status = "Dump process is idle."
-
+
return config.readTemplate("progress.html") % {
"status": status,
"items": "\n".join(states)}
-
+
def updateIndex():
outputFileName = os.path.join(config.publicDir, config.index)
diff --git a/worker.py b/worker.py
index 6092f8a..da68de9 100644
--- a/worker.py
+++ b/worker.py
@@ -70,7 +70,7 @@
self.dbName = wiki.dbName
self.prefetch = prefetch
self.spawn = spawn
-
+
if date:
# Override, continuing a past dump?
self.date = date
@@ -265,9 +265,9 @@
XmlLogging("Pull out all logging data")]
if self.wiki.hasFlaggedRevs():
self.items.append(
- PublicTable( "flaggedpages", "This contains a
row for each flagged article, containing the stable revision ID, if the lastest
edit was flagged, and how long edits have been pending." ))
+ PublicTable("flaggedpages", "This contains a
row for each flagged article, containing the stable revision ID, if the lastest
edit was flagged, and how long edits have been pending."))
self.items.append(
- PublicTable( "flaggedrevs", "This contains a
row for each flagged revision, containing who flagged it, when it was flagged,
reviewer comments, the flag values, and the quality tier those flags fall
under." ))
+ PublicTable("flaggedrevs", "This contains a row
for each flagged revision, containing who flagged it, when it was flagged,
reviewer comments, the flag values, and the quality tier those flags fall
under."))
if not self.wiki.isBig():
self.items.append(
@@ -465,7 +465,7 @@
# Report on the file size & status of the current output and output a
link if were done
def reportFile(self, file, status):
filepath = self.publicPath(file)
- if status == "in-progress" and exists (filepath):
+ if status == "in-progress" and exists(filepath):
size = prettySize(getsize(filepath))
return "<li class='file'>%s %s (written) </li>" %
(file, size)
elif status == "done" and exists(filepath):
@@ -668,7 +668,7 @@
def listFiles(self, runner):
return ["stub-meta-history.xml.gz",
"stub-meta-current.xml.gz",
- "stub-articles.xml.gz",]
+ "stub-articles.xml.gz", ]
def run(self, runner):
history = runner.publicPath("stub-meta-history.xml.gz")
@@ -679,16 +679,16 @@
os.remove(filename)
command = """
%s -q %s/maintenance/dumpBackup.php \
- --wiki=%s \
- --full \
- --stub \
- --report=10000 \
- %s \
- --server=%s \
- --output=gzip:%s \
- --output=gzip:%s \
+ --wiki=%s \
+ --full \
+ --stub \
+ --report=10000 \
+ %s \
+ --server=%s \
+ --output=gzip:%s \
+ --output=gzip:%s \
--filter=latest \
- --output=gzip:%s \
+ --output=gzip:%s \
--filter=latest \
--filter=notalk \
--filter=namespace:\!NS_USER \
@@ -722,12 +722,12 @@
os.remove(logging)
command = """
%s -q %s/maintenance/dumpBackup.php \
- --wiki=%s \
- --logs \
- --report=10000 \
- %s \
- --server=%s \
- --output=gzip:%s \
+ --wiki=%s \
+ --logs \
+ --report=10000 \
+ %s \
+ --server=%s \
+ --output=gzip:%s \
""" % shellEscape((
runner.config.php,
runner.config.wikiDir,
@@ -804,13 +804,13 @@
dumpCommand = """
%s -q %s/maintenance/dumpTextPass.php \
- --wiki=%s \
- %s \
- %s \
- %s \
- --report=1000 \
- --server=%s \
- %s""" % shellEscape((
+ --wiki=%s \
+ %s \
+ %s \
+ %s \
+ --report=1000 \
+ --server=%s \
+ %s""" % shellEscape((
runner.config.php,
runner.config.wikiDir,
runner.dbName,
@@ -914,12 +914,12 @@
def run(self, runner):
command = """
%s -q %s/maintenance/dumpBackup.php \
- --wiki=%s \
- --plugin=AbstractFilter:%s/extensions/ActiveAbstract/AbstractFilter.php \
- --current \
- --report=1000 \
- %s \
- --server=%s \
+ --wiki=%s \
+ --plugin=AbstractFilter:%s/extensions/ActiveAbstract/AbstractFilter.php
\
+ --current \
+ --report=1000 \
+ %s \
+ --server=%s \
""" % shellEscape((
runner.config.php,
runner.config.wikiDir,
@@ -929,9 +929,9 @@
runner.dbServer))
for variant in self._variants(runner):
command = command + """ --output=file:%s \
- --filter=namespace:NS_MAIN \
- --filter=noredirect \
- --filter=abstract%s \
+ --filter=namespace:NS_MAIN \
+ --filter=noredirect \
+ --filter=abstract%s \
""" % shellEscape((
runner.publicPath(self._variantFile(variant)),
self._variantOption(variant)))
--
To view, visit https://gerrit.wikimedia.org/r/50371
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I198cb8ac04071f2d461990910e0a9a0ec0872cbf
Gerrit-PatchSet: 1
Gerrit-Project: operations/dumps
Gerrit-Branch: master
Gerrit-Owner: Hashar <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
