ArielGlenn has uploaded a new change for review.
https://gerrit.wikimedia.org/r/248867
Change subject: dumps: camelcases in wikiDumps.py (part 2)
......................................................................
dumps: camelcases in wikiDumps.py (part 2)
also a few more fixes in the adds changed dumps for the new location
of the dumps library files and to make it whine a little less with
pylint
Change-Id: Ide6dc299cce157d03f41310172935ae4269ba0b0
---
M xmldumps-backup/incrementals/IncrDumpLib.py
M xmldumps-backup/incrementals/generateincrementals.py
2 files changed, 55 insertions(+), 47 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/dumps
refs/changes/67/248867/1
diff --git a/xmldumps-backup/incrementals/IncrDumpLib.py
b/xmldumps-backup/incrementals/IncrDumpLib.py
index 62a9512..76bc1b3 100644
--- a/xmldumps-backup/incrementals/IncrDumpLib.py
+++ b/xmldumps-backup/incrementals/IncrDumpLib.py
@@ -3,8 +3,8 @@
import sys
import re
import ConfigParser
-import WikiDump
-from WikiDump import FileUtils, TimeUtils, MiscUtils
+import dumps.WikiDump
+from dumps.WikiDump import FileUtils, TimeUtils, MiscUtils
from os.path import exists
import socket
import subprocess
@@ -27,7 +27,7 @@
return
os.path.join(self.incrDir.getIncDir(self.wikiName),self.getFileName())
def getFileInfo(self):
- return FileUtils.fileInfo(self.getPath())
+ return FileUtils.file_info(self.getPath())
class MaxRevIDFile(ContentFile):
def getFileName(self):
@@ -88,11 +88,11 @@
def getStatus(self, date = None):
status = ""
if exists(self.statusFile.getPath(date)):
- status = FileUtils.readFile(self.statusFile.getPath(date)).rstrip()
+ status =
FileUtils.read_file(self.statusFile.getPath(date)).rstrip()
return(status)
def setStatus(self, status):
- FileUtils.writeFileInPlace(self.statusFile.getPath(),status,
self._config.fileperms)
+ FileUtils.write_file_in_place(self.statusFile.getPath(),status,
self._config.fileperms)
class Lock(object):
def __init__(self, config, date, wikiName):
@@ -108,7 +108,7 @@
try:
if not exists(self._config.incrementalsDir):
os.makedirs(self._config.incrementalsDir)
- f = FileUtils.atomicCreate(self.lockFile.getPath(), "w")
+ f = FileUtils.atomic_create(self.lockFile.getPath(), "w")
f.write("%s %d" % (socket.getfqdn(), os.getpid()))
f.close()
return True
@@ -151,7 +151,7 @@
self.wikiName = wikiName
self.lockFile = MaxRevIDLockFile(self._config, self.date,
self.wikiName)
-class Config(WikiDump.Config):
+class Config(dumps.WikiDump.Config):
def __init__(self, configFile=False):
self.projectName = False
@@ -177,7 +177,7 @@
"delay": "43200",
"maxrevidstaleinterval": "3600",
#"database": {
- # moved defaults to getDbUserAndPassword
+ # moved defaults to get_db_user_and_password
#"tools": {
"mediawiki" : "",
"php": "/bin/php",
@@ -202,15 +202,18 @@
print "The mandatory setting 'mediawiki' in the section 'wiki' was
not defined."
raise ConfigParser.NoOptionError('wiki','mediawiki')
+ self.db_user = None
+ self.db_password = None
+
self.parseConfFile()
- self.getDbUserAndPassword() # get from MW adminsettings file if not
set in conf file
def parseConfFile(self):
self.mediawiki = self.conf.get("wiki", "mediawiki")
- self.allWikisList = MiscUtils.dbList(self.conf.get("wiki",
"allwikislist"))
- self.privateWikisList = MiscUtils.dbList(self.conf.get("wiki",
"privatewikislist"))
- self.closedWikisList = MiscUtils.dbList(self.conf.get("wiki",
"closedwikislist"))
- self.skipWikisList = MiscUtils.dbList(self.conf.get("wiki",
"skipwikislist"))
+ self.wiki_dir = self.mediawiki
+ self.allWikisList = MiscUtils.db_list(self.conf.get("wiki",
"allwikislist"))
+ self.privateWikisList = MiscUtils.db_list(self.conf.get("wiki",
"privatewikislist"))
+ self.closedWikisList = MiscUtils.db_list(self.conf.get("wiki",
"closedwikislist"))
+ self.skipWikisList = MiscUtils.db_list(self.conf.get("wiki",
"skipwikislist"))
if not self.conf.has_section('output'):
self.conf.add_section('output')
@@ -240,19 +243,19 @@
self.keep = self.conf.getint("cleanup", "keep")
self.wikiDir = self.mediawiki # the parent class methods want this
- self.dbUser = None
- self.dbPassword = None
+ self.db_user = None
+ self.db_password = None
if not self.conf.has_section('database'):
self.conf.add_section('database')
if self.conf.has_option('database', 'user'):
- self.dbUser = self.conf.get("database", "user")
+ self.db_user = self.conf.get("database", "user")
if self.conf.has_option('database', 'password'):
- self.dbPassword = self.conf.get("database", "password")
- self.getDbUserAndPassword() # get from MW adminsettings file if not
set in conf file
+ self.db_password = self.conf.get("database", "password")
+ self.get_db_user_and_password() # get from MW adminsettings file if
not set in conf file
def readTemplate(self, name):
template = os.path.join(self.templateDir, name)
- return FileUtils.readFile(template)
+ return FileUtils.read_file(template)
class RunSimpleCommand(object):
def runWithOutput(command, maxtries = 3, shell=False):
@@ -331,9 +334,9 @@
"""Put together a command to execute an sql query to the server for
this DB."""
if (not exists( self.config.mysql ) ):
raise BackupError("mysql command %s not found" % self.config.mysql)
- command = "/bin/echo '%s' | %s -h %s -u %s " % ( query,
self.config.mysql, self.dbServer, self.config.dbUser )
- if self.config.dbPassword != "":
- command = command + "-p" + self.config.dbPassword
+ command = "/bin/echo '%s' | %s -h %s -u %s " % ( query,
self.config.mysql, self.dbServer, self.config.db_user )
+ if self.config.db_password != "":
+ command = command + "-p" + self.config.db_password
command = command + " -r --silent " + self.wikiName
return command
@@ -409,7 +412,7 @@
elif revidok:
maxRevIDFile = MaxRevIDFile(self._config, dump,
self.wikiName)
if exists(maxRevIDFile.getPath()):
- revid =
FileUtils.readFile(maxRevIDFile.getPath().rstrip())
+ revid =
FileUtils.read_file(maxRevIDFile.getPath().rstrip())
if int(revid) > 0:
previous = dump
else:
diff --git a/xmldumps-backup/incrementals/generateincrementals.py
b/xmldumps-backup/incrementals/generateincrementals.py
index b85360f..bc683aa 100644
--- a/xmldumps-backup/incrementals/generateincrementals.py
+++ b/xmldumps-backup/incrementals/generateincrementals.py
@@ -12,7 +12,7 @@
from IncrDumpLib import MaxRevIDFile, StatusFile, IndexFile
from IncrDumpLib import StubFile, RevsFile, MD5File, IncDumpDirs
from IncrDumpLib import IncrDumpLock, StatusInfo
-from WikiDump import FileUtils, TimeUtils
+from dumps.WikiDump import FileUtils, TimeUtils
from os.path import exists
import hashlib
import traceback
@@ -38,15 +38,15 @@
self.getMaxRevID(wikiName)
if not self.dryrun:
fileObj = MaxRevIDFile(self._config, self.date, wikiName)
- FileUtils.writeFileInPlace(fileObj.getPath(), self.maxID,
- self._config.fileperms)
+ FileUtils.write_file_in_place(fileObj.getPath(), self.maxID,
+ self._config.fileperms)
def readMaxRevIDFromFile(self, wikiName, date=None):
if date is None:
date = self.date
try:
fileObj = MaxRevIDFile(self._config, date, wikiName)
- return FileUtils.readFile(fileObj.getPath().rstrip())
+ return FileUtils.read_file(fileObj.getPath().rstrip())
except:
return None
@@ -82,8 +82,8 @@
text = text + "<li>" + result + "</li>\n"
indexText = (self._config.readTemplate("incrs-index.html")
% {"items": text})
- FileUtils.writeFileInPlace(self.indexFile.getPath(),
- indexText, self._config.fileperms)
+ FileUtils.write_file_in_place(self.indexFile.getPath(),
+ indexText, self._config.fileperms)
def doOneWiki(self, w, date=None):
if (w not in self._config.privateWikisList and
@@ -129,7 +129,7 @@
revs = RevsFile(self._config, incrDate, w)
(revsDate, revsSize) = revs.getFileInfo()
- log(verbose, "revs for %s %s %s"
+ log(self.verbose, "revs for %s %s %s"
% (w, safe(revsDate), safe(revsSize)))
if revsDate:
revsText = ("revs: %s (size %s)"
@@ -140,7 +140,7 @@
revsText = None
stat = StatusFile(self._config, incrDate, w)
- statContents = FileUtils.readFile(stat.getPath())
+ statContents = FileUtils.read_file(stat.getPath())
log(self.verbose, "status for %s %s" % (w, safe(statContents)))
if statContents:
statText = "(%s)" % (statContents)
@@ -211,12 +211,12 @@
os.makedirs(self.incrDir.getIncDir(self.wikiName))
status = self.statusInfo.getStatus()
- if status == "done" and not forcerun:
+ if status == "done" and not self.forcerun:
log(self.verbose, "wiki %s skipped, adds/changes dump already"
" complete" % self.wikiName)
return retCodes.OK
- if not dryrun:
+ if not self.dryrun:
lock = IncrDumpLock(self._config, self.date, self.wikiName)
if not lock.getLock():
log(self.verbose, "wiki %s skipped, wiki is locked,"
@@ -237,27 +237,27 @@
if not prevRevID:
return retCodes.FAILED
- if doStubs:
+ if self.doStubs:
if not self.dumpStub(prevRevID, maxRevID):
return retCodes.FAILED
- if doRevs:
+ if self.doRevs:
if not self.dumpRevs():
return retCodes.FAILED
- if not dryrun:
+ if not self.dryrun:
if not self.md5sums():
return retCodes.FAILED
self.statusInfo.setStatus("done")
lock.unlock()
- if doIndexUpdate:
- index = Index(config, date, verbose)
+ if self.doIndexUpdate:
+ index = Index(self._config, self.date, self.verbose)
index.doAllWikis()
except:
if (self.verbose):
traceback.print_exc(file=sys.stdout)
- if not dryrun:
+ if not self.dryrun:
lock.unlock()
return retCodes.FAILED
log(self.verbose, "Success! Wiki %s incremental dump complete."
@@ -297,7 +297,7 @@
log(self.verbose, "Wiki %s retrieving prevRevId from db."
% self.wikiName)
prevRevIDObj = MaxRevID(self._config, prevDate,
- cutoffFromDate(prevDate),
+ cutoffFromDate(prevDate, self._config),
self.dryrun)
prevRevIDObj.recordMaxRevID(self.wikiName)
prevRevID = prevRevIDObj.maxID
@@ -324,7 +324,7 @@
"--output=gzip:%s" % self.stubFile.getPath(),
"--revrange", "--revstart=%s" % startRevID,
"--revend=%s" % endRevID])
- if dryrun:
+ if self.dryrun:
print "would run command for stubs dump:", command
else:
error = RunSimpleCommand.runWithNoOutput(command, shell=False)
@@ -344,7 +344,7 @@
"--quiet",
"--spawn=%s" % self._config.php,
"--output=bzip2:%s" % self.revsFile.getPath()])
- if dryrun:
+ if self.dryrun:
print "would run command for revs dump:", command
else:
error = RunSimpleCommand.runWithNoOutput(command, shell=False)
@@ -376,8 +376,8 @@
files.append(self.revsFile.getPath())
for f in files:
text = text + "%s\n" % self.md5sumOneFile(f)
- FileUtils.writeFileInPlace(md5File.getPath(),
- text, self._config.fileperms)
+ FileUtils.write_file_in_place(md5File.getPath(),
+ text, self._config.fileperms)
return True
except:
return False
@@ -436,7 +436,7 @@
time.sleep(300)
-def cutoffFromDate(date):
+def cutoffFromDate(date, config):
return time.strftime("%Y%m%d%H%M%S",
time.gmtime(calendar.timegm(time.strptime(
date + "235900UTC", "%Y%m%d%H%M%S%Z"))
@@ -466,7 +466,7 @@
sys.stderr.write(usage_message)
sys.exit(1)
-if __name__ == "__main__":
+def main():
configFile = False
result = False
date = None
@@ -520,7 +520,7 @@
cutoff = time.strftime("%Y%m%d%H%M%S",
time.gmtime(time.time() - config.delay))
else:
- cutoff = cutoffFromDate(date)
+ cutoff = cutoffFromDate(date, config)
if len(remainder) > 0:
dump = IncrDump(config, date, cutoff, remainder[0], doStubs,
@@ -530,3 +530,8 @@
dump = IncrDumpLoop(config, date, cutoff, doStubs, doRevs,
doIndexUpdate, dryrun, verbose, forcerun)
dump.doAllWikisTilDone(3)
+
+
+if __name__ == "__main__":
+ main()
+
--
To view, visit https://gerrit.wikimedia.org/r/248867
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ide6dc299cce157d03f41310172935ae4269ba0b0
Gerrit-PatchSet: 1
Gerrit-Project: operations/dumps
Gerrit-Branch: ariel
Gerrit-Owner: ArielGlenn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
