ArielGlenn has submitted this change and it was merged.
Change subject: dumps: fix camelcases in WikiDumps.py (part 1)
......................................................................
dumps: fix camelcases in WikiDumps.py (part 1)
Change-Id: I512c498974d5c9074781bfe5890606b45ff3f0d0
---
M xmldumps-backup/dumps/WikiDump.py
M xmldumps-backup/dumps/fileutils.py
M xmldumps-backup/dumps/jobs.py
M xmldumps-backup/dumps/runnerutils.py
M xmldumps-backup/dumps/utils.py
M xmldumps-backup/monitor.py
M xmldumps-backup/worker.py
M xmldumps-backup/xmlabstracts.py
M xmldumps-backup/xmllogs.py
M xmldumps-backup/xmlstubs.py
10 files changed, 330 insertions(+), 331 deletions(-)
Approvals:
ArielGlenn: Verified; Looks good to me, approved
jenkins-bot: Verified
diff --git a/xmldumps-backup/dumps/WikiDump.py
b/xmldumps-backup/dumps/WikiDump.py
index 18ef7e9..6a678c2 100644
--- a/xmldumps-backup/dumps/WikiDump.py
+++ b/xmldumps-backup/dumps/WikiDump.py
@@ -13,15 +13,15 @@
class FileUtils(object):
- def fileAge(filename):
+ def file_age(filename):
return time.time() - os.stat(filename).st_mtime
- def atomicCreate(filename, mode='w'):
+ def atomic_create(filename, mode='w'):
"""Create a file, aborting if it already exists..."""
- fd = os.open(filename, os.O_EXCL + os.O_CREAT + os.O_WRONLY)
- return os.fdopen(fd, mode)
+ fhandle = os.open(filename, os.O_EXCL + os.O_CREAT + os.O_WRONLY)
+ return os.fdopen(fhandle, mode)
- def writeFile(dirname, filename, text, perms=0):
+ def write_file(dirname, filename, text, perms=0):
"""Write text to a file, as atomically as possible,
via a temporary file in a specified directory.
Arguments: dirname = where temp file is created,
@@ -35,16 +35,16 @@
raise IOError("The given directory '%s' is neither "
"a directory nor can it be created" % dirname)
- (fd, temp_filename) = tempfile.mkstemp("_txt", "wikidump_", dirname)
- os.write(fd, text)
- os.close(fd)
+ (fhandle, temp_filename) = tempfile.mkstemp("_txt", "wikidump_",
dirname)
+ os.write(fhandle, text)
+ os.close(fhandle)
if perms:
os.chmod(temp_filename, perms)
# This may fail across filesystems or on Windows.
# Of course nothing else will work on Windows. ;)
shutil.move(temp_filename, filename)
- def writeFileInPlace(filename, text, perms=0):
+ def write_file_in_place(filename, text, perms=0):
"""Write text to a file, after opening it for write with truncation.
This assumes that only one process or thread accesses the given file
at a time.
Arguments: filename = full path to actual file, text = contents
@@ -57,14 +57,14 @@
if perms:
os.chmod(filename, perms)
- def readFile(filename):
+ def read_file(filename):
"""Read text from a file in one fell swoop."""
filehdl = open(filename, "r")
text = filehdl.read()
filehdl.close()
return text
- def splitPath(path):
+ def split_path(path):
# For some reason, os.path.split only does one level.
parts = []
(path, filename) = os.path.split(path)
@@ -76,29 +76,29 @@
(path, filename) = os.path.split(path)
return parts
- def relativePath(path, base):
+ def relative_path(path, base):
"""Return a relative path to 'path' from the directory 'base'."""
- path = FileUtils.splitPath(path)
- base = FileUtils.splitPath(base)
+ path = FileUtils.split_path(path)
+ base = FileUtils.split_path(base)
while base and path[0] == base[0]:
path.pop(0)
base.pop(0)
- for prefix in base:
+ for prefix_unused in base:
path.insert(0, "..")
return os.path.join(*path)
- def prettySize(size):
+ def pretty_size(size):
"""Return a string with an attractively formatted file size."""
quanta = ("%d bytes", "%d KB", "%0.1f MB", "%0.1f GB", "%0.1f TB")
- return FileUtils._prettySize(size, quanta)
+ return FileUtils._pretty_size(size, quanta)
- def _prettySize(size, quanta):
+ def _pretty_size(size, quanta):
if size < 1024 or len(quanta) == 1:
return quanta[0] % size
else:
- return FileUtils._prettySize(size / 1024.0, quanta[1:])
+ return FileUtils._pretty_size(size / 1024.0, quanta[1:])
- def fileInfo(path):
+ def file_info(path):
"""Return a tuple of date/time and size of a file, or None, None"""
try:
timestamp = time.gmtime(os.stat(path).st_mtime)
@@ -108,16 +108,16 @@
except:
return(None, None)
- fileAge = staticmethod(fileAge)
- atomicCreate = staticmethod(atomicCreate)
- writeFile = staticmethod(writeFile)
- writeFileInPlace = staticmethod(writeFileInPlace)
- readFile = staticmethod(readFile)
- splitPath = staticmethod(splitPath)
- relativePath = staticmethod(relativePath)
- prettySize = staticmethod(prettySize)
- _prettySize = staticmethod(_prettySize)
- fileInfo = staticmethod(fileInfo)
+ file_age = staticmethod(file_age)
+ atomic_create = staticmethod(atomic_create)
+ write_file = staticmethod(write_file)
+ write_file_in_place = staticmethod(write_file_in_place)
+ read_file = staticmethod(read_file)
+ split_path = staticmethod(split_path)
+ relative_path = staticmethod(relative_path)
+ pretty_size = staticmethod(pretty_size)
+ _pretty_size = staticmethod(_pretty_size)
+ file_info = staticmethod(file_info)
class TimeUtils(object):
@@ -125,20 +125,20 @@
def today():
return time.strftime("%Y%m%d", time.gmtime())
- def prettyTime():
+ def pretty_time():
return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
- def prettyDate(key):
+ def pretty_date(key):
"Prettify a MediaWiki date key"
return "-".join((key[0:4], key[4:6], key[6:8]))
today = staticmethod(today)
- prettyTime = staticmethod(prettyTime)
- prettyDate = staticmethod(prettyDate)
+ pretty_time = staticmethod(pretty_time)
+ pretty_date = staticmethod(pretty_date)
class MiscUtils(object):
- def dbList(filename):
+ def db_list(filename):
"""Read database list from a file"""
if not filename:
return []
@@ -152,7 +152,7 @@
dbs.sort()
return dbs
- def shellEscape(param):
+ def shell_escape(param):
"""Escape a string parameter, or set of strings, for the shell."""
if isinstance(param, basestring):
return "'" + param.replace("'", "'\\''") + "'"
@@ -160,15 +160,15 @@
# A blank string might actually be needed; None means we can leave
it out
return ""
else:
- return tuple([MiscUtils.shellEscape(x) for x in param])
+ return tuple([MiscUtils.shell_escape(x) for x in param])
- dbList = staticmethod(dbList)
- shellEscape = staticmethod(shellEscape)
+ db_list = staticmethod(db_list)
+ shell_escape = staticmethod(shell_escape)
class Config(object):
def __init__(self, config_file=False):
- self.projectName = False
+ self.project_name = False
home = os.path.dirname(sys.argv[0])
if not config_file:
@@ -205,7 +205,7 @@
"smtpserver": "localhost",
"staleage": "3600",
# "database": {
- # these are now set in getDbUserAndPassword() if needed
+ # these are now set in get_db_user_and_password() if needed
"user": "",
"password": "",
# "tools": {
@@ -221,7 +221,7 @@
"grep": "/bin/grep",
"checkforbz2footer": "/usr/local/bin/checkforbz2footer",
"writeuptopageid": "/usr/local/bin/writeuptopageid",
- "recompressxml": "/usr/local/bin/recompressxml",
+ "recoompressxml": "/usr/local/bin/recompressxml",
# "cleanup": {
"keep": "3",
# "chunks": {
@@ -261,13 +261,13 @@
print "The mandatory setting 'dir' in the section 'wiki' was not
defined."
raise ConfigParser.NoOptionError('wiki', 'dir')
- self.dbUser = None
- self.dbPassword = None
- self.parseConfFileGlobally()
- self.parseConfFilePerProject()
- self.getDbUserAndPassword() # get from MW adminsettings file if not
set in conf file
+ self.db_user = None
+ self.db_password = None
+ self.parse_conffile_globally()
+ self.parse_conffile_per_project()
+ self.get_db_user_and_password() # get from MW adminsettings file if
not set in conf file
- def parsePHPAssignment(self, line):
+ def parse_php_assignment(self, line):
# not so much parse as grab a string to the right of the equals sign,
# we expect a line that has ... = "somestring" ;
# with single or double quotes, spaes or not. but nothing more
complicated.
@@ -278,23 +278,23 @@
else:
return ""
- def getDbUserAndPassword(self):
+ def get_db_user_and_password(self):
# check MW adminsettings file for these if we didn't have values for
# them in the conf file; failing that we fall back on defaults
specified
# here
- if self.dbUser: # already set via conf file, don't override
+ if self.db_user: # already set via conf file, don't override
return
default_dbuser = "root"
default_dbpassword = ""
if not self.conf.has_option("wiki", "adminsettings"):
- self.dbUser = default_dbuser
- self.dbPassword = default_dbpassword
+ self.db_user = default_dbuser
+ self.db_password = default_dbpassword
return
- adminfile = open(os.path.join(self.wikiDir, self.conf.get("wiki",
"adminsettings")), "r")
+ adminfile = open(os.path.join(self.wiki_dir, self.conf.get("wiki",
"adminsettings")), "r")
lines = adminfile.readlines()
adminfile.close()
@@ -303,53 +303,52 @@
# $wgDBadminuser = 'something';
# $wgDBuser = $wgDBadminuser = "something" ;
- for l in lines:
- if "$wgDBadminuser" in l:
- self.dbUser = self.parsePHPAssignment(l)
- elif "$wgDBuser" in l:
- default_dbuser = self.parsePHPAssignment(l)
- elif "$wgDBadminpassword" in l:
- self.dbPassword = self.parsePHPAssignment(l)
- elif "$wgDBpassword" in l:
- default_dbpassword = self.parsePHPAssignment(l)
+ for line in lines:
+ if "$wgDBadminuser" in line:
+ self.db_user = self.parse_php_assignment(line)
+ elif "$wgDBuser" in line:
+ default_dbuser = self.parse_php_assignment(line)
+ elif "$wgDBadminpassword" in line:
+ self.db_password = self.parse_php_assignment(line)
+ elif "$wgDBpassword" in line:
+ default_dbpassword = self.parse_php_assignment(line)
- if not self.dbUser:
- self.dbUser = default_dbuser
- if not self.dbPassword:
- self.dbPassword = default_dbpassword
+ if not self.db_user:
+ self.db_user = default_dbuser
+ if not self.db_password:
+ self.db_password = default_dbpassword
return
- def parseConfFileGlobally(self):
- self.dbList = MiscUtils.dbList(self.conf.get("wiki", "dblist"))
- self.skipDbList = MiscUtils.dbList(self.conf.get("wiki", "skipdblist"))
- self.privateList = MiscUtils.dbList(self.conf.get("wiki",
"privatelist"))
- self.flaggedRevsList = MiscUtils.dbList(self.conf.get("wiki",
"flaggedrevslist"))
- self.wikidataList = MiscUtils.dbList(self.conf.get("wiki",
"wikidatalist"))
- self.globalUsageList = MiscUtils.dbList(self.conf.get("wiki",
"globalusagelist"))
- self.wikidataClientList = MiscUtils.dbList(self.conf.get("wiki",
"wikidataclientlist"))
- self.forceNormal = self.conf.getint("wiki", "forcenormal")
+ def parse_conffile_globally(self):
+ self.db_list = MiscUtils.db_list(self.conf.get("wiki", "dblist"))
+ self.skip_db_list = MiscUtils.db_list(self.conf.get("wiki",
"skipdblist"))
+ self.private_list = MiscUtils.db_list(self.conf.get("wiki",
"privatelist"))
+ self.flagged_revs_list = MiscUtils.db_list(self.conf.get("wiki",
"flaggedrevslist"))
+ self.wikidata_list = MiscUtils.db_list(self.conf.get("wiki",
"wikidatalist"))
+ self.global_usage_list = MiscUtils.db_list(self.conf.get("wiki",
"globalusagelist"))
+ self.wikidata_client_list = MiscUtils.db_list(self.conf.get("wiki",
"wikidataclientlist"))
self.halt = self.conf.getint("wiki", "halt")
- self.dbList = list(set(self.dbList) - set(self.skipDbList))
+ self.db_list = list(set(self.db_list) - set(self.skip_db_list))
if not self.conf.has_section('output'):
self.conf.add_section('output')
- self.publicDir = self.conf.get("output", "public")
- self.privateDir = self.conf.get("output", "private")
- self.tempDir = self.conf.get("output", "temp")
- self.webRoot = self.conf.get("output", "webroot")
+ self.public_dir = self.conf.get("output", "public")
+ self.private_dir = self.conf.get("output", "private")
+ self.temp_dir = self.conf.get("output", "temp")
+ self.web_root = self.conf.get("output", "webroot")
self.index = self.conf.get("output", "index")
- self.templateDir = self.conf.get("output", "templatedir")
- self.perDumpIndex = self.conf.get("output", "perdumpindex")
+ self.template_dir = self.conf.get("output", "templatedir")
+ self.perdump_index = self.conf.get("output", "perdumpindex")
self.log_file = self.conf.get("output", "logfile")
self.fileperms = self.conf.get("output", "fileperms")
self.fileperms = int(self.fileperms, 0)
if not self.conf.has_section('reporting'):
self.conf.add_section('reporting')
- self.adminMail = self.conf.get("reporting", "adminmail")
- self.mailFrom = self.conf.get("reporting", "mailfrom")
- self.smtpServer = self.conf.get("reporting", "smtpserver")
- self.staleAge = self.conf.getint("reporting", "staleage")
+ self.admin_mail = self.conf.get("reporting", "adminmail")
+ self.mail_from = self.conf.get("reporting", "mailfrom")
+ self.smtp_server = self.conf.get("reporting", "smtpserver")
+ self.stale_age = self.conf.getint("reporting", "staleage")
if not self.conf.has_section('tools'):
self.conf.add_section('tools')
@@ -371,7 +370,7 @@
self.conf.add_section('cleanup')
self.keep = self.conf.getint("cleanup", "keep")
- def parseConfFilePerProject(self, project_name=False):
+ def parse_conffile_per_project(self, project_name=False):
# we need to read from the project section without falling back
# to the defaults, which has_option() normally does, ugh. so set
# up a local conf instance without the defaults
@@ -379,57 +378,57 @@
conf.read(self.files)
if project_name:
- self.projectName = project_name
+ self.project_name = project_name
if not self.conf.has_section('database'):
self.conf.add_section('database')
- dbuser = self.getOptionForProjectOrDefault(conf, "database", "user", 0)
+ dbuser = self.get_opt_for_proj_or_default(conf, "database", "user", 0)
if dbuser:
- self.dbUser = dbuser
- dbpassword = self.getOptionForProjectOrDefault(conf, "database",
"password", 0)
+ self.db_user = dbuser
+ dbpassword = self.get_opt_for_proj_or_default(conf, "database",
"password", 0)
if dbpassword:
- self.dbPassword = dbpassword
+ self.db_password = dbpassword
if not self.conf.has_section('chunks'):
self.conf.add_section('chunks')
- self.chunksEnabled = self.getOptionForProjectOrDefault(
+ self.chunks_enabled = self.get_opt_for_proj_or_default(
conf, "chunks", "chunksEnabled", 1)
- self.pagesPerChunkHistory = self.getOptionForProjectOrDefault(
+ self.pages_per_chunk_history = self.get_opt_for_proj_or_default(
conf, "chunks", "pagesPerChunkHistory", 0)
- self.revsPerChunkHistory = self.getOptionForProjectOrDefault(
+ self.revs_per_chunk_history = self.get_opt_for_proj_or_default(
conf, "chunks", "revsPerChunkHistory", 0)
- self.chunksForAbstract = self.getOptionForProjectOrDefault(
+ self.chunks_for_abstract = self.get_opt_for_proj_or_default(
conf, "chunks", "chunksForAbstract", 0)
- self.pagesPerChunkAbstract = self.getOptionForProjectOrDefault(
+ self.pages_per_chunk_abstract = self.get_opt_for_proj_or_default(
conf, "chunks", "pagesPerChunkAbstract", 0)
- self.recombineHistory = self.getOptionForProjectOrDefault(
+ self.recombine_history = self.get_opt_for_proj_or_default(
conf, "chunks", "recombineHistory", 1)
- self.checkpointTime = self.getOptionForProjectOrDefault(
+ self.checkpoint_time = self.get_opt_for_proj_or_default(
conf, "chunks", "checkpointTime", 1)
if not self.conf.has_section('otherformats'):
self.conf.add_section('otherformats')
- self.multistreamEnabled = self.getOptionForProjectOrDefault(
+ self.multistream_enabled = self.get_opt_for_proj_or_default(
conf, 'otherformats', 'multistream', 1)
if not self.conf.has_section('wiki'):
self.conf.add_section('wiki')
- self.wikiDir = self.getOptionForProjectOrDefault(conf, "wiki", "dir",
0)
+ self.wiki_dir = self.get_opt_for_proj_or_default(conf, "wiki", "dir",
0)
- def getOptionForProjectOrDefault(self, conf, section_name, item_name,
is_int):
- if conf.has_section(self.projectName):
- if conf.has_option(self.projectName, item_name):
+ def get_opt_for_proj_or_default(self, conf, section_name, item_name,
is_int):
+ if conf.has_section(self.project_name):
+ if conf.has_option(self.project_name, item_name):
if is_int:
- return conf.getint(self.projectName, item_name)
+ return conf.getint(self.project_name, item_name)
else:
- return conf.get(self.projectName, item_name)
+ return conf.get(self.project_name, item_name)
if is_int:
return self.conf.getint(section_name, item_name)
else:
return self.conf.get(section_name, item_name)
- def dbListByAge(self, use_status_time=False):
+ def db_list_by_age(self, use_status_time=False):
"""
Sort wikis in reverse order of last successful dump :
@@ -452,15 +451,15 @@
"""
available = []
today = int(TimeUtils.today())
- for db in self.dbList:
- wiki = Wiki(self, db)
+ for dbname in self.db_list:
+ wiki = Wiki(self, dbname)
age = sys.maxint
date = sys.maxint
- last = wiki.latestDump()
+ last = wiki.latest_dump()
status = ''
if last:
- dump_status = os.path.join(wiki.publicDir(), last,
"status.html")
+ dump_status = os.path.join(wiki.public_dir(), last,
"status.html")
try:
if use_status_time:
# only use the status file time, not the dir date
@@ -469,29 +468,29 @@
date = today - int(last)
# tack on the file mtime so that if we have multiple wikis
# dumped on the same day, they get ordered properly
- age = FileUtils.fileAge(dump_status)
- status = FileUtils.readFile(dump_status)
+ age = FileUtils.file_age(dump_status)
+ status = FileUtils.read_file(dump_status)
except:
print "dump dir missing status file %s?" % dump_status
dump_failed = (status == '') or ('dump aborted' in status)
- available.append((dump_failed, date, age, db))
+ available.append((dump_failed, date, age, dbname))
available.sort()
- return [db for (_failed, _date, _age, db) in available]
+ return [dbname for (_failed, _date, _age, dbname) in available]
- def readTemplate(self, name):
- template = os.path.join(self.templateDir, name)
- return FileUtils.readFile(template)
+ def read_template(self, name):
+ template = os.path.join(self.template_dir, name)
+ return FileUtils.read_file(template)
def mail(self, subject, body):
"""Send out a quickie email."""
message = MIMEText.MIMEText(body)
message["Subject"] = subject
- message["From"] = self.mailFrom
- message["To"] = self.adminMail
+ message["From"] = self.mail_from
+ message["To"] = self.admin_mail
try:
- server = smtplib.SMTP(self.smtpServer)
- server.sendmail(self.mailFrom, self.adminMail, message.as_string())
+ server = smtplib.SMTP(self.smtp_server)
+ server.sendmail(self.mail_from, self.admin_mail,
message.as_string())
server.close()
except:
print "MAIL SEND FAILED! GODDAMIT! Was sending this mail:"
@@ -501,57 +500,57 @@
class Wiki(object):
def __init__(self, config, db_name):
self.config = config
- self.dbName = db_name
+ self.db_name = db_name
self.date = None
self.watchdog = None
- def isPrivate(self):
- return self.dbName in self.config.privateList
+ def is_private(self):
+ return self.db_name in self.config.private_list
- def hasFlaggedRevs(self):
- return self.dbName in self.config.flaggedRevsList
+ def has_flagged_revs(self):
+ return self.db_name in self.config.flagged_revs_list
- def hasWikidata(self):
- return self.dbName in self.config.wikidataList
+ def has_wikidata(self):
+ return self.db_name in self.config.wikidata_list
def has_global_usage(self):
- return self.dbName in self.config.globalUsageList
+ return self.db_name in self.config.global_usage_list
- def isWikidataClient(self):
- return self.dbName in self.config.wikidataClientList
+ def is_wikidata_client(self):
+ return self.db_name in self.config.wikidata_client_list
- def isLocked(self):
- return os.path.exists(self.lockFile())
+ def is_locked(self):
+ return os.path.exists(self.lock_file())
- def isStale(self):
- if not self.isLocked():
+ def is_stale(self):
+ if not self.is_locked():
return False
try:
- age = self.lockAge()
- return age > self.config.staleAge
+ age = self.lock_age()
+ return age > self.config.stale_age
except:
# Lock file vanished while we were looking
return False
# Paths and directories...
- def publicDir(self):
- if self.isPrivate():
- return self.privateDir()
+ def public_dir(self):
+ if self.is_private():
+ return self.private_dir()
else:
- return os.path.join(self.config.publicDir, self.dbName)
+ return os.path.join(self.config.public_dir, self.db_name)
- def privateDir(self):
- return os.path.join(self.config.privateDir, self.dbName)
+ def private_dir(self):
+ return os.path.join(self.config.private_dir, self.db_name)
- def webDir(self):
- web_root = self.config.webRoot
+ def web_dir(self):
+ web_root = self.config.web_root
if web_root[-1] == '/':
web_root = web_root[:-1]
- return "/".join((web_root, self.dbName))
+ return "/".join((web_root, self.db_name))
- def webDirRelative(self):
- web_root_rel = self.webDir()
+ def web_dir_relative(self):
+ web_root_rel = self.web_dir()
i = web_root_rel.find("://")
if i >= 0:
web_root_rel = web_root_rel[i+3:]
@@ -563,121 +562,121 @@
# Actions!
def lock(self):
- if not os.path.isdir(self.privateDir()):
+ if not os.path.isdir(self.private_dir()):
try:
- os.makedirs(self.privateDir())
+ os.makedirs(self.private_dir())
except:
# Maybe it was just created (race condition)?
- if not os.path.isdir(self.privateDir()):
+ if not os.path.isdir(self.private_dir()):
raise
- lockf = FileUtils.atomicCreate(self.lockFile(), "w")
+ lockf = FileUtils.atomic_create(self.lock_file(), "w")
lockf.write("%s %d" % (socket.getfqdn(), os.getpid()))
lockf.close()
- self.watchdog = LockWatchdog(self.lockFile())
+ self.watchdog = LockWatchdog(self.lock_file())
self.watchdog.start()
return True
def unlock(self):
if self.watchdog:
- self.watchdog.stopWatching()
+ self.watchdog.stop_watching()
self.watchdog = None
- os.remove(self.lockFile())
+ os.remove(self.lock_file())
- def setDate(self, date):
+ def set_date(self, date):
self.date = date
- def cleanupStaleLock(self):
- date = self.latestDump()
+ def cleanup_stale_lock(self):
+ date = self.latest_dump()
if date:
- self.setDate(date)
- self.writeStatus(self.reportStatusLine(
+ self.set_date(date)
+ self.write_status(self.report_statusline(
"<span class=\"failed\">dump aborted</span>"))
self.unlock()
- def writePerDumpIndex(self, html):
- index = os.path.join(self.publicDir(), self.date,
self.config.perDumpIndex)
- FileUtils.writeFileInPlace(index, html, self.config.fileperms)
+ def write_perdump_index(self, html):
+ index = os.path.join(self.public_dir(), self.date,
self.config.perdump_index)
+ FileUtils.write_file_in_place(index, html, self.config.fileperms)
- def existsPerDumpIndex(self):
- index = os.path.join(self.publicDir(), self.date,
self.config.perDumpIndex)
+ def exists_perdump_index(self):
+ index = os.path.join(self.public_dir(), self.date,
self.config.perdump_index)
return os.path.exists(index)
- def writeStatus(self, message):
- index = os.path.join(self.publicDir(), self.date, "status.html")
- FileUtils.writeFileInPlace(index, message, self.config.fileperms)
+ def write_status(self, message):
+ index = os.path.join(self.public_dir(), self.date, "status.html")
+ FileUtils.write_file_in_place(index, message, self.config.fileperms)
- def statusLine(self):
- date = self.latestDump()
+ def status_line(self):
+ date = self.latest_dump()
if date:
- status = os.path.join(self.publicDir(), date, "status.html")
+ status = os.path.join(self.public_dir(), date, "status.html")
try:
- return FileUtils.readFile(status)
+ return FileUtils.read_file(status)
except:
- return self.reportStatusLine("missing status record")
+ return self.report_statusline("missing status record")
else:
- return self.reportStatusLine("has not yet been dumped")
+ return self.report_statusline("has not yet been dumped")
- def reportStatusLine(self, status, error=False):
+ def report_statusline(self, status, error=False):
if error:
# No state information, hide the timestamp
- stamp = "<span style=\"visible: none\">" + TimeUtils.prettyTime()
+ "</span>"
+ stamp = "<span style=\"visible: none\">" + TimeUtils.pretty_time()
+ "</span>"
else:
- stamp = TimeUtils.prettyTime()
- if self.isPrivate():
- link = "%s (private data)" % self.dbName
+ stamp = TimeUtils.pretty_time()
+ if self.is_private():
+ link = "%s (private data)" % self.db_name
else:
if self.date:
- link = "<a href=\"%s/%s\">%s</a>" % (self.dbName, self.date,
self.dbName)
+ link = "<a href=\"%s/%s\">%s</a>" % (self.db_name, self.date,
self.db_name)
else:
- link = "%s (new)" % self.dbName
+ link = "%s (new)" % self.db_name
return "<li>%s %s: %s</li>\n" % (stamp, link, status)
- def latestDump(self, index=-1, all=False):
+ def latest_dump(self, index=-1, return_all=False):
"""Find the last (or slightly less than last) dump for a db."""
- dirs = self.dumpDirs()
+ dirs = self.dump_dirs()
if dirs:
- if all:
+ if return_all:
return dirs
else:
return dirs[index]
else:
return None
- def dateTouchedLatestDump(self):
+ def date_touched_latest_dump(self):
mtime = 0
- last = self.latestDump()
+ last = self.latest_dump()
if last:
- dump_status = os.path.join(self.publicDir(), last, "status.html")
+ dump_status = os.path.join(self.public_dir(), last, "status.html")
try:
mtime = os.stat(dump_status).st_mtime
except:
pass
return time.strftime("%Y%m%d", time.gmtime(mtime))
- def dumpDirs(self, private=False):
+ def dump_dirs(self, private=False):
"""List all dump directories for the given database."""
if private:
- base = self.privateDir()
+ base = self.private_dir()
else:
- base = self.publicDir()
+ base = self.public_dir()
digits = re.compile(r"^\d{4}\d{2}\d{2}$")
dates = []
try:
- for dir in os.listdir(base):
- if digits.match(dir):
- dates.append(dir)
+ for dirname in os.listdir(base):
+ if digits.match(dirname):
+ dates.append(dirname)
except OSError:
return []
dates.sort()
return dates
# private....
- def lockFile(self):
- return os.path.join(self.privateDir(), "lock")
+ def lock_file(self):
+ return os.path.join(self.private_dir(), "lock")
- def lockAge(self):
- return FileUtils.fileAge(self.lockFile())
+ def lock_age(self):
+ return FileUtils.file_age(self.lock_file())
class LockWatchdog(threading.Thread):
@@ -692,7 +691,7 @@
self.trigger = threading.Event()
self.finished = threading.Event()
- def stopWatching(self):
+ def stop_watching(self):
"""Run me outside..."""
# Ask the thread to stop...
self.trigger.set()
@@ -706,13 +705,13 @@
def run(self):
LockWatchdog.threads.append(self)
while not self.trigger.isSet():
- self.touchLock()
+ self.touch_lock()
self.trigger.wait(10)
self.trigger.clear()
self.finished.set()
LockWatchdog.threads.remove(self)
- def touchLock(self):
+ def touch_lock(self):
"""Run me inside..."""
os.utime(self.lockfile, None)
@@ -720,7 +719,7 @@
def cleanup():
"""Call cleanup handlers for any background threads..."""
for watchdog in LockWatchdog.threads:
- watchdog.stopWatching()
+ watchdog.stop_watching()
if __name__ == "__main__":
config_unused = Config()
diff --git a/xmldumps-backup/dumps/fileutils.py
b/xmldumps-backup/dumps/fileutils.py
index ab175f7..ef3ddbc 100644
--- a/xmldumps-backup/dumps/fileutils.py
+++ b/xmldumps-backup/dumps/fileutils.py
@@ -178,7 +178,7 @@
date = self.wiki.date
# fixme do the right thing in case no filetype or no ext
parts = []
- parts.append(self.wiki.dbName + "-" + date + "-" + dump_name + "%s" %
chunk)
+ parts.append(self.wiki.db_name + "-" + date + "-" + dump_name + "%s" %
chunk)
if checkpoint:
filetype = filetype + "-" + checkpoint
if filetype:
@@ -280,7 +280,7 @@
if not exists(self._wiki.config.head):
raise BackupError("head command %s not found" %
self._wiki.config.head)
head = self._wiki.config.head
- head_esc = MiscUtils.shellEscape(head)
+ head_esc = MiscUtils.shell_escape(head)
pipeline.append([head, "-500"])
# without shell
proc = CommandPipeline(pipeline, quiet=True)
@@ -411,7 +411,7 @@
If a different date is specified, use that instead"""
if not date_string:
date_string = self._wiki.date
- return os.path.join(self._wiki.privateDir(), date_string,
dump_file.filename)
+ return os.path.join(self._wiki.private_dir(), date_string,
dump_file.filename)
def filename_public_path(self, dump_file, date_string=None):
"""Given a DumpFilename object produce the full path to the filename
in the date subdir
@@ -420,21 +420,21 @@
If a different date is specified, use that instead"""
if not date_string:
date_string = self._wiki.date
- return os.path.join(self._wiki.publicDir(), date_string,
dump_file.filename)
+ return os.path.join(self._wiki.public_dir(), date_string,
dump_file.filename)
def latest_dir(self):
"""Return 'latest' directory for the current project being dumped, e.g.
if the current project is enwiki, this would return something like
/mnt/data/xmldatadumps/public/enwiki/latest (if the directory
/mnt/data/xmldatadumps/public
is the path to the directory for public dumps)."""
- return os.path.join(self._wiki.publicDir(), "latest")
+ return os.path.join(self._wiki.public_dir(), "latest")
def web_path(self, dump_file, date_string=None):
"""Given a DumpFilename object produce the full url to the filename
for the date of
the dump for the selected database."""
if not date_string:
date_string = self._wiki.date
- return os.path.join(self._wiki.webDir(), date_string,
dump_file.filename)
+ return os.path.join(self._wiki.web_dir(), date_string,
dump_file.filename)
def web_path_relative(self, dump_file, date_string=None):
"""Given a DumpFilename object produce the url relative
@@ -442,12 +442,12 @@
the dump for the selected database."""
if not date_string:
date_string = self._wiki.date
- return os.path.join(self._wiki.webDirRelative(), date_string,
dump_file.filename)
+ return os.path.join(self._wiki.web_dir_relative(), date_string,
dump_file.filename)
def dir_cache_outdated(self, date):
if not date:
date = self._wiki.date
- directory = os.path.join(self._wiki.publicDir(), date)
+ directory = os.path.join(self._wiki.public_dir(), date)
if exists(directory):
dir_time_stamp = os.stat(directory).st_mtime
if date not in self._dir_cache or dir_time_stamp >
self._dir_cache_time[date]:
@@ -462,7 +462,7 @@
if not date:
date = self._wiki.date
if self.dir_cache_outdated(date):
- directory = os.path.join(self._wiki.publicDir(), date)
+ directory = os.path.join(self._wiki.public_dir(), date)
if exists(directory):
dir_time_stamp = os.stat(directory).st_mtime
files = os.listdir(directory)
diff --git a/xmldumps-backup/dumps/jobs.py b/xmldumps-backup/dumps/jobs.py
index b7d656a..3f5b244 100644
--- a/xmldumps-backup/dumps/jobs.py
+++ b/xmldumps-backup/dumps/jobs.py
@@ -82,7 +82,7 @@
def set_status(self, status, set_updated=True):
self.runinfo.set_status(status)
if set_updated:
- self.runinfo.set_updated(TimeUtils.prettyTime())
+ self.runinfo.set_updated(TimeUtils.pretty_time())
def set_updated(self, updated):
self.runinfo.set_updated(updated)
@@ -227,23 +227,23 @@
# we assume the result is always going to be run in a subshell.
# much quicker than this script trying to read output
# and pass it to a subprocess
- output_filename_esc = MiscUtils.shellEscape(output_filename)
- head_esc = MiscUtils.shellEscape(head)
- tail_esc = MiscUtils.shellEscape(tail)
- grep_esc = MiscUtils.shellEscape(grep)
+ output_filename_esc = MiscUtils.shell_escape(output_filename)
+ head_esc = MiscUtils.shell_escape(head)
+ tail_esc = MiscUtils.shell_escape(tail)
+ grep_esc = MiscUtils.shell_escape(grep)
uncompression_command_esc = uncompression_command[:]
for command in uncompression_command_esc:
- command = MiscUtils.shellEscape(command)
+ command = MiscUtils.shell_escape(command)
for command in compression_command:
- command = MiscUtils.shellEscape(command)
+ command = MiscUtils.shell_escape(command)
if not files:
raise BackupError("No files for the recombine step found in %s." %
self.name())
for file_obj in files:
# uh oh FIXME
- # f = MiscUtils.shellEscape(file_obj.filename)
+ # f = MiscUtils.shell_escape(file_obj.filename)
fpath = runner.dump_dir.filename_public_path(file_obj)
chunknum = chunknum + 1
pipeline = []
@@ -1092,7 +1092,7 @@
self.wiki.config.writeuptopageid)
inputfile_path = runner.dump_dir.filename_public_path(input_file)
- output_file_path = os.path.join(self.wiki.config.tempDir,
output_file.filename)
+ output_file_path = os.path.join(self.wiki.config.temp_dir,
output_file.filename)
if input_file.file_ext == "gz":
command1 = "%s -dc %s" % (self.wiki.config.gzip, inputfile_path)
command2 = "%s > %s" % (self.wiki.config.gzip, output_file_path)
@@ -1163,7 +1163,7 @@
self.checkpoint_file.first_page_id,
str(int(self.checkpoint_file.last_page_id)
+ 1), runner)
stub_option = ("--stub=gzip:%s" % os.path.join(
- self.wiki.config.tempDir, stub_output_file.filename))
+ self.wiki.config.temp_dir, stub_output_file.filename))
elif self.page_id_range:
# two cases. redoing a specific chunk, OR no chunks,
# redoing the whole output file. ouch, hope it isn't huge.
@@ -1184,7 +1184,7 @@
self.write_partial_stub(stub_input_file, stub_output_file,
first_page_id, last_page_id, runner)
- stub_option = "--stub=gzip:%s" %
os.path.join(self.wiki.config.tempDir,
+ stub_option = "--stub=gzip:%s" %
os.path.join(self.wiki.config.temp_dir,
stub_output_file.filename)
else:
stub_option = "--stub=gzip:%s" %
runner.dump_dir.filename_public_path(outfile)
@@ -1225,7 +1225,7 @@
raise BackupError("php command %s not found" %
self.wiki.config.php)
if self._checkpoints_enabled:
- checkpoint_time = "--maxtime=%s" %
(self.wiki.config.checkpointTime)
+ checkpoint_time = "--maxtime=%s" %
(self.wiki.config.checkpoint_time)
checkpoint_file = "--checkpointfile=%s" % output_file.new_filename(
output_file.dumpname, output_file.file_type,
output_file.file_ext,
output_file.date, output_file.chunk, "p%sp%s", None)
@@ -1343,7 +1343,7 @@
start_page_id = 1
end_page_id = None
- dumps = self.wiki.dumpDirs()
+ dumps = self.wiki.dump_dirs()
dumps.sort()
dumps.reverse()
for date in dumps:
diff --git a/xmldumps-backup/dumps/runnerutils.py
b/xmldumps-backup/dumps/runnerutils.py
index d59b217..71433ea 100644
--- a/xmldumps-backup/dumps/runnerutils.py
+++ b/xmldumps-backup/dumps/runnerutils.py
@@ -81,9 +81,9 @@
for htype in self.hashtypes:
tmp_filename = self._get_checksum_filename_tmp(htype)
real_filename = self._get_checksum_filename(htype)
- text = FileUtils.readFile(tmp_filename)
- FileUtils.writeFile(self.wiki.config.tempDir, real_filename,
text,
- self.wiki.config.fileperms)
+ text = FileUtils.read_file(tmp_filename)
+ FileUtils.write_file(self.wiki.config.temp_dir, real_filename,
text,
+ self.wiki.config.fileperms)
def get_checksum_filename_basename(self, htype):
if htype == "md5":
@@ -106,7 +106,7 @@
return self.dump_dir.filename_public_path(file_obj)
def _getmd5file_dir_name(self):
- return os.path.join(self.wiki.publicDir(), self.wiki.date)
+ return os.path.join(self.wiki.public_dir(), self.wiki.date)
# everything that has to do with reporting the status of a piece
@@ -115,7 +115,7 @@
def __init__(self, wiki, dump_dir, items, checksums, enabled, email=True,
notice_file=None, error_callback=None, verbose=False):
self.wiki = wiki
- self.db_name = wiki.dbName
+ self.db_name = wiki.db_name
self.dump_dir = dump_dir
self.items = items
self.checksums = checksums
@@ -132,13 +132,13 @@
def report_failure(self):
if self._enabled and self.email:
- if self.wiki.config.adminMail and
self.wiki.config.adminMail.lower() != 'nomail':
+ if self.wiki.config.admin_mail and
self.wiki.config.admin_mail.lower() != 'nomail':
subject = "Dump failure for " + self.db_name
- message = self.wiki.config.readTemplate("errormail.txt") % {
+ message = self.wiki.config.read_template("errormail.txt") % {
"db": self.db_name,
"date": self.wiki.date,
- "time": TimeUtils.prettyTime(),
- "url": "/".join((self.wiki.config.webRoot, self.db_name,
self.wiki.date, ''))}
+ "time": TimeUtils.pretty_time(),
+ "url": "/".join((self.wiki.config.web_root, self.db_name,
self.wiki.date, ''))}
self.wiki.config.mail(subject, message)
# this is a per-dump-item report (well, per file generated by the item)
@@ -150,7 +150,7 @@
else:
item_status = "missing"
size = 0
- size = FileUtils.prettySize(size)
+ size = FileUtils.pretty_size(size)
if item_status == "in-progress":
return "<li class='file'>%s %s (written) </li>" %
(file_obj.filename, size)
elif item_status == "done":
@@ -168,9 +168,9 @@
and links to completed files, as well as a summary status in a
separate file."""
try:
# Comprehensive report goes here
- self.wiki.writePerDumpIndex(self._report_dbstatus_detailed(done))
+ self.wiki.write_perdump_index(self._report_dbstatus_detailed(done))
# Short line for report extraction goes here
- self.wiki.writeStatus(self._report_database_status_summary(done))
+ self.wiki.write_status(self._report_database_status_summary(done))
except:
if self.verbose:
exc_type, exc_value, exc_traceback = sys.exc_info()
@@ -185,7 +185,7 @@
def _report_database_status_summary(self, done=False):
"""Put together a brief status summary and link for the current
database."""
status = self._report_status_summary_line(done)
- html = self.wiki.reportStatusLine(status)
+ html = self.wiki.report_statusline(status)
active_items = [x for x in self.items if x.status() == "in-progress"]
if active_items:
@@ -208,7 +208,7 @@
checksums = [self.get_checksum_html(htype)
for htype in self.checksums.hashtypes]
checksums_html = ", ".join(checksums)
- return self.wiki.config.readTemplate("report.html") % {
+ return self.wiki.config.read_template("report.html") % {
"db": self.db_name,
"date": self.wiki.date,
"notice": self.notice_file.notice,
@@ -226,7 +226,7 @@
# runs going at once (think en pedia, one finishing up the history,
another
# starting at the beginning to get the new abstracts and stubs).
try:
- dumps_in_order = self.wiki.latestDump(all=True)
+ dumps_in_order = self.wiki.latest_dump(all=True)
me_index = dumps_in_order.index(self.wiki.date)
# don't wrap around to the newest dump in the list!
if me_index > 0:
@@ -243,7 +243,7 @@
sys.stderr.write(repr(
traceback.format_exception(exc_type, exc_value,
exc_traceback)))
return "No prior dumps of this database stored."
- pretty_date = TimeUtils.prettyDate(raw_date)
+ pretty_date = TimeUtils.pretty_date(raw_date)
if done:
prefix = ""
message = "Last dumped on"
@@ -312,19 +312,19 @@
# addnotice, stuff notice in a file for other jobs etc
elif self.notice != "":
# notice_dir = self._get_notice_dir()
- FileUtils.writeFile(self.wiki.config.tempDir, notice_file,
self.notice,
- self.wiki.config.fileperms)
+ FileUtils.write_file(self.wiki.config.temp_dir, notice_file,
self.notice,
+ self.wiki.config.fileperms)
# default case. if there is a file get the contents, otherwise
# we have empty contents, all good
else:
if exists(notice_file):
- self.notice = FileUtils.readFile(notice_file)
+ self.notice = FileUtils.read_file(notice_file)
def refresh_notice(self):
# if the notice file has changed or gone away, we comply.
notice_file = self._get_notice_filename()
if exists(notice_file):
- self.notice = FileUtils.readFile(notice_file)
+ self.notice = FileUtils.read_file(notice_file)
else:
self.notice = ""
@@ -332,10 +332,10 @@
# functions internal to class
#
def _get_notice_filename(self):
- return os.path.join(self.wiki.publicDir(), self.wiki.date,
"notice.txt")
+ return os.path.join(self.wiki.public_dir(), self.wiki.date,
"notice.txt")
# def _get_notice_dir(self):
-# return os.path.join(self.wiki.publicDir(), self.wiki.date)
+# return os.path.join(self.wiki.public_dir(), self.wiki.date)
class SymLinks(object):
@@ -382,7 +382,7 @@
else:
self.logfn("What the hell dude, %s is not a symlink" %
link)
raise BackupError("What the hell dude, %s is not a
symlink" % link)
- relative = FileUtils.relativePath(realfile, os.path.dirname(link))
+ relative = FileUtils.relative_path(realfile, os.path.dirname(link))
# if we removed the link cause it's obsolete, make the new one
if exists(realfile) and not exists(link):
self.debugfn("Adding symlink %s -> %s" % (link, relative))
@@ -452,7 +452,7 @@
self.make_dir(self.dump_dir.latest_dir())
filename_and_path = self.dump_dir.web_path(file_obj)
web_path = os.path.dirname(filename_and_path)
- rss_text = self.wiki.config.readTemplate("feed.xml") % {
+ rss_text = self.wiki.config.read_template("feed.xml") % {
"chantitle": file_obj.basename,
"chanlink": web_path,
"chandesc": "Wikimedia dump updates for %s" % self.db_name,
@@ -466,8 +466,8 @@
self.db_name + "-latest-" +
file_obj.basename +
"-rss.xml")
self.debugfn("adding rss feed file %s " % rss_path)
- FileUtils.writeFile(self.wiki.config.tempDir, rss_path,
- rss_text, self.wiki.config.fileperms)
+ FileUtils.write_file(self.wiki.config.temp_dir, rss_path,
+ rss_text, self.wiki.config.fileperms)
def cleanup_feeds(self):
# call this after sym links in this dir have been cleaned up.
diff --git a/xmldumps-backup/dumps/utils.py b/xmldumps-backup/dumps/utils.py
index 579e374..1ad572f 100644
--- a/xmldumps-backup/dumps/utils.py
+++ b/xmldumps-backup/dumps/utils.py
@@ -21,14 +21,14 @@
return " ".join(MultiVersion.mw_script_as_array(config,
maintenance_script))
def mw_script_as_array(config, maintenance_script):
- mw_script_location = os.path.join(config.wikiDir, "multiversion",
"MWScript.php")
+ mw_script_location = os.path.join(config.wiki_dir, "multiversion",
"MWScript.php")
if exists(mw_script_location):
return [mw_script_location, maintenance_script]
else:
- return ["%s/maintenance/%s" % (config.wikiDir, maintenance_script)]
+ return ["%s/maintenance/%s" % (config.wiki_dir,
maintenance_script)]
def mw_version(config, db_name):
- get_version_location = os.path.join(config.wikiDir, "multiversion",
"getMWVersion")
+ get_version_location = os.path.join(config.wiki_dir, "multiversion",
"getMWVersion")
if exists(get_version_location):
# run the command for the wiki and get the version
command = get_version_location + " " + db_name
@@ -57,10 +57,10 @@
if not exists(self.wiki.config.php):
raise BackupError("php command %s not found" %
self.wiki.config.php)
command_list = MultiVersion.mw_script_as_array(self.wiki.config,
"getSlaveServer.php")
- php_command = MiscUtils.shellEscape(self.wiki.config.php)
- db_name = MiscUtils.shellEscape(self.db_name)
+ php_command = MiscUtils.shell_escape(self.wiki.config.php)
+ db_name = MiscUtils.shell_escape(self.db_name)
for i in range(0, len(command_list)):
- command_list[i] = MiscUtils.shellEscape(command_list[i])
+ command_list[i] = MiscUtils.shell_escape(command_list[i])
command = " ".join(command_list)
command = "%s -q %s --wiki=%s --group=dump --globals" % (php_command,
command, db_name)
results = RunSimpleCommand.run_and_return(command,
self.error_callback).strip()
@@ -96,7 +96,7 @@
params = ["-h", "%s" % host] # Host
if self.db_port:
params += ["--port", "%s" % self.db_port] # Port
- params += ["-u", "%s" % self.wiki.config.dbUser] # Username
+ params += ["-u", "%s" % self.wiki.config.db_user] # Username
params += ["%s" % self.password_option()] # Password
return params
@@ -139,10 +139,10 @@
def password_option(self):
"""If you pass '-pfoo' mysql uses the password 'foo',
but if you pass '-p' it prompts. Sigh."""
- if self.wiki.config.dbPassword == "":
+ if self.wiki.config.db_password == "":
return None
else:
- return "-p" + self.wiki.config.dbPassword
+ return "-p" + self.wiki.config.db_password
class RunSimpleCommand(object):
@@ -251,8 +251,8 @@
return 0
# ok, there was no info there to be had, try the index file. yuck.
- index_filename = os.path.join(runner.wiki.publicDir(),
- date, runner.wiki.config.perDumpIndex)
+ index_filename = os.path.join(runner.wiki.public_dir(),
+ date, runner.wiki.config.perdump_index)
status = self._get_status_from_html(index_filename, job_desc)
if status == "done":
return 1
@@ -288,15 +288,15 @@
# sometimes need to get this info for an older run to check status of
a file for
# possible prefetch
if date:
- return os.path.join(self.wiki.publicDir(), date, "dumpruninfo.txt")
+ return os.path.join(self.wiki.public_dir(), date,
"dumpruninfo.txt")
else:
- return os.path.join(self.wiki.publicDir(), self.wiki.date,
"dumpruninfo.txt")
+ return os.path.join(self.wiki.public_dir(), self.wiki.date,
"dumpruninfo.txt")
def _get_dump_runinfo_dirname(self, date=None):
if date:
- return os.path.join(self.wiki.publicDir(), date)
+ return os.path.join(self.wiki.public_dir(), date)
else:
- return os.path.join(self.wiki.publicDir(), self.wiki.date)
+ return os.path.join(self.wiki.public_dir(), self.wiki.date)
# format: name:%; updated:%; status:%
def _get_old_runinfo_from_line(self, line):
@@ -318,8 +318,8 @@
def _write_dump_runinfo_file(self, text):
dump_runinfo_filename = self._get_dump_runinfo_filename()
-# FileUtils.writeFile(directory, dumpRunInfoFilename, text,
self.wiki.config.fileperms)
- FileUtils.writeFileInPlace(dump_runinfo_filename, text,
self.wiki.config.fileperms)
+# FileUtils.write_file(directory, dumpRunInfoFilename, text,
self.wiki.config.fileperms)
+ FileUtils.write_file_in_place(dump_runinfo_filename, text,
self.wiki.config.fileperms)
# format: name:%; updated:%; status:%
def _get_status_from_runinfo_line(self, line, job_name):
@@ -417,7 +417,7 @@
self._to_run = to_run
-# so if the pages/revsPerChunkAbstract/History are just one number it means
+# so if the pages/revs_per_chunk_abstr/_history are just one number it means
# use that number for all the chunks, figure out yourself how many.
# otherwise we get passed alist that says "here's now many for each chunk and
it's this many chunks.
# extra pages/revs go in the last chunk, stuck on the end. too bad. :-P
@@ -426,26 +426,26 @@
self._db_name = db_name
self.wiki = wiki
- self._chunks_enabled = self.wiki.config.chunksEnabled
+ self._chunks_enabled = self.wiki.config.chunks_enabled
if self._chunks_enabled:
self.stats = PageAndEditStats(self.wiki, self._db_name,
error_callback)
if not self.stats.total_edits or not self.stats.total_pages:
raise BackupError("Failed to get DB stats, exiting")
- if self.wiki.config.chunksForAbstract:
+ if self.wiki.config.chunks_for_abstract:
# we add 200 padding to cover new pages that may be added
pages_per_chunk = 200 + self.stats.total_pages/int(
- self.wiki.config.chunksForAbstract)
+ self.wiki.config.chunks_for_abstract)
self._pages_per_chunk_abstract = [pages_per_chunk for i in
range(
- 0, int(self.wiki.config.chunksForAbstract))]
+ 0, int(self.wiki.config.chunks_for_abstract))]
else:
self._pages_per_chunk_abstract = self.convert_comma_sep(
- self.wiki.config.pagesPerChunkAbstract)
+ self.wiki.config.pages_per_chunk_abstract)
self._pages_per_chunk_history = self.convert_comma_sep(
- self.wiki.config.pagesPerChunkHistory)
+ self.wiki.config.pages_per_chunk_history)
self._revs_per_chunk_history = self.convert_comma_sep(
- self.wiki.config.revsPerChunkHistory)
- self._recombine_history = self.wiki.config.recombineHistory
+ self.wiki.config.revs_per_chunk_history)
+ self._recombine_history = self.wiki.config.recombine_history
else:
self._pages_per_chunk_history = False
self._revs_per_chunk_history = False
diff --git a/xmldumps-backup/monitor.py b/xmldumps-backup/monitor.py
index 5c4715c..9a2bc1a 100644
--- a/xmldumps-backup/monitor.py
+++ b/xmldumps-backup/monitor.py
@@ -16,31 +16,31 @@
states = []
if sorted_by_db:
- dbs = sorted(config.dbList)
+ dbs = sorted(config.db_list)
else:
- dbs = config.dbListByAge()
+ dbs = config.db_list_by_age()
for db_name in dbs:
wiki = Wiki(config, db_name)
- if wiki.isStale():
+ if wiki.is_stale():
print db_name + " is stale"
- wiki.cleanupStaleLock()
+ wiki.cleanup_stale_lock()
if showlocks:
- if wiki.isLocked():
+ if wiki.is_locked():
try:
- filehdl = open(wiki.lockFile(), 'r')
+ filehdl = open(wiki.lock_file(), 'r')
(host, pid) = filehdl.readline().split(" ")
filehdl.close()
print db_name, "is locked by pid", pid, "on", host
except:
print db_name, "is locked"
- running = running or wiki.isLocked()
- states.append(wiki.statusLine())
+ running = running or wiki.is_locked()
+ states.append(wiki.status_line())
if running:
status = "Dumps are in progress..."
elif exists("maintenance.txt"):
- status = FileUtils.readFile("maintenance.txt")
+ status = FileUtils.read_file("maintenance.txt")
else:
status = "Dump process is idle."
@@ -55,16 +55,16 @@
other_index_link = ('Also view sorted by <a href="%s">%s</a>'
% (os.path.basename(other_indexhtml),
other_sortedby))
- return config.readTemplate("download-index.html") % {
+ return config.read_template("download-index.html") % {
"otherIndexLink": other_index_link,
"status": status,
"items": "\n".join(states)}
def update_index(config):
- output_fname = os.path.join(config.publicDir, config.index)
+ output_fname = os.path.join(config.public_dir, config.index)
output_fname_sorted_by_db = add_to_filename(os.path.join(
- config.publicDir, config.index), "bydb")
+ config.public_dir, config.index), "bydb")
temp_fname = output_fname + ".tmp"
filehdl = open(temp_fname, "wt")
diff --git a/xmldumps-backup/worker.py b/xmldumps-backup/worker.py
index 66bc60a..0c1d48b 100644
--- a/xmldumps-backup/worker.py
+++ b/xmldumps-backup/worker.py
@@ -65,10 +65,10 @@
def __init__(self, wiki, prefetch, spawn, chunk_to_do, checkpoint_file,
singleJob, skip_jobs, chunk_info, page_id_range,
runinfo_file, dump_dir):
self.wiki = wiki
- self._has_flagged_revs = self.wiki.hasFlaggedRevs()
- self._has_wikidata = self.wiki.hasWikidata()
+ self._has_flagged_revs = self.wiki.has_flagged_revs()
+ self._has_wikidata = self.wiki.has_wikidata()
self._has_global_usage = self.wiki.has_global_usage()
- self._is_wikidata_client = self.wiki.isWikidataClient()
+ self._is_wikidata_client = self.wiki.is_wikidata_client()
self._prefetch = prefetch
self._spawn = spawn
self.chunk_info = chunk_info
@@ -80,7 +80,7 @@
self.dump_dir = dump_dir
self.page_id_range = page_id_range
- if self.wiki.config.checkpointTime:
+ if self.wiki.config.checkpoint_time:
checkpoints = True
else:
checkpoints = False
@@ -177,7 +177,7 @@
AllTitleDump("allpagetitlesdump", "List of all page
titles"),
AbstractDump("abstractsdump", "Extracted page
abstracts for Yahoo",
-
self._get_chunk_to_do("abstractsdump"), self.wiki.dbName,
+
self._get_chunk_to_do("abstractsdump"), self.wiki.db_name,
self.chunk_info.get_pages_per_chunk_abstract())]
if self.chunk_info.chunks_enabled():
@@ -334,7 +334,7 @@
"most mirror sites won't want or need this.",
self.find_item_by_name('metahistory7zdump'), self.wiki))
# doing this only for recombined/full articles dump
- if self.wiki.config.multistreamEnabled:
+ if self.wiki.config.multistream_enabled:
if self.chunk_info.chunks_enabled():
input_for_multistream = "articlesdumprecombine"
else:
@@ -465,11 +465,11 @@
class Runner(object):
def __init__(self, wiki, prefetch=True, spawn=True, job=None,
skip_jobs=None,
- restart=False, notice="", dryrun=False, loggingEnabled=False,
+ restart=False, notice="", dryrun=False, logging_enabled=False,
chunk_to_do=False, checkpoint_file=None, page_id_range=None,
skipdone=False, verbose=False):
self.wiki = wiki
- self.db_name = wiki.dbName
+ self.db_name = wiki.db_name
self.prefetch = prefetch
self.spawn = spawn
self.chunk_info = Chunk(wiki, self.db_name, self.log_and_print)
@@ -494,7 +494,7 @@
"of checkpoint file %s to redo",
self.checkpoint_file)
self.checkpoint_file = fname
- self._logging_enabled = loggingEnabled
+ self._logging_enabled = logging_enabled
self._status_enabled = True
self._checksummer_enabled = True
self._runinfo_file_enabled = True
@@ -576,7 +576,7 @@
file_obj = DumpFilename(self.wiki)
file_obj.new_from_filename(self.wiki.config.log_file)
self.log_filename = self.dump_dir.filename_private_path(file_obj)
- self.make_dir(os.path.join(self.wiki.privateDir(), self.wiki.date))
+ self.make_dir(os.path.join(self.wiki.private_dir(),
self.wiki.date))
self.log = Logger(self.log_filename)
thread.start_new_thread(self.log_queue_reader, (self.log,))
self.runinfo_file = RunInfoFile(wiki, self._runinfo_file_enabled,
@@ -680,7 +680,7 @@
return 1
def debug(self, stuff):
- self.log_and_print("%s: %s %s" % (TimeUtils.prettyTime(),
self.db_name, stuff))
+ self.log_and_print("%s: %s %s" % (TimeUtils.pretty_time(),
self.db_name, stuff))
def run_handle_failure(self):
if self.status.fail_count < 1:
@@ -703,7 +703,7 @@
def run(self):
if self.job_requested:
- if not self.dump_item_list.old_runinfo_retrieved and
self.wiki.existsPerDumpIndex():
+ if not self.dump_item_list.old_runinfo_retrieved and
self.wiki.exists_perdump_index():
# There was a previous run of all or part of this date, but...
# There was no old RunInfo to be had (or an error was
encountered getting it)
@@ -730,8 +730,8 @@
Maintenance.exit_if_in_maintenance_mode(
"In maintenance mode, exiting dump of %s" % self.db_name)
- self.make_dir(os.path.join(self.wiki.publicDir(), self.wiki.date))
- self.make_dir(os.path.join(self.wiki.privateDir(), self.wiki.date))
+ self.make_dir(os.path.join(self.wiki.public_dir(), self.wiki.date))
+ self.make_dir(os.path.join(self.wiki.private_dir(), self.wiki.date))
self.show_runner_state("Cleaning up old dumps for %s" % self.db_name)
self.clean_old_dumps()
@@ -788,10 +788,10 @@
# special case
if self.job_requested == "createdirs":
- if not os.path.exists(os.path.join(self.wiki.publicDir(),
self.wiki.date)):
- os.makedirs(os.path.join(self.wiki.publicDir(),
self.wiki.date))
- if not os.path.exists(os.path.join(self.wiki.privateDir(),
self.wiki.date)):
- os.makedirs(os.path.join(self.wiki.privateDir(),
self.wiki.date))
+ if not os.path.exists(os.path.join(self.wiki.public_dir(),
self.wiki.date)):
+ os.makedirs(os.path.join(self.wiki.public_dir(),
self.wiki.date))
+ if not os.path.exists(os.path.join(self.wiki.private_dir(),
self.wiki.date)):
+ os.makedirs(os.path.join(self.wiki.private_dir(),
self.wiki.date))
if self.dump_item_list.all_possible_jobs_done():
# All jobs are either in status "done", "waiting", "failed",
"skipped"
@@ -839,10 +839,10 @@
and not removed."""
if self._clean_old_dumps_enabled:
if private:
- old = self.wiki.dumpDirs(private=True)
+ old = self.wiki.dump_dirs(private=True)
dumptype = 'private'
else:
- old = self.wiki.dumpDirs()
+ old = self.wiki.dump_dirs()
dumptype = 'public'
if old:
if old[-1] == self.wiki.date:
@@ -857,9 +857,9 @@
self.show_runner_state("Purging old %s dump %s for %s" %
(dumptype, dump, self.db_name))
if private:
- base = os.path.join(self.wiki.privateDir(), dump)
+ base = os.path.join(self.wiki.private_dir(), dump)
else:
- base = os.path.join(self.wiki.publicDir(), dump)
+ base = os.path.join(self.wiki.public_dir(), dump)
shutil.rmtree("%s" % base)
else:
self.show_runner_state("No old %s dumps to purge." % dumptype)
@@ -937,7 +937,7 @@
return False
if date == 'last':
- dumps = sorted(wiki.dumpDirs())
+ dumps = sorted(wiki.dump_dirs())
if dumps:
date = dumps[-1]
else:
@@ -947,11 +947,11 @@
if not job and prereqs:
return True
- wiki.setDate(date)
+ wiki.set_date(date)
runinfo_file = RunInfoFile(wiki, False)
- chunk_info = Chunk(wiki, wiki.dbName)
- dump_dir = DumpDir(wiki, wiki.dbName)
+ chunk_info = Chunk(wiki, wiki.db_name)
+ dump_dir = DumpDir(wiki, wiki.db_name)
dump_item_list = DumpItemList(wiki, False, False, chunk_to_do,
checkpoint_file,
job, skipjobs, chunk_info, page_id_range,
runinfo_file, dump_dir)
if not dump_item_list.old_runinfo_retrieved:
@@ -1002,7 +1002,7 @@
sys.stderr.write("Dump process halted by config.\n")
return None
- nextdbs = config.dbListByAge(bystatustime)
+ nextdbs = config.db_list_by_age(bystatustime)
nextdbs.reverse()
if verbose and not cutoff:
@@ -1015,7 +1015,7 @@
for dbname in nextdbs:
wiki = Wiki(config, dbname)
if cutoff:
- last_updated = wiki.dateTouchedLatestDump()
+ last_updated = wiki.date_touched_latest_dump()
if last_updated >= cutoff:
continue
if check_job_status:
@@ -1247,11 +1247,11 @@
if cutoff:
# fixme if we asked for a specific job then check that job only
# not the dir
- last_ran = wiki.latestDump()
+ last_ran = wiki.latest_dump()
if last_ran >= cutoff:
wiki = None
if wiki is not None and locks_enabled:
- if force_lock and wiki.isLocked():
+ if force_lock and wiki.is_locked():
wiki.unlock()
if locks_enabled:
wiki.lock()
@@ -1279,10 +1279,10 @@
if wiki is not None and wiki:
# process any per-project configuration options
- config.parseConfFilePerProject(wiki.dbName)
+ config.parse_conffile_per_project(wiki.db_name)
if date == 'last':
- dumps = sorted(wiki.dumpDirs())
+ dumps = sorted(wiki.dump_dirs())
if dumps:
date = dumps[-1]
else:
@@ -1290,7 +1290,7 @@
if date is None or not date:
date = TimeUtils.today()
- wiki.setDate(date)
+ wiki.set_date(date)
if after_checkpoint:
fname = DumpFilename(wiki)
@@ -1315,11 +1315,11 @@
if restart:
sys.stderr.write("Running %s, restarting from job %s...\n" %
- (wiki.dbName, job_requested))
+ (wiki.db_name, job_requested))
elif job_requested:
- sys.stderr.write("Running %s, job %s...\n" % (wiki.dbName,
job_requested))
+ sys.stderr.write("Running %s, job %s...\n" % (wiki.db_name,
job_requested))
else:
- sys.stderr.write("Running %s...\n" % wiki.dbName)
+ sys.stderr.write("Running %s...\n" % wiki.db_name)
result = runner.run()
if result is not None and result:
exitcode = 0
diff --git a/xmldumps-backup/xmlabstracts.py b/xmldumps-backup/xmlabstracts.py
index 478a4d6..97a86db 100644
--- a/xmldumps-backup/xmlabstracts.py
+++ b/xmldumps-backup/xmlabstracts.py
@@ -30,7 +30,7 @@
for filetype in outfiles:
outfiles[filetype]['temp'] = os.path.join(
- wikiconf.tempDir,
+ wikiconf.temp_dir,
os.path.basename(outfiles[filetype]['name']) + "_tmp")
if dryrun:
outfiles[filetype]['compr'] = None
@@ -41,7 +41,7 @@
"dumpBackup.php")
command = [wikiconf.php, "-q"] + script_command
version = MultiVersion.mw_version(wikiconf, wikidb)
- abstract_cmd_dir = wikiconf.wikiDir
+ abstract_cmd_dir = wikiconf.wiki_dir
if version:
abstract_cmd_dir = abstract_cmd_dir + "/" + version
abstract_filter = ("--plugin=AbstractFilter:"
@@ -169,7 +169,7 @@
"different number supplied")
wikiconf = Config(configfile)
- wikiconf.parseConfFilePerProject(wiki)
+ wikiconf.parse_conffile_per_project(wiki)
do_abstractsbackup(wiki, output_files, variants, wikiconf,
start, end, dryrun)
diff --git a/xmldumps-backup/xmllogs.py b/xmldumps-backup/xmllogs.py
index bccfa50..de22c25 100644
--- a/xmldumps-backup/xmllogs.py
+++ b/xmldumps-backup/xmllogs.py
@@ -24,7 +24,7 @@
'''
outfiles = {'logs': {'name': outfile}}
for filetype in outfiles:
- outfiles[filetype]['temp'] = os.path.join(wikiconf.tempDir,
os.path.basename(outfiles[filetype]['name']) + "_tmp")
+ outfiles[filetype]['temp'] = os.path.join(wikiconf.temp_dir,
os.path.basename(outfiles[filetype]['name']) + "_tmp")
if dryrun:
outfiles[filetype]['compr'] = None
else:
@@ -132,7 +132,7 @@
usage("no such file found: " + configfile)
wikiconf = Config(configfile)
- wikiconf.parseConfFilePerProject(wiki)
+ wikiconf.parse_conffile_per_project(wiki)
dologsbackup(wiki, output_file, wikiconf, start, end, dryrun)
if __name__ == '__main__':
diff --git a/xmldumps-backup/xmlstubs.py b/xmldumps-backup/xmlstubs.py
index 31a1bb4..99c8916 100644
--- a/xmldumps-backup/xmlstubs.py
+++ b/xmldumps-backup/xmlstubs.py
@@ -27,7 +27,7 @@
'articles': {'name': articles_file}}
for filetype in outfiles:
outfiles[filetype]['temp'] = os.path.join(
- wikiconf.tempDir, os.path.basename(outfiles[filetype]['name']) +
"_tmp")
+ wikiconf.temp_dir, os.path.basename(outfiles[filetype]['name']) +
"_tmp")
if dryrun:
outfiles[filetype]['compr'] = None
else:
@@ -152,7 +152,7 @@
usage("no such file found: " + configfile)
wikiconf = Config(configfile)
- wikiconf.parseConfFilePerProject(wiki)
+ wikiconf.parse_conffile_per_project(wiki)
dostubsbackup(wiki, history_file, current_file, articles_file, wikiconf,
start, end, dryrun)
if __name__ == '__main__':
--
To view, visit https://gerrit.wikimedia.org/r/248866
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I512c498974d5c9074781bfe5890606b45ff3f0d0
Gerrit-PatchSet: 1
Gerrit-Project: operations/dumps
Gerrit-Branch: ariel
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
