ArielGlenn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/280101 )
Change subject: prep-dumps-deploy full pylint and pep8 cleanup
......................................................................
prep-dumps-deploy full pylint and pep8 cleanup
Change-Id: I1d80b9d4679e6e51ca2ce9df5e9360a3c8ff04e5
---
M deployment/obsolete/prep-dumps-deploy.py
1 file changed, 106 insertions(+), 76 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/deployment/obsolete/prep-dumps-deploy.py
b/deployment/obsolete/prep-dumps-deploy.py
index e81edd0..8e393dc 100644
--- a/deployment/obsolete/prep-dumps-deploy.py
+++ b/deployment/obsolete/prep-dumps-deploy.py
@@ -1,56 +1,75 @@
-import os, sys, time, shutil, re
-# this script copies the most recent files into a deployment directory with
the current date
-# run on fenari/bastion host as root or fail.
+"""
+this script copies the most recent files into
+ a deployment directory with the current date
+run on fenari/bastion host as root or fail.
+"""
-basedir = "/home/wikipedia/downloadserver/snapshothosts/dumps"
-deploy = "deploy" # subdir where deployment trees are stored by date
-confs = "confs" # subdir where config files live
+import os
+import sys
+import time
+import shutil
+import re
+
+
+BASEDIR = "/home/wikipedia/downloadserver/snapshothosts/dumps"
+DEPLOY = "deploy" # subdir where deployment trees are stored by date
+CONFS = "confs" # subdir where config files live
+MONTHNAMES = ["jan", "feb", "mar", "apr", "may", "jun",
+ "jul", "aug", "sep", "oct", "nov", "dec"]
+
class Error(Exception):
"""Base class for exceptions in this module."""
pass
-def dateToDigits(dateString):
- if '-' not in dateString:
+
+def date_to_digits(date_string):
+ """convert date string in form mon-dd-yyyy to tuple
+ of year, monthnum, day and return it, or None
+ on error"""
+ if '-' not in date_string:
return None
- month, day, year = dateString.split('-', 2)
+ month, day, year = date_string.split('-', 2)
if not month.isdigit():
- if not month in monthNames:
+ if month not in MONTHNAMES:
return None
else:
- month = int(monthNames.index(month)) + 1
+ month = int(MONTHNAMES.index(month)) + 1
day = int(day)
- return "%s%02d%02d" % ( year, month, day)
+ return "%s%02d%02d" % (year, month, day)
-def getLatestDeployDate(deploydir):
+
+def get_latest_depl_date(deploydir):
try:
subdirs = os.listdir(deploydir)
except:
sys.stderr.write("Failed to read contents of %s\n" % deploydir)
raise
- deployDates = {}
- for d in subdirs:
- if not os.path.isdir(os.path.join(deploydir, d)):
+ deploy_dates = {}
+ for dname in subdirs:
+ if not os.path.isdir(os.path.join(deploydir, dname)):
continue
# expect mon-dd-yyyy
- canonicalDirName = dateToDigits(d)
- if not canonicalDirName:
+ canonical_dirname = date_to_digits(dname)
+ if not canonical_dirname:
continue
- deployDates[canonicalDirName] = d
- if not len(deployDates.keys()):
+ deploy_dates[canonical_dirname] = dname
+ if not len(deploy_dates.keys()):
return None
- dates = deployDates.keys()
- dates.sort(reverse = True)
- return deployDates[dates[0]]
+ dates = deploy_dates.keys()
+ dates.sort(reverse=True)
+ return deploy_dates[dates[0]]
+
# modified from the python copytree implementation
-def doCopy(sourcedir, targetdir):
+def do_copy(sourcedir, targetdir):
"""only for regular files, symlinks, dirs:
will attempt to remove files/symlinks in target dir that
are to be copied from the source dir, leaving any other contents
of target dir tree in place"""
names = os.listdir(sourcedir)
+ errors = []
if not os.path.isdir(targetdir):
# if this fails we want to give up on the spot since
# there will be no target directory to receive the contents
@@ -58,10 +77,9 @@
os.makedirs(targetdir)
# fixme is this the right set of errors?
except (IOError, os.error), why:
- errors.append((sourcepath, targetpath, str(why)))
+ errors.append((sourcedir, targetdir, str(why)))
raise Error(errors)
-
- errors = []
+
for name in names:
sourcepath = os.path.join(sourcedir, name)
targetpath = os.path.join(targetdir, name)
@@ -79,14 +97,14 @@
linkto = os.readlink(sourcepath)
os.symlink(linkto, targetpath)
elif os.path.isdir(sourcepath):
- doCopy(sourcepath, targetpath)
+ do_copy(sourcepath, targetpath)
elif os.path.isfile(sourcepath):
shutil.copy2(sourcepath, targetpath)
else:
errors.append("refusingto remove %s, not file or dir or
symlink\n")
except (IOError, os.error), why:
errors.append((sourcepath, targetpath, str(why)))
- # catch the Error from the recursive doCopy so that we can
+ # catch the Error from the recursive do_copy so that we can
# continue with other files
except Error, err:
errors.extend(err.args[0])
@@ -97,88 +115,100 @@
if errors:
raise Error(errors)
-def setConfPerms(path):
- confDir = os.path.join(path, confs)
- for f in os.listdir(confDir):
- os.chmod(os.path.join(confDir,f), 0600)
-def usage(message = None):
+def set_conf_perms(path):
+ """make sure all files in the conf dir are read only
+ by owner"""
+ conf_dir = os.path.join(path, CONFS)
+ for fname in os.listdir(conf_dir):
+ os.chmod(os.path.join(conf_dir, fname), 0600)
+
+
+def usage(message=None):
if message:
sys.stderr.write("%s\n" % message)
- sys.stderr.write("Usage: python prep-dumps-deploy.py [deploydate]\n")
- sys.stderr.write("\n")
- sys.stderr.write("This script copies the currently deployed junk to a new
directory tree named\n")
- sys.stderr.write("with today's date, in preparation for the user updating
specific files for\n")
- sys.stderr.write("push to a cluster of hosts.\n")
- sys.stderr.write("\n")
- sys.stderr.write("This script takes one option, the deploydate, which is
the name of the new directory\n")
- sys.stderr.write("to create or update. This can be specified either as
mm-dd-yyy or as mon-dd-yyyy where\n")
- sys.stderr.write("mon is the first three letters of the month, in lower
case, eg. 'mar-12-2012'\n")
- sys.stderr.write("If this argument is not specified, today's date will be
used to generate the dir name.\n")
+ usage_message = """
+Usage: python prep-dumps-deploy.py [deploydate]
+
+This script copies the currently deployed junk to a new directory
+tree named with today's date, in preparation for the user updating
+specific files for push to a cluster of hosts.
+
+This script takes one option, the deploydate, which is the name
+of the new directory to create or update. This can be specified
+either as mm-dd-yyy or as mon-dd-yyyy where 'mon' is the first three
+letters of the month, in lower case, eg. 'mar-12-2012'. If this
+argument is not specified, today's date will be used to generate
+ the dir name.
+"""
+ sys.stderr.write(usage_message)
sys.exit(1)
+
def main():
subdir = None
- monthNames = [ "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug",
"sep", "oct", "nov", "dec" ]
- deploydir = os.path.join(basedir, deploy)
+ deploydir = os.path.join(BASEDIR, DEPLOY)
- today=time.strftime("%m-%d-%Y", time.gmtime())
+ today = time.strftime("%m-%d-%Y", time.gmtime())
month, day, year = today.split('-')
- todayDir = "%s-%s-%s" % (monthNames[int(month)-1], day, year)
+ today_dir = "%s-%s-%s" % (MONTHNAMES[int(month)-1], day, year)
if len(sys.argv) > 1:
- newDate = sys.argv[1]
- month, rest = newDate.split('-', 1)
+ new_date = sys.argv[1]
+ month, rest = new_date.split('-', 1)
if not re.match("^[0-9][0-9]-20[0-9][0-9]$", rest):
usage("bad date format")
if month.isdigit():
- subdir = "%s-%s" % (monthNames[int(month)-1], rest)
- elif month not in monthNames:
+ subdir = "%s-%s" % (MONTHNAMES[int(month)-1], rest)
+ elif month not in MONTHNAMES:
usage("bad date format")
else:
- subdir = newDate
-
+ subdir = new_date
+
if os.geteuid() != 0:
sys.stderr.write("Script must be run as root.\n")
sys.exit(1)
if not subdir:
- subdir = todayDir
+ subdir = today_dir
- fullPathDest = os.path.join(deploydir, subdir)
+ full_path_dest = os.path.join(deploydir, subdir)
if not os.path.isdir(deploydir):
- sys.stderr.write("Directory %s does not exist or is not a directory,
giving up.\n" % deploydir)
+ sys.stderr.write("Directory %s does not exist or "
+ "is not a directory, giving up.\n" % deploydir)
sys.exit(1)
-
+
# what's the dir date that's most recent?
- latestDeployDate = getLatestDeployDate(deploydir)
- if not latestDeployDate:
- sys.stderr.write("There seems to be no deployment directory in %s we
can copy, giving up \n" % deploydir)
+ latest_deploy_date = get_latest_depl_date(deploydir)
+ if not latest_deploy_date:
+ sys.stderr.write("There seems to be no deployment directory"
+ " in %s we can copy, giving up \n" % deploydir)
sys.exit(1)
-
- fullPathSrc = os.path.join(deploydir,latestDeployDate)
- print "Setting up deployment dir", fullPathDest, "from", latestDeployDate
+ full_path_src = os.path.join(deploydir, latest_deploy_date)
- if os.path.isdir(fullPathDest):
- print "New deployment dir already exists. overwrite, remove and copy,
or skip [O/r/s]? ",
- input = sys.stdin.readline().strip()
- if input == 'O' or input == 'o':
- doCopy(fullPathSrc, fullPathDest)
- elif input == 'r' or input == 'R':
- shutil.rmtree(fullPathDest)
- doCopy(fullPathSrc, fullPathDest)
- elif input == 's' or input == 'S':
+ print "Setting up deployment dir", full_path_dest, "from",
latest_deploy_date
+
+ if os.path.isdir(full_path_dest):
+ print ("New deployment dir already exists. overwrite,"
+ "remove and copy, or skip [O/r/s]? "),
+ reply = sys.stdin.readline().strip()
+ if reply == 'O' or reply == 'o':
+ do_copy(full_path_src, full_path_dest)
+ elif reply == 'r' or reply == 'R':
+ shutil.rmtree(full_path_dest)
+ do_copy(full_path_src, full_path_dest)
+ elif reply == 's' or reply == 'S':
print "Skipping at user's request"
sys.exit(0)
- else:
+ else:
print "Unknown response, giving up."
sys.exit(1)
else:
- doCopy(fullPathSrc, fullPathDest)
- setConfPerms(fullPathDest)
+ do_copy(full_path_src, full_path_dest)
+ set_conf_perms(full_path_dest)
if __name__ == "__main__":
main()
--
To view, visit https://gerrit.wikimedia.org/r/280101
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I1d80b9d4679e6e51ca2ce9df5e9360a3c8ff04e5
Gerrit-PatchSet: 2
Gerrit-Project: operations/dumps
Gerrit-Branch: ariel
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
