Diederik has submitted this change and it was merged.
Change subject: Make code pep8 compliant.
......................................................................
Make code pep8 compliant.
Change-Id: I99e2cd4a0f444ec6b03474639072a482eb363255
---
M labs-migration-assistant/ansistrm.py
M labs-migration-assistant/fabfile.py
2 files changed, 241 insertions(+), 199 deletions(-)
Approvals:
Diederik: Verified; Looks good to me, approved
diff --git a/labs-migration-assistant/ansistrm.py
b/labs-migration-assistant/ansistrm.py
index f53a558..4d4c2a3 100644
--- a/labs-migration-assistant/ansistrm.py
+++ b/labs-migration-assistant/ansistrm.py
@@ -4,7 +4,8 @@
import ctypes
import logging
import os
-
+
+
class ColorizingStreamHandler(logging.StreamHandler):
# color names to indices
color_map = {
@@ -17,8 +18,8 @@
'cyan': 6,
'white': 7,
}
-
- #levels to (background, foreground, bold/intense)
+
+ # levels to (background, foreground, bold/intense)
if os.name == 'nt':
level_map = {
logging.DEBUG: (None, 'green', True),
@@ -37,12 +38,12 @@
}
csi = '\x1b['
reset = '\x1b[0m'
-
+
@property
def is_tty(self):
isatty = getattr(self.stream, 'isatty', None)
return isatty and isatty()
-
+
def emit(self, record):
try:
message = self.format(record)
@@ -57,14 +58,14 @@
raise
except:
self.handleError(record)
-
+
if os.name != 'nt':
def output_colorized(self, message):
self.stream.write(message)
else:
import re
ansi_esc = re.compile(r'\x1b\[((?:\d+)(?:;(?:\d+))*)m')
-
+
nt_color_map = {
0: 0x00, # black
1: 0x04, # red
@@ -75,7 +76,7 @@
6: 0x03, # cyan
7: 0x07, # white
}
-
+
def output_colorized(self, message):
parts = self.ansi_esc.split(message)
write = self.stream.write
@@ -83,7 +84,7 @@
fd = getattr(self.stream, 'fileno', None)
if fd is not None:
fd = fd()
- if fd in (1, 2): # stdout or stderr
+ if fd in (1, 2): # stdout or stderr
h = ctypes.windll.kernel32.GetStdHandle(-10 - fd)
while parts:
text = parts.pop(0)
@@ -100,13 +101,14 @@
elif 30 <= p <= 37:
color |= self.nt_color_map[p - 30]
elif p == 1:
- color |= 0x08 # foreground intensity on
- elif p == 0: # reset to default color
+ color |= 0x08 # foreground intensity on
+ elif p == 0: # reset to default color
color = 0x07
else:
- pass # error condition ignored
- ctypes.windll.kernel32.SetConsoleTextAttribute(h,
color)
-
+ pass # error condition ignored
+ ctypes.windll.kernel32.SetConsoleTextAttribute(
+ h, color)
+
def colorize(self, message, record):
if record.levelno in self.level_map:
bg, fg, bold = self.level_map[record.levelno]
@@ -121,7 +123,7 @@
message = ''.join((self.csi, ';'.join(params),
'm', message, self.reset))
return message
-
+
def format(self, record):
message = logging.StreamHandler.format(self, record)
if self.is_tty:
@@ -130,7 +132,8 @@
parts[0] = self.colorize(parts[0], record)
message = '\n'.join(parts)
return message
-
+
+
def main():
root = logging.getLogger()
root.setLevel(logging.DEBUG)
@@ -140,6 +143,6 @@
logging.warning('WARNING')
logging.error('ERROR')
logging.critical('CRITICAL')
-
+
if __name__ == '__main__':
- main()
\ No newline at end of file
+ main()
diff --git a/labs-migration-assistant/fabfile.py
b/labs-migration-assistant/fabfile.py
index a3b56c0..82c0230 100644
--- a/labs-migration-assistant/fabfile.py
+++ b/labs-migration-assistant/fabfile.py
@@ -4,7 +4,7 @@
'''
labs-migration-assistant: a script to assess the readyness of
a Wikimedia Labs instance to be migrated from the Tampa dc to
-the Ashburn dc.
+the Ashburn dc.
Copyright (C) 2014 Diederik van Liere, Wikimedia Foundation
@@ -41,12 +41,13 @@
logger = logging.getLogger()
logger.setLevel(logging.INFO)
-formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s -
%(message)s')
+formatter = logging.Formatter(
+ '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler = ColorizingStreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
'''
-Fabric logging messages have their own hardcoded format and will thus not
follow the
+Fabric logging messages have their own hardcoded format and will thus not
follow the
formatter format. See also https://github.com/fabric/fabric/issues/163
'''
@@ -62,40 +63,44 @@
class LabInstance:
- def __init__(self, name, project, datacenter):
- self.tasks = ['detect_self_puppetmaster',
'detect_last_puppet_run', 'detect_shared_storage_for_projects',
'detect_shared_storage_for_home']
- self.name = name
- self.project = project
- self.datacenter = datacenter
- self.connect = None
- self.add_instance_to_fabric_env()
- for task in self.tasks:
- setattr(self, task, True)
- def __str__(self):
- return '%s.%s.wmflabs' % (self.name, self.datacenter)
+ def __init__(self, name, project, datacenter):
+ self.tasks = ['detect_self_puppetmaster', 'detect_last_puppet_run',
+ 'detect_shared_storage_for_projects',
'detect_shared_storage_for_home']
+ self.name = name
+ self.project = project
+ self.datacenter = datacenter
+ self.connect = None
+ self.add_instance_to_fabric_env()
+ for task in self.tasks:
+ setattr(self, task, True)
- def __repr__(self):
- return str(self)
+ def __str__(self):
+ return '%s.%s.wmflabs' % (self.name, self.datacenter)
- def add_instance_to_fabric_env(self):
- env.hosts.append(str(self))
+ def __repr__(self):
+ return str(self)
- def count_errors(self):
- return sum([1 for task in self.tasks if getattr(self, task) ==
False])
+ def add_instance_to_fabric_env(self):
+ env.hosts.append(str(self))
+
+ def count_errors(self):
+ return sum([1 for task in self.tasks if getattr(self, task) is False])
+
def check_connection(func, *args, **kwargs):
- '''Check connection decorator'''
- @functools.wraps(func)
- def wrapper(*args, **kwargs):
- print kwargs
- print args
- output = func(*args, **kwargs)
- print output
- #if not labinstances[env.host_string].connect:
- # logging.info('Skipping task because during first task
was not able to connect to labsinstance.')
- return output
- return wrapper
+ '''Check connection decorator'''
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ print kwargs
+ print args
+ output = func(*args, **kwargs)
+ print output
+ # if not labinstances[env.host_string].connect:
+ # logging.info('Skipping task because during first task was not
able to connect to labsinstance.')
+ return output
+ return wrapper
+
def logged(func):
'''Logging decorator.'''
@@ -107,183 +112,217 @@
return output
return wrapper
+
def parse_lab_instances(html_raw):
- labinstances = {}
- soup = BeautifulSoup(html_raw)
- container = soup.find('div', {'id': 'mw-content-text'})
- if container:
- for tag in container.children:
- if tag.name == 'h2':
- project = tag.get('id')
- logging.info('Found project: %s' % project)
- elif tag.name == 'div':
- dc = tag.find('h3').text.strip()
- logging.info('Datacenter: %s' % dc)
- table = tag.find('table')
- '''
- TODO: the last <tr> from the table is not
found, probably me doing something dumb
- but that means that the final labs instance
will not be analyzed :(
- '''
- cells = table.find('tr')
- for cell in cells:
- if 'novainstancename' in
cell.attrs['class']:
- logging.info('Instance: %s' %
cell.text)
- labinstance =
LabInstance(cell.text, project, dc)
- labinstances[str(labinstance)]
= labinstance
- return labinstances
+ labinstances = {}
+ soup = BeautifulSoup(html_raw)
+ container = soup.find('div', {'id': 'mw-content-text'})
+ if container:
+ for tag in container.children:
+ if tag.name == 'h2':
+ project = tag.get('id')
+ logging.info('Found project: %s' % project)
+ elif tag.name == 'div':
+ dc = tag.find('h3').text.strip()
+ logging.info('Datacenter: %s' % dc)
+ table = tag.find('table')
+ '''
+ TODO: the last <tr> from the table is not found, probably me
doing something dumb
+ but that means that the final labs instance will not be
analyzed :(
+ '''
+ cells = table.find('tr')
+ for cell in cells:
+ if 'novainstancename' in cell.attrs['class']:
+ logging.info('Instance: %s' % cell.text)
+ labinstance = LabInstance(cell.text, project, dc)
+ labinstances[str(labinstance)] = labinstance
+ return labinstances
+
def fetch_lab_instances():
- get_token_url =
'https://wikitech.wikimedia.org/w/api.php?action=login&lgname=%s&lgpassword=%s&format=json'
% (env.wiki_username, env.wiki_password)
- url = 'https://wikitech.wikimedia.org/wiki/Special:NovaResources'
- verify = False
- result = ''
- try:
- session = requests.Session()
- token_request = session.post(get_token_url, verify=verify)
- lgtoken = json.loads(token_request.text).get('login',
{}).get('token')
- sessionid = json.loads(token_request.text).get('login',
{}).get('sessionid')
- confirm_token_url = '%s&lgtoken=%s' % (get_token_url, lgtoken)
- headers = {'sessionid' : sessionid}
- confirm_request = session.post(confirm_token_url,
headers=headers, verify=verify)
- request = session.get(url, verify=verify)
- result = request.text
- except requests.exceptions.ConnectionError, e:
- logging.error(e)
- finally:
- session.close()
- return result
+ get_token_url =
'https://wikitech.wikimedia.org/w/api.php?action=login&lgname=%s&lgpassword=%s&format=json'
% (
+ env.wiki_username, env.wiki_password)
+ url = 'https://wikitech.wikimedia.org/wiki/Special:NovaResources'
+ verify = False
+ result = ''
+ try:
+ session = requests.Session()
+ token_request = session.post(get_token_url, verify=verify)
+ lgtoken = json.loads(token_request.text).get('login', {}).get('token')
+ sessionid = json.loads(token_request.text).get(
+ 'login', {}).get('sessionid')
+ confirm_token_url = '%s&lgtoken=%s' % (get_token_url, lgtoken)
+ headers = {'sessionid': sessionid}
+ confirm_request = session.post(
+ confirm_token_url, headers=headers, verify=verify)
+ request = session.get(url, verify=verify)
+ result = request.text
+ except requests.exceptions.ConnectionError, e:
+ logging.error(e)
+ finally:
+ session.close()
+ return result
+
@task
def detect_self_puppetmaster(labinstances):
- try:
- result = run('grep "^server = virt0.wikimedia.org"
/etc/puppet/puppet.conf | wc -l')
- if result == 0:
- logging.info('You are not using a self-hosted puppet
master. [OK]')
- else:
- logging.error('You are running your own self-hosted
puppet master. [FAIL]')
- labinstances[env.host_string].detect_self_puppetmaster
= False
- labinstances[env.host_string].connect = True
- except SystemExit:
- logging.error('Could not connect to %s.' % env.host_string)
- labinstances[env.host_string].connect = False # we only to set
this once if we cannot connect to the instance.
-
+ try:
+ result = run(
+ 'grep "^server = virt0.wikimedia.org" /etc/puppet/puppet.conf | wc
-l')
+ if result == 0:
+ logging.info('You are not using a self-hosted puppet master. [OK]')
+ else:
+ logging.error(
+ 'You are running your own self-hosted puppet master. [FAIL]')
+ labinstances[env.host_string].detect_self_puppetmaster = False
+ labinstances[env.host_string].connect = True
+ except SystemExit:
+ logging.error('Could not connect to %s.' % env.host_string)
+ # we only to set this once if we cannot connect to the instance.
+ labinstances[env.host_string].connect = False
+
@task
def detect_last_puppet_run(labinstances):
- '''
- TODO: refactor this check into a decorator function, but Fabric is not
really cooperating
- '''
- if not labinstances[env.host_string].connect:
- if not labinstances[env.host_string].connect:
- logging.info('Skipping task because during first task
was not able to connect to labsinstance.')
- return
+ '''
+ TODO: refactor this check into a decorator function, but Fabric is not
really cooperating
+ '''
+ if not labinstances[env.host_string].connect:
+ if not labinstances[env.host_string].connect:
+ logging.info(
+ 'Skipping task because during first task was not able to
connect to labsinstance.')
+ return
- try:
- result = sudo('cat /var/lib/puppet/state/last_run_summary.yaml')
- doc = yaml.load(result)
- epoch = doc.get('time', {}).get('last_run', 0)
- if epoch == 0:
- logging.error('We could not determine the last time
puppet was run. [FAIL]')
- else:
- epoch = datetime.fromtimestamp(epoch)
- now = datetime.now()
- dt = now - epoch
- if dt.days > 29:
- logging.error('The last puppet run was %d days
ago, please run puppet. [FAIL]' % dt.days)
-
labinstances[env.host_string].detect_last_puppet_run = False
- else:
- logging.info('Puppet is up-to-date. [OK]')
- except SystemExit:
- logging.error('Could not connect to %s.' % env.host_string)
+ try:
+ result = sudo('cat /var/lib/puppet/state/last_run_summary.yaml')
+ doc = yaml.load(result)
+ epoch = doc.get('time', {}).get('last_run', 0)
+ if epoch == 0:
+ logging.error(
+ 'We could not determine the last time puppet was run. [FAIL]')
+ else:
+ epoch = datetime.fromtimestamp(epoch)
+ now = datetime.now()
+ dt = now - epoch
+ if dt.days > 29:
+ logging.error(
+ 'The last puppet run was %d days ago, please run puppet.
[FAIL]' % dt.days)
+ labinstances[env.host_string].detect_last_puppet_run = False
+ else:
+ logging.info('Puppet is up-to-date. [OK]')
+ except SystemExit:
+ logging.error('Could not connect to %s.' % env.host_string)
+
@task
def detect_shared_storage_for_projects(labinstances):
- '''
- TODO: refactor this check into a decorator function, but Fabric is not
really cooperating
- '''
- if not labinstances[env.host_string].connect:
- if not labinstances[env.host_string].connect:
- logging.info('Skipping task because during first task
was not able to connect to labsinstance.')
- return
+ '''
+ TODO: refactor this check into a decorator function, but Fabric is not
really cooperating
+ '''
+ if not labinstances[env.host_string].connect:
+ if not labinstances[env.host_string].connect:
+ logging.info(
+ 'Skipping task because during first task was not able to
connect to labsinstance.')
+ return
- try:
- result = run('ls -l | wc -l')
- result = int(result)
- if result == 0:
- logging.info('You seem not to be using your home folder
for storing files. [OK]')
- else:
- logging.error('You seem to be using your home folder
for storing files and folders. Please migrate your files to /data/projects/.
[FAIL]')
-
labinstances[env.host_string].detect_shared_storage_for_projects = False
- except SystemExit:
- logging.error('Could not connect to %s.' % env.host_string)
+ try:
+ result = run('ls -l | wc -l')
+ result = int(result)
+ if result == 0:
+ logging.info(
+ 'You seem not to be using your home folder for storing files.
[OK]')
+ else:
+ logging.error(
+ 'You seem to be using your home folder for storing files and
folders. Please migrate your files to /data/projects/. [FAIL]')
+ labinstances[
+ env.host_string].detect_shared_storage_for_projects = False
+ except SystemExit:
+ logging.error('Could not connect to %s.' % env.host_string)
+
@task
def detect_shared_storage_for_home(labinstances):
- '''
- TODO: refactor this check into a decorator function, but Fabric is not
really cooperating
- '''
- if not labinstances[env.host_string].connect:
- if not labinstances[env.host_string].connect:
- logging.info('Skipping task because during first task
was not able to connect to labsinstance.')
- return
+ '''
+ TODO: refactor this check into a decorator function, but Fabric is not
really cooperating
+ '''
+ if not labinstances[env.host_string].connect:
+ if not labinstances[env.host_string].connect:
+ logging.info(
+ 'Skipping task because during first task was not able to
connect to labsinstance.')
+ return
- try:
- result = run('df $HOME')
- result = result.splitlines()[1]
- if result.find(':') > -1:
- logging.info('You seem to be using the shared storage
space for your home folder. [OK]')
- else:
- logging.error('You do not seem to be using the shared
storage space for your home folder. [FAIL]')
-
labinstances[env.host_string].detect_shared_storage_for_home = False
- except SystemExit:
- logging.error('Could not connect to %s.' % env.host_string)
+ try:
+ result = run('df $HOME')
+ result = result.splitlines()[1]
+ if result.find(':') > -1:
+ logging.info(
+ 'You seem to be using the shared storage space for your home
folder. [OK]')
+ else:
+ logging.error(
+ 'You do not seem to be using the shared storage space for your
home folder. [FAIL]')
+ labinstances[
+ env.host_string].detect_shared_storage_for_home = False
+ except SystemExit:
+ logging.error('Could not connect to %s.' % env.host_string)
+
@task(default=True)
def migrate_ready():
- if 'debug' in env and env.debug == True:
- test_instance = LabInstance('limn0', 'analytics', 'pmtpa')
- labinstances = {'limn0.pmtpa.wmflabs': test_instance}
- else:
- html_raw = fetch_lab_instances()
- labinstances = parse_lab_instances(html_raw)
- hosts = ['%s.%s.wmflabs' % (labinstance.name, labinstance.datacenter)
for labinstance in labinstances.values()]
+ if 'debug' in env and env.debug is True:
+ test_instance = LabInstance('limn0', 'analytics', 'pmtpa')
+ labinstances = {'limn0.pmtpa.wmflabs': test_instance}
+ else:
+ html_raw = fetch_lab_instances()
+ labinstances = parse_lab_instances(html_raw)
+ hosts = ['%s.%s.wmflabs' % (labinstance.name, labinstance.datacenter)
+ for labinstance in labinstances.values()]
- if len(hosts) == 0:
- logging.error('I was either not able to parse the Wikitech page
containing your lab instances or you are not the administrator for any lab
instance.')
- exit(-1)
+ if len(hosts) == 0:
+ logging.error(
+ 'I was either not able to parse the Wikitech page containing your
lab instances or you are not the administrator for any lab instance.')
+ exit(-1)
- execute(detect_self_puppetmaster, hosts=hosts,
labinstances=labinstances)
- execute(detect_last_puppet_run, hosts=hosts, labinstances=labinstances)
- execute(detect_shared_storage_for_projects, hosts=hosts,
labinstances=labinstances)
- execute(detect_shared_storage_for_home, hosts=hosts,
labinstances=labinstances)
-
- for labsinstance in labinstances.values():
- if not labsinstance.connect:
- logging.error('There were problems connecting
to instance %s, please fix those problems first and then rerun this script.' %
labsinstance)
- else:
- logging.error('%s does not yet seem to be ready
for migration to eqiad. Please fix the %d identified problems.' %
(labsinstance, labsinstance.count_errors()))
- logging.info('Summary of failed tasks:')
- problems = 1
- for task in labinstance.tasks:
- result = getattr(labsinstance, task)
- if not result:
- logging.info('Problem %d: task
%s [FAIL]' % (problems, task))
- problems +=1
- if problems == 1:
- logging.info('Congratulations! %s seems
ready for migration!' % labinstance)
+ execute(detect_self_puppetmaster, hosts=hosts, labinstances=labinstances)
+ execute(detect_last_puppet_run, hosts=hosts, labinstances=labinstances)
+ execute(detect_shared_storage_for_projects,
+ hosts=hosts, labinstances=labinstances)
+ execute(detect_shared_storage_for_home,
+ hosts=hosts, labinstances=labinstances)
+
+ for labsinstance in labinstances.values():
+ if not labsinstance.connect:
+ logging.error(
+ 'There were problems connecting to instance %s, please fix
those problems first and then rerun this script.' %
+ labsinstance)
+ else:
+ logging.error(
+ '%s does not yet seem to be ready for migration to eqiad.
Please fix the %d identified problems.' %
+ (labsinstance, labsinstance.count_errors()))
+ logging.info('Summary of failed tasks:')
+ problems = 1
+ for task in labinstance.tasks:
+ result = getattr(labsinstance, task)
+ if not result:
+ logging.info('Problem %d: task %s [FAIL]' %
+ (problems, task))
+ problems += 1
+ if problems == 1:
+ logging.info('Congratulations! %s seems ready for migration!' %
+ labinstance)
run = logged(run)
+
def main():
- print 'You should not run this script directly but instead call it as:'
- print 'fab migrate_ready --set
wiki_username=YOUR_WIKI_USERNAME,wiki_password=YOUR_WIKI_PASSWORD'
- print
- print 'The wiki that we are referring to is Wikitech.'
- print 'You might need to pass additional paramaters like your password
for your SSH key, but this'
- print 'depends on the actual setup of your system.'
- print 'For an overview of possible parameters run fab --help'
- exit(-1)
+ print 'You should not run this script directly but instead call it as:'
+ print 'fab migrate_ready --set
wiki_username=YOUR_WIKI_USERNAME,wiki_password=YOUR_WIKI_PASSWORD'
+ print
+ print 'The wiki that we are referring to is Wikitech.'
+ print 'You might need to pass additional paramaters like your password for
your SSH key, but this'
+ print 'depends on the actual setup of your system.'
+ print 'For an overview of possible parameters run fab --help'
+ exit(-1)
if __name__ == '__main__':
- main()
+ main()
--
To view, visit https://gerrit.wikimedia.org/r/107915
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I99e2cd4a0f444ec6b03474639072a482eb363255
Gerrit-PatchSet: 1
Gerrit-Project: labs/migration-assistant
Gerrit-Branch: master
Gerrit-Owner: Diederik <[email protected]>
Gerrit-Reviewer: Diederik <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
