Giuseppe Lavagetto has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/342492 )
Change subject: Initial import
......................................................................
Initial import
Bug: T160178
Change-Id: I6c64268e5a86be13afaa6d8185a6b1533fe71770
---
A .gitignore
A doc/examples/t01_example.py
A setup.py
A switchdc/__init__.py
A switchdc/log.py
A switchdc/menu.py
A switchdc/remote.py
A switchdc/stages/__init__.py
A switchdc/switch.py
A tox.ini
10 files changed, 369 insertions(+), 0 deletions(-)
Approvals:
Giuseppe Lavagetto: Verified; Looks good to me, approved
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..83769f1
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,8 @@
+/.eggs
+/.tox
+/*.egg-info
+/.coverage
+/setup.cfg
+/.venv
+/logs
+*.pyc
diff --git a/doc/examples/t01_example.py b/doc/examples/t01_example.py
new file mode 100644
index 0000000..b0cca69
--- /dev/null
+++ b/doc/examples/t01_example.py
@@ -0,0 +1,23 @@
+from switchdc import remote
+from switchdc.log import logger
+
+
+__title__ = "Example task description"
+
+
+def execute(dc_from, dc_to):
+ """Entry point to execute this task
+
+ Must not raise execeptions, all exceptions must be catched here and
managed.
+ Returns 0 on success, a positive integer on failure.
+
+ Arguments:
+ dc_from -- the name of the datacenter to switch from
+ dc_to -- the name of the datacenter to switch to
+ """
+ logger.debug(__name__)
+ print('Executed with {dc_from} - {dc_to}'.format(dc_from=dc_from,
dc_to=dc_to))
+ rc, worker = remote.run('R:Class = Role::Memcached', 'sync', ['date'],
+ success_threshold=0.9, batch_size=5, batch_sleep=5)
+
+ return rc
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..6946858
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+"""Package configuration."""
+
+from setuptools import find_packages, setup
+
+setup(
+ author='Riccardo Coccioli',
+ author_email='[email protected]',
+ description='Datacenter switchover automation',
+ entry_points={
+ 'console_scripts': [
+ 'switchdc = switchdc.switch:main',
+ ],
+ },
+ install_requires=['pyyaml'],
+ name='switchdc',
+ packages=find_packages(),
+ version='0.0.1',
+ zip_safe=False,
+)
diff --git a/switchdc/__init__.py b/switchdc/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/switchdc/__init__.py
diff --git a/switchdc/log.py b/switchdc/log.py
new file mode 100644
index 0000000..db644e6
--- /dev/null
+++ b/switchdc/log.py
@@ -0,0 +1,12 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+_log_formatter = logging.Formatter(
+ fmt='%(asctime)s [%(levelname)s %(filename)s:%(lineno)s in %(funcName)s]
%(message)s')
+_log_handler = logging.FileHandler('/var/log/switchdc.log')
+_log_handler.setFormatter(_log_formatter)
+logger.addHandler(_log_handler)
+logger.raiseExceptions = False
+logger.setLevel(logging.INFO)
diff --git a/switchdc/menu.py b/switchdc/menu.py
new file mode 100644
index 0000000..9ff04cf
--- /dev/null
+++ b/switchdc/menu.py
@@ -0,0 +1,115 @@
+from switchdc.log import logger
+
+
+class Menu(object):
+ """Menu class."""
+
+ def __init__(self, title, parent=None):
+ """Menu constructor
+
+ Arguments:
+ title -- the menu title to be show
+ parent -- reference to a parent menu if this is a submenu. [optional,
default: None]
+ """
+ self.title = title
+ self.parent = parent
+ self.items = []
+
+ @property
+ def status(self):
+ """Getter for the menu status, returns a string representation of the
status of it's tasks."""
+ completed, total = Menu.calculate_status(self)
+ return '{completed}/{total}'.format(completed=completed, total=total)
+
+ def append(self, item):
+ """Append an item or a submenu to this menu.
+
+ Arguments:
+ item -- the item to append
+ """
+ if type(item) == Menu:
+ item.parent = self
+ self.items.append(item)
+
+ def run(self):
+ """For menu run is equivalent to show."""
+ self.show()
+
+ def show(self):
+ """Print the menu to stdout."""
+ print(self.title)
+ for i, item in enumerate(self.items):
+ print(' {i: >2} [{status}] {title}'.format(i=i + 1,
title=item.title, status=item.status))
+
+ if self.parent is not None:
+ print(' b - Back to parent menu')
+
+ print(' q - Quit')
+
+ @staticmethod
+ def calculate_status(menu):
+ """Calculate the status of a menu, checking the status of all it's
tasks recursively.
+
+ Arguments:
+ menu -- the meny for which to calculate the status
+ """
+ completed = 0
+ total = 0
+ for item in menu.items:
+ item_type = type(item)
+ if item_type == Menu:
+ sub_completed, sub_total = Menu.calculate_status(item)
+ completed += sub_completed
+ total += sub_total
+ elif item_type == Item:
+ total += 1
+ if item.status != Item.todo:
+ completed += 1
+
+ return completed, total
+
+
+class Item(object):
+ """Menu item class."""
+
+ statuses = ('TODO', 'PASS', 'FAIL') # Status labels
+ todo, success, failed = statuses # Valid statuses variables
+
+ def __init__(self, name, title, function, args=None, kwargs=None):
+ """Item constructor.
+
+ Arguments
+ name -- the name of the module for this task
+ title -- the item's title to be shown in the menu
+ function -- the function to call when the task is run
+ args -- the list of positional arguments to pass to the function.
[optional, default: None]
+ kwargs -- the dictionary of keyword arguments to pass to the
function. [optional, default: None]
+ """
+ self.name = name
+ self.title = title
+ self.status = self.todo
+ self.function = function
+ if args is not None:
+ self.args = args
+ else:
+ self.args = []
+ if kwargs is not None:
+ self.kwargs = kwargs
+ else:
+ self.kwargs = {}
+
+ def run(self):
+ """Run the item callind the configured function."""
+ try:
+ retval = self.function(*self.args, **self.kwargs)
+ except Exception as e:
+ retval = -1
+ print('FAILED: {msg}'.format(msg=e.message))
+ logger.exception(e)
+
+ if retval == 0:
+ self.status = self.success
+ else:
+ self.status = self.failed
+
+ return retval
diff --git a/switchdc/remote.py b/switchdc/remote.py
new file mode 100644
index 0000000..07c6f4e
--- /dev/null
+++ b/switchdc/remote.py
@@ -0,0 +1,73 @@
+import yaml
+
+from cumin.query import QueryBuilder
+from cumin.transport import Transport
+
+from switchdc.log import logger
+
+
+# Load cumin's configuration
+with open('/etc/cumin/config.yaml', 'r') as f:
+ cumin_config = yaml.safe_load(f)
+
+
+def run(query_string, mode, commands, success_threshold=1.0, batch_size=None,
batch_sleep=0):
+ """High level Cumin run of commands on hosts matching the query.
+
+ Arguments:
+ query_string -- the hosts selection query to use with Cumin's
configured backend
+ mode -- the Cumin's mode of execution. Accepted values: sync,
async
+ commands -- the list of commands to execute on the matching hosts
+ success_threshold -- the threshold to consider the execution still
successful. A float between 0.0 and 1.0.
+ [optional, default: 1.0]
+ batch_size -- the batch size to use in cumin. [optional, default:
None]
+ batch_sleep -- the batch sleep in seconds to use in Cumin before
scheduling the next host.
+ [optional, default: 0]
+ """
+ hosts = query(query_string)
+ return execute(hosts, mode, commands, success_threshold, batch_size,
batch_sleep)
+
+
+def query(query_string):
+ """Lower level Cumin's backend query to find matching hosts. Use run()
when possible.
+
+ Arguments:
+ query_string -- the hosts selection query to use with Cumin's configured
backend
+ """
+ query = QueryBuilder(query_string, cumin_config, logger).build()
+ return query.execute()
+
+
+def execute(hosts, mode, commands, success_threshold=1.0, batch_size=None,
batch_sleep=0):
+ """Lower level Cumin's execution of commands on a list o hosts. Use run()
when possible.
+
+ Arguments:
+ hosts -- the list of matching hosts to use as a target for
Cumin's transport
+ mode -- the Cumin's mode of execution. Accepted values: sync,
async
+ commands -- the list of commands to execute on the matching hosts
+ success_threshold -- the threshold to consider the execution still
successful. A float between 0.0 and 1.0.
+ [optional, default: 1.0]
+ batch_size -- the batch size to use in cumin. [optional, default:
None]
+ batch_sleep -- the batch sleep in seconds to use in Cumin before
scheduling the next host.
+ [optional, default: 0]
+ """
+ worker = Transport.new(cumin_config, logger)
+ worker.hosts = hosts
+ worker.commands = commands
+ worker.handler = mode
+ worker.success_threshold = success_threshold
+ worker.batch_size = batch_size
+ worker.batch_sleep = batch_sleep
+
+ rc = worker.execute()
+
+ return rc, worker
+
+
+def get_puppet_agent_command(noop=False):
+ """Return puppet agent command equivalent to --test without
--detailed-exitcodes."""
+ command = 'puppet agent -ov --ignorecache --no-daemonize
--no-usecacheonfailure --no-splay --show_diff'
+ if noop:
+ command += ' --noop'
+
+ return command
diff --git a/switchdc/stages/__init__.py b/switchdc/stages/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/switchdc/stages/__init__.py
diff --git a/switchdc/switch.py b/switchdc/switch.py
new file mode 100644
index 0000000..3fe8099
--- /dev/null
+++ b/switchdc/switch.py
@@ -0,0 +1,115 @@
+import argparse
+import glob
+import importlib
+import os
+
+from switchdc.menu import Item, Menu
+
+
+def run(menu, dc_from, dc_to):
+ """Run the swithcdc interactive menu.
+
+ Arguments:
+ menu -- the Menu instance to use for the run
+ dc_from -- the name of the datacenter to migrate from
+ dc_to -- the name of the datacenter to migrate to
+ """
+ while True:
+ print('#--- DATACENTER SWITCHOVER FROM {dc_from} TO {dc_to}
---#'.format(dc_from=dc_from, dc_to=dc_to))
+ menu.show()
+ try:
+ answer = raw_input('>>> ')
+ except (EOFError, KeyboardInterrupt):
+ print # Nicer output
+ break # Ctrl+d or Ctrl+c pressed while waiting for input
+
+ if not answer:
+ continue
+ elif answer == 'q':
+ break
+ elif answer == 'b':
+ menu = menu.parent
+
+ try:
+ index = int(answer)
+ except Exception:
+ print('Invalid answer')
+ continue
+
+ if index > 0 and index <= len(menu.items):
+ item = menu.items[index - 1]
+ if type(item) == Menu:
+ menu = item
+ elif type(item) == Item:
+ item.run()
+ else:
+ print('==> Invalid input <==')
+ continue
+
+
+def parse_args():
+ """Parse command line arguments and return them."""
+ parser = argparse.ArgumentParser(description='Datacenter Switchover for
Mediawiki')
+ parser.add_argument('-f', '--dc-from', required=True, help='Name of the
datacenter to migrate **from**')
+ parser.add_argument('-t', '--dc-to', required=True, help='Name of the
datacenter to migrate **to**')
+ parser.add_argument('--task', help='If specified, run this task only in an
non-interactive way and exit')
+ parser.add_argument(
+ '--stage', help='If specified, run all the tasks of this stage in an
non-interactive way and exit')
+
+ return parser.parse_args()
+
+
+def generate_menu(dc_from, dc_to):
+ """Automatically generate the menu with items and submenus based on the
available modules.
+
+ Arguments:
+ dc_from -- the name of the datacenter to migrate from
+ dc_to -- the name of the datacenter to migrate to
+ """
+ menu = Menu('Datacenter switchover automation')
+ stage = '00'
+ submenu = None
+
+ for module_file in sorted(glob.glob(os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), 'stages',
't[0-9][0-9]_*.py'))):
+
+ module_name = os.path.basename(module_file)[:-3]
+ module_stage = module_name[1:3]
+ module =
importlib.import_module('switchdc.stages.{module_name}'.format(module_name=module_name))
+
+ if module_stage != stage:
+ if submenu is not None:
+ menu.append(submenu)
+ stage = module_stage
+ submenu = Menu('Stage {stage}'.format(stage=stage))
+
+ submenu.append(Item(module.__name__, module.__title__, module.execute,
args=[dc_from, dc_to]))
+ else:
+ if submenu is not None:
+ menu.append(submenu)
+
+ return menu
+
+
+def main():
+ """Entry point, run the tool."""
+ args = parse_args()
+ menu = generate_menu(args.dc_from, args.dc_to)
+
+ if args.task is not None:
+ # Run a single task in non-interactive mode
+ for item in menu.items[int(args.task[1:3]) - 1].items:
+ if item.name.split('.')[-1] == args.task:
+ item.run()
+ break
+ elif args.stage is not None:
+ # Run all tasks in a stage in non-interactive mode
+ for item in menu.items[int(args.stage) - 1].items:
+ item.run()
+ else:
+ # Run the interactive menu
+ run(menu, args.dc_from, args.dc_to)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..f4d0f4c
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,3 @@
+[flake8]
+max-line-length=120
+statistics = True
--
To view, visit https://gerrit.wikimedia.org/r/342492
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I6c64268e5a86be13afaa6d8185a6b1533fe71770
Gerrit-PatchSet: 4
Gerrit-Project: operations/switchdc
Gerrit-Branch: master
Gerrit-Owner: Volans <[email protected]>
Gerrit-Reviewer: Faidon Liambotis <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: Volans <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
