Giuseppe Lavagetto has submitted this change and it was merged. Change subject: puppet-compiler: first commit ......................................................................
puppet-compiler: first commit This aims to become a full replacements for both current compilers. What will it offer: - a clean interface to submit jobs via jenkins - an understandable html output representing the results - a fully unit tested replacement for both compilers - a clean python code with no rot from endless hacks - hopefully, a solid and reasonably fast execution environment what it lacks right now - stored configs - A way to extract which hosts to run onto for changes to simple classes Bug: T96802 Change-Id: Id015fa2b0ff343f3dfd5ed28297f90539ce926ad --- A .gitignore A LICENSE A puppet_compiler/__init__.py A puppet_compiler/cli.py A puppet_compiler/controller.py A puppet_compiler/nodegen.py A puppet_compiler/prepare.py A puppet_compiler/presentation/__init__.py A puppet_compiler/presentation/html.py A puppet_compiler/puppet.py A puppet_compiler/templates/hostpage.jinja2 A puppet_compiler/templates/index.jinja2 A puppet_compiler/tests/fixtures/exit_diff A puppet_compiler/tests/fixtures/exit_nodiff A puppet_compiler/tests/fixtures/modules/puppetmaster/files/production.hiera.yaml A puppet_compiler/tests/fixtures/puppet_var/ssl/test/hello A puppet_compiler/tests/fixtures/test_config.yaml A puppet_compiler/tests/test_controller.py A puppet_compiler/tests/test_hostworker.py A puppet_compiler/tests/test_prepare.py A puppet_compiler/tests/test_puppet.py A puppet_compiler/threads.py A setup.py 23 files changed, 1,275 insertions(+), 0 deletions(-) Approvals: Giuseppe Lavagetto: Verified; Looks good to me, approved diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..77fd0d2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +Puppet compiler + +Copyright (c) 2015 Giuseppe Lavagetto +Copyright (c) 2015 Wikimedia Foundation Inc. +Except where explicitly stated in the file header. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Full license is available at https://www.gnu.org/licenses/gpl-2.0.txt diff --git a/puppet_compiler/__init__.py b/puppet_compiler/__init__.py new file mode 100644 index 0000000..61bb262 --- /dev/null +++ b/puppet_compiler/__init__.py @@ -0,0 +1,2 @@ +import logging +_log = logging.getLogger(__name__) diff --git a/puppet_compiler/cli.py b/puppet_compiler/cli.py new file mode 100644 index 0000000..b7cefc0 --- /dev/null +++ b/puppet_compiler/cli.py @@ -0,0 +1,50 @@ +import argparse +import logging +import os +import sys +import re +from puppet_compiler import _log, controller + +parser = argparse.ArgumentParser( + description="Puppet Compiler - allows to see differences in catalogs between revisions" +) +parser.add_argument('--debug', action='store_true', default=False, help="Print debug output") + +change = int(os.environ.get('CHANGE')) +nodes = os.environ.get('NODES', []) +job_id = int(os.environ.get('BUILD_NUMBER')) +configfile = os.environ.get('PC_CONFIG', '/etc/puppet-compiler.conf') +if nodes: + nodes = re.split('\s*,\s*', nodes) + + +def main(): + try: + opts = parser.parse_args() + if opts.debug: + lvl = logging.DEBUG + else: + lvl = logging.INFO + + logging.basicConfig( + format='%(asctime)s %(levelname)s: %(message)s', + level=lvl, + datefmt='[ %Y-%m-%dT%H:%M:%S ]' + ) + if not change: + _log.critical("No change provided, we cannot do anything") + sys.exit(2) + if not job_id: + _log.critical("No build number, are you running through jenkins?") + sys.exit(2) + + _log.info("Working on change %d", change) + c = controller.Controller(configfile, job_id, + change, nodes) + success = c.run() + # If the run is marked as failed, exit with a non-zero exit code + if not success: + sys.exit(1) + except Exception as e: + _log.critical("Build run failed: %s", e, exc_info=True) + sys.exit(1) diff --git a/puppet_compiler/controller.py b/puppet_compiler/controller.py new file mode 100644 index 0000000..d511ab6 --- /dev/null +++ b/puppet_compiler/controller.py @@ -0,0 +1,292 @@ +import os +import re +import subprocess +import shutil +import sys +import yaml +from puppet_compiler import prepare, _log, threads, puppet, nodegen +from puppet_compiler.presentation import html +""" +How data are organized: + +- each job has its base directory, e.g. '/tmp/differ/2456' +- inside this directory we have the proper puppet directories, 'production' + and 'change', and the 'diff' directory which - you guessed it - contains the + diffs. +- Also in the base, we have a output directory, that holds the final output for + the user (which includes the compiled catalog, the errors and warnings, and a + html page with some status and the diffs nicely represented, if any) + +""" + + +class Controller(object): + + def __init__(self, configfile, job_id, change_id, host_list=[]): + self.config = { + # Url under which results will be found + 'http_url': 'https://puppet-compiler.wmflabs.org/html', + # Base working directory of the compiler + 'base': '/mnt/jenkins-workspace', + # Location (either on disk, or at a remote HTTP location) + # of the operations/puppet repository + 'puppet_src': 'https://gerrit.wikimedia.org/r/operations/puppet', + # Location (either on disk, or at a remote HTTP location) + # of the labs/private repository + 'puppet_private': 'https://gerrit.wikimedia.org/r/labs/private', + # Directory hosting all of puppet's runtime files usually + # under /var/lib/puppet on debian-derivatives + 'puppet_var': '/var/lib/catalog-differ/puppet' + } + try: + if configfile is not None: + self._parse_conf(configfile) + except yaml.parser.ParserError as e: + _log.exception("Configuration file %s contains malformed yaml: %s", + configfile, e) + sys.exit(2) + except Exception: + _log.exception("Couldn't load the configuration from %s", configfile) + + self.count = 0 + if not host_list: + _log.info("No host list provided, generating the nodes list") + self.hosts = nodegen.get_nodes(self.config) + else: + self.hosts = host_list + + self.m = prepare.ManageCode(self.config, job_id, change_id) + self.outdir = os.path.join(self.config['base'], 'output', str(job_id)) + # State of all nodes + self.state = {'noop': set(), 'diff': set(), + 'err': set(), 'fail': set()} + # Set up variables to be used by the html output class + html.change_id = change_id + html.job_id = job_id + + def _parse_conf(self, configfile): + with open(configfile, 'r') as f: + data = yaml.load(f) + # TODO: add data validation here + self.config.update(data) + + def run(self): + _log.info("Refreshing the common repos from upstream if needed") + # If using local filesystem repositories, we need to refresh them + # before of a run. + if self.config['puppet_src'].startswith('/'): + _log.debug("refreshing %s", self.config['puppet_src']) + self.m.refresh(self.config['puppet_src']) + if self.config['puppet_private'].startswith('/'): + _log.debug("refreshing %s", self.config['puppet_private']) + self.m.refresh(self.config['puppet_private']) + + _log.info("Creating directories under %s", self.config['base']) + self.m.prepare() + + threadpool = threads.ThreadOrchestrator( + self.config.get('pool_size', 2)) + + # For each host, the threadpool will execute + # Controller._run_host with the host as the only argument. + # When this main payload is executed in a thread, the presentation + # work is executed in the main thread via + # Controller.on_node_compiled + for host in self.hosts: + h = HostWorker(self.m, host, self.outdir) + threadpool.add(h.run_host, hostname=host) + threadpool.fetch(self.on_node_compiled) + index = html.Index(self.outdir) + index.render(self.state) + _log.info('Run finished; see your results at %s/%s/', self.config['http_url'], html.job_id) + return self.success + + @property + def success(self): + """ + Try to determine if the build failed. + + Currently it just tries to see if more than half of the hosts are + marked "fail" + """ + if not self.count: + # We still didn't run + return True + f = len(self.state['fail']) + return (2*f < self.count) + + def on_node_compiled(self, payload): + """ + This callback is called in the main thread once one payload has been + completed in a worker thread. + """ + self.count += 1 + hostname = payload.kwargs['hostname'] + if payload.is_error: + # Running _run_host returned with an exception, this is unexpected + self.state['fail'].add(hostname) + _log.critical("Unexpected error running the payload: %s", + payload.value) + else: + self.state[payload.value].add(hostname) + + if not self.count % 5: + index = html.Index(self.outdir) + index.render(self.state) + _log.info('Index updated, you can see detailed progress for your work at %s', self.config['http_url']) + _log.info( + "Nodes: %s NOOP %s DIFF %s ERROR %s FAIL", + len(self.state['noop']), + len(self.state['diff']), + len(self.state['err']), + len(self.state['fail']) + ) + + +class HostWorker(object): + E_OK = 0 + E_PROD = 1 + E_CHANGE = 2 + + def __init__(self, manager, hostname, outdir): + self.m = manager + self.files = { + 'prod': { + 'catalog': os.path.join(self.m.prod_dir, 'catalogs', + hostname + '.pson'), + 'errors': os.path.join(self.m.prod_dir, 'catalogs', + hostname + '.err') + }, + 'change': { + 'catalog': os.path.join(self.m.change_dir, 'catalogs', + hostname + '.pson'), + 'errors': os.path.join(self.m.change_dir, 'catalogs', + hostname + '.err') + } + } + self.hostname = hostname + self.outdir = os.path.join(outdir, self.hostname) + + def run_host(self, *args, **kwdargs): + """ + Compiles and diffs an host. Gets delegated to a worker thread + """ + if not self.m.find_yaml(self.hostname): + _log.error('Unable to find facts for host %s, skipping', + self.hostname) + return 'fail' + errors = self._compile_all() + retcode = self._make_diff(errors) + try: + self._make_output() + self._build_html(retcode) + except Exception as e: + _log.error('Error preparing output for %s: %s', self.hostname, e, + exc_info=True) + return retcode + + def _compile_all(self): + """ + Does the grindwork of compiling the catalogs + """ + errors = self.E_OK + try: + _log.info("Compiling host %s (production)", self.hostname) + puppet.compile(self.hostname, + self.m.prod_dir, + self.m.puppet_var) + except subprocess.CalledProcessError as e: + _log.error("Compilation failed for hostname %s " + " with the current tree.", self.hostname) + _log.info("Compilation exited with code %d", e.returncode) + _log.debug("Failed command: %s", e.cmd) + errors += self.E_PROD + try: + _log.info("Compiling host %s (change)", self.hostname) + puppet.compile(self.hostname, + self.m.change_dir, + self.m.puppet_var) + except subprocess.CalledProcessError as e: + _log.error("Compilation failed for hostname %s " + " with the change.", self.hostname) + _log.info("Compilation exited with code %d", e.returncode) + _log.debug("Failed command: %s", e.cmd) + errors += self.E_CHANGE + return errors + + def _make_diff(self, errors): + """ + Based on the outcome of compilation, will or will not produce diffs + """ + if errors == self.E_OK: + # Both nodes compiled correctly + _log.info("Calculating diffs for %s",self.hostname) + try: + puppet.diff(self.m.base_dir, self.hostname) + except subprocess.CalledProcessError as e: + _log.error("Diffing the catalogs failed: %s", self.hostname) + _log.info("Diffing exited with code %d", e.returncode) + _log.debug("Failed command: %s", e.cmd) + return 'fail' + else: + if self._get_diff(): + return 'diff' + else: + return 'noop' + elif errors == self.E_PROD: + # Production didn't compile, the changed tree did. + # Declare this a success + return 'noop' + elif errors == self.E_CHANGE: + # Ouch, the change didn't work + return 'err' + else: + # This is most probably the compiler's fault, + # let's call this a fail + return 'fail' + + def _make_output(self): + """ + Prepare the node output, copying the relevant files in place + in the output directory + """ + os.makedirs(self.outdir, 0755) + for env in self.files: + for label in 'catalog', 'errors': + filename = self.files[env][label] + if os.path.isfile(filename): + name = os.path.basename(filename) + newname = os.path.join(self.outdir, env + '.' + name) + shutil.copy(filename, newname) + + diff = self._get_diff() + if diff: + shutil.copy(diff, self.outdir) + return True + + return False + + def _get_diff(self): + """ + Get diffs name if the file exists + """ + diff = os.path.join(self.m.diff_dir, self.hostname + '.diff') + if os.path.isfile(diff) and os.path.getsize(diff) > 0: + with open(diff, 'r') as f: + for line in f: + # If no changes were detected, puppet catalog diff outputs + # a line with 'fqdn 0.0%' + # we use that to detect a noop change since we have no other + # means to detect that. + if re.match('^{}\s+0.0\%'.format(self.hostname), line): + return False + return diff + return False + + def _build_html(self, retcode): + """ + build the HTML output + """ + # TODO: implement the actual html parsing + host = html.Host(self.hostname, self.m.diff_dir, self.outdir, retcode) + host.htmlpage(self.files) diff --git a/puppet_compiler/nodegen.py b/puppet_compiler/nodegen.py new file mode 100644 index 0000000..7e0c2f8 --- /dev/null +++ b/puppet_compiler/nodegen.py @@ -0,0 +1,66 @@ +import re +import os +from puppet_compiler import _log + + +def get_nodes(config): + """ + Get all the available nodes that have separate declarations in site.pp + """ + facts_dir = os.path.join(config['puppet_var'], 'yaml', 'facts') + _log.info("Walking dir %s", facts_dir) + site_pp = os.path.join(config['puppet_src'], 'manifests', 'site.pp') + # Read site.pp + with open(site_pp, 'r') as sitepp: + n = NodeFinder(sitepp) + n.match_physical_nodes(nodelist(facts_dir)) + + +def nodelist(facts_dir): + for subdir in os.walk(facts_dir): + for node in subdir[2]: + yield node.replace('.yaml', '') + + + +class NodeFinder(object): + regexp_node = re.compile('^node\s+/([^/]+)/') + exact_node = re.compile("node\s*\'([^\']+)\'") + + def __init__(self, sitepp): + self.regexes = set() + self.nodes = set() + for line in sitepp.readlines(): + m = self.regexp_node.search(line) + if m: + _log.debug('Found regex in line %s', line.rstrip()) + self.regexes.add(re.compile(m.group(1))) + continue + m = self.exact_node.search(line) + if m: + _log.debug('Found node in line %s', line.rstrip()) + self.nodes.add(m.group(1)) + + def match_physical_nodes(self, nodelist): + nodes = [] + for node in nodelist: + discarded = None + if node in self.nodes: + _log.debug('Found node %s', node) + nodes.append(node) + self.nodes.discard(node) + continue + for regex in self.regexes: + # TODO: this may be very slow, should calculate this + m = regex.search(node) + if m: + _log.debug('Found match for node %s: %s', node, + regex.pattern) + nodes.append(node) + discarded=regex + continue + + if discarded is not None: + self.regexes.discard(discarded) + + return nodes diff --git a/puppet_compiler/prepare.py b/puppet_compiler/prepare.py new file mode 100644 index 0000000..9d8f414 --- /dev/null +++ b/puppet_compiler/prepare.py @@ -0,0 +1,170 @@ +from contextlib import contextmanager +import json +import subprocess +import os +import shutil +import requests +from puppet_compiler import _log + + +@contextmanager +def pushd(dirname): + cur_dir = os.getcwd() + os.chdir(dirname) + yield + os.chdir(cur_dir) + + +class ManageCode(object): + private_modules = ['passwords', 'contacts', 'privateexim'] + + def __init__(self, config, jobid, changeid): + self.base_dir = os.path.join(config['base'], + str(jobid)) + self.puppet_src = config['puppet_src'] + self.puppet_private = config['puppet_private'] + self.puppet_var = config['puppet_var'] + self.change_id = changeid + self.change_dir = os.path.join(self.base_dir, 'change') + self.prod_dir = os.path.join(self.base_dir, 'production') + self.diff_dir = os.path.join(self.base_dir, 'diffs') + self.output_dir = os.path.join(config['base'], 'output', str(jobid)) + self.git = Git() + + def find_yaml(self, hostname): + """ Finds facts file for the current hostname """ + facts_file = os.path.join(self.puppet_var, 'yaml', 'facts', + '{}.yaml'.format(hostname)) + if os.path.isfile(facts_file): + return facts_file + return None + + def cleanup(self): + """ + Remove the whole change tree. + """ + shutil.rmtree(self.base_dir, True) + + def prepare(self): + _log.debug("Creating directories under %s", self.base_dir) + # Create the base directory now + os.mkdir(self.base_dir, 0755) + for dirname in [self.prod_dir, self.change_dir]: + os.makedirs(os.path.join(dirname, 'catalogs'), 0755) + os.makedirs(self.diff_dir, 0755) + os.makedirs(self.output_dir, 0755) + + # Production + self._prepare_dir(self.prod_dir) + prod_src = os.path.join(self.prod_dir, 'src') + with pushd(prod_src): + self._copy_hiera(self.prod_dir) + + # Change + self._prepare_dir(self.change_dir) + change_src = os.path.join(self.change_dir, 'src') + with pushd(change_src): + self._fetch_change() + # Re-do in case of hiera config changes + self._copy_hiera(self.change_dir) + + def refresh(self, gitdir): + """ + Refresh a git repository + """ + with pushd(gitdir): + self.git.pull('-q', '--rebase') + + # Private methods + def _prepare_dir(self, dirname): + """ + prepare a specific directory to compile puppet + """ + _log.debug("Cloning directories...") + src = os.path.join(dirname, 'src') + self.git.clone('-q', self.puppet_src, src) + priv = os.path.join(dirname, 'private') + self.git.clone('-q', self.puppet_private, priv) + with pushd(src): + self.git.submodule('-q', 'init') + self.git.submodule('-q', 'update') + + _log.debug('Adding symlinks') + for module in self.private_modules: + source = os.path.join(priv, 'modules', module) + dst = os.path.join(src, 'modules', module) + os.symlink(source, dst) + + shutil.copytree(os.path.join(self.puppet_var, 'ssl'), + os.path.join(src, 'ssl')) + + @staticmethod + def _copy_hiera(dirname): + """ + Copy the hiera file + """ + hiera_file = 'modules/puppetmaster/files/production.hiera.yaml' + priv = os.path.join(dirname, 'private') + pub = os.path.join(dirname, 'src') + with open(hiera_file, 'r') as g, open('hiera.yaml', 'w') as f: + for line in g: + l = line.replace( + '/etc/puppet/private', priv + ).replace( + '/etc/puppet', pub) + f.write(l) + + def _fetch_change(self): + """get changes from the change directly""" + git = Git() + headers = {'Accept': 'application/json', + 'Content-Type': 'application/json; charset=UTF-8'} + change = requests.get( + 'https://gerrit.wikimedia.org/r/changes/%d?o=CURRENT_REVISION' % + self.change_id, headers=headers) + change.raise_for_status() + + # Workaround the broken gerrit response... + json_data = change.text.split("\n")[-2:][0] + res = json.loads(json_data) + revision = res["revisions"].values()[0]["_number"] + ref = 'refs/changes/%02d/%d/%d' % ( + self.change_id % 100, + self.change_id, + revision) + _log.debug( + 'Downloading patch for change %d, revision %d', + self.change_id, revision) + git.fetch('-q', 'https://gerrit.wikimedia.org/r/operations/puppet', + ref) + git.cherry_pick('FETCH_HEAD') + + def _sh(command): + try: + subprocess.check_call(command, shell=True) + except subprocess.CalledProcessError as e: + _log.error("Command '%s' failed with exit code '%s'", + e.cmd, e.returncode) + raise + + +class Git(): + ''' + This class is not strictly needed. It's just a container for the member + functions, so that they are not in the global namespace. There is no point + in instantiating it ever. + + Partly salvaged from utils/new_wmf_service + ''' + + def __getattr__(self, action): + action = action.replace('_', '-') + + def git_exec(*args, **kwdargs): + return self._execute_command(action, *args) + return git_exec + + def _execute_command(self, command, *args): + cmd = ['git', command] + cmd.extend(args) + return subprocess.call(cmd) diff --git a/puppet_compiler/presentation/__init__.py b/puppet_compiler/presentation/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/puppet_compiler/presentation/__init__.py diff --git a/puppet_compiler/presentation/html.py b/puppet_compiler/presentation/html.py new file mode 100644 index 0000000..e356e5c --- /dev/null +++ b/puppet_compiler/presentation/html.py @@ -0,0 +1,54 @@ +import os +from jinja2 import Environment, PackageLoader +from puppet_compiler import _log + +env = Environment(loader=PackageLoader('puppet_compiler', 'templates')) +change_id = None +job_id = None + + +class Host(object): + + def __init__(self, hostname, diff_dir, outdir, retcode): + self.retcode = retcode + self.hostname = hostname + self.outdir = outdir + self.diff_file = os.path.join(diff_dir, hostname + '.diff') + + def htmlpage(self, files): + """ + Create the html page + """ + _log.debug("Rendering index page for %s", self.hostname) + data = {'retcode': self.retcode} + if self.retcode == 'diff': + with open(self.diff_file, 'r') as f: + data['diffs'] = f.read() + data['files'] = files + if self.retcode == 'noop': + data['desc'] = 'no change' + elif self.retcode == 'diff': + data['desc'] = 'changes detected' + elif self.retcode == 'err': + data['desc'] = 'change fails' + else: + data['desc'] = 'compiler failure' + t = env.get_template('hostpage.jinja2') + page = t.render(host=self.hostname, jid=job_id, chid=change_id, **data) + with open(os.path.join(self.outdir, 'index.html'), 'w') as f: + f.write(page) + + +class Index(object): + def __init__(self, outdir): + self.outfile = os.path.join(outdir, 'index.html') + + def render(self, state): + """ + Render the index page with info coming from state + """ + _log.debug("Rendering the main index page") + t = env.get_template('index.jinja2') + page = t.render(state=state, jid=job_id, chid=change_id) + with open(self.outfile, 'w') as f: + f.write(page) diff --git a/puppet_compiler/puppet.py b/puppet_compiler/puppet.py new file mode 100644 index 0000000..54ecd15 --- /dev/null +++ b/puppet_compiler/puppet.py @@ -0,0 +1,62 @@ +import os +import re +import subprocess +from tempfile import SpooledTemporaryFile as spoolfile +from puppet_compiler import _log + + +def compile(hostname, basedir, vardir): + """ + Compile the catalog + """ + env = os.environ.copy() + srcdir = os.path.join(basedir, 'src') + privdir = os.path.join(basedir, 'private') + env['RUBYLIB'] = os.path.join(srcdir, 'modules/wmflib/lib/') + + catalogdir = os.path.join(basedir, 'catalogs') + tpldir = os.path.join(srcdir, 'templates') + cmd = ['puppet', 'master', + '--vardir=%s' % vardir, + '--modulepath=%s:%s' % (os.path.join(privdir, 'modules'), + os.path.join(srcdir, 'modules')), + '--confdir=%s' % srcdir, + '--templatedir=%s' % tpldir, + '--compile=%s' % hostname, + '--color=false' + ] + hostfile = os.path.join(catalogdir, hostname) + + with open(hostfile + ".err", 'w') as err: + out = spoolfile() + subprocess.check_call(cmd, stdout=out, stderr=err, env=env) + + # Puppet outputs a lot of garbage to stdout... + with open(hostfile + ".pson", "w") as f: + out.seek(0) + for line in out: + if not re.match('(Info|[Nn]otice|[Ww]arning)', line): + f.write(line) + + +def diff(basedir, hostname): + """ + Compute the diffs between the two changes + """ + prod_catalog = os.path.join(basedir, + 'production', + 'catalogs', + hostname + '.pson') + change_catalog = os.path.join(basedir, + 'change', + 'catalogs', + hostname + '.pson') + output = os.path.join(basedir, 'diffs', hostname + '.diff') + cmd = [ 'puppet', 'catalog', 'diff', '--show_resource_diff', + '--content_diff', prod_catalog, change_catalog] + temp = spoolfile() + subprocess.check_call(cmd, stdout=temp) + with open(output, 'w') as out: + temp.seek(0) + # Remove bold from term output + out.write(temp.read().replace('\x1b[1m', '').replace('\x1b[0m', '')) diff --git a/puppet_compiler/templates/hostpage.jinja2 b/puppet_compiler/templates/hostpage.jinja2 new file mode 100644 index 0000000..8e28ac9 --- /dev/null +++ b/puppet_compiler/templates/hostpage.jinja2 @@ -0,0 +1,49 @@ +<!doctype html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>Compilation results for {{ host }}</title> + <style type="text/css"> + h1, h2, h3, h4 { + font-family: "HelveticaNeue-CondensedBold","Helvetica Neue","Arial Narrow",Arial,sans-serif; + } + .source { + font-family: Consolas, "Andale Mono", "Courier New", monospace + } + .err .fail { + color: red; + } + .noop { + color: darkgreen; + } + .diff { + color: darkgoldenrod; + } + </style> + </head> + <body> + <div id="main"> + {% block body %} + <h1>Compilation results for {{ host }}: <span class="{{retcode}}">{{ desc }}</span></h1> + {% if retcode == "diff" %} + <h2>Catalog differences</h2> + <textarea class="source">{{ diffs }}</textarea> + {% endif %} + <h2>Relevant files</h2> + <ul> + <li><a href="prod.{{ host }}.pson">Production catalog</a></li> + <li><a href="change.{{ host }}.pson">Change catalog</a></li> + <li><a href="prod.{{ host }}.err">Production errors/warnings</a></li> + <li><a href="change.{{ host }}.err">Change errors/warnings</a></li> + </ul> + {% endblock %} + {% block extnav %} + <p> + <a href="https://integration.wikimedia.org/ci/job/operations-puppet-catalog-compiler/{{ jid }}/" target="_blank">Back to jenkins job</a> + <a href="https://gerrit.wikimedia.org/r/#/c/{{ chid }}/" target="_blank">See gerrit change</a> + </p> + {% endblock %} + </div> + </body> +</html> diff --git a/puppet_compiler/templates/index.jinja2 b/puppet_compiler/templates/index.jinja2 new file mode 100644 index 0000000..9c6b35c --- /dev/null +++ b/puppet_compiler/templates/index.jinja2 @@ -0,0 +1,30 @@ +{% extends "hostpage.jinja2" %} +{% block body %} + <h1>Results for change <a href="https://gerrit.wikimedia.org/r/#/c/{{ ch_id }}/" target="_blank">{{ chid }}</a></h1> + <div id="list"> + <h2>Hosts that have no differences (or compile correctly only with the change)</h2> + <ul> + {% for host in state.noop %} + <li> <a href="{{ host }}/">{{ host }}</a> + {% endfor %} + </ul> + <h2>Hosts that have changes</h2> + <ul> + {% for host in state.diff %} + <li> <a href="{{ host }}/">{{ host }}</a> + {% endfor %} + </ul> + <h2>Hosts that fail to compile when the change is applied</h2> + <ul> + {% for host in state.err %} + <li> <a href="{{ host }}/">{{ host }}</a> + {% endfor %} + </ul> + <h2>Hosts that have failed to compile completely</h2> + <ul> + {% for host in state.fail %} + <li> <a href="{{ host }}/">{{ host }}</a> + {% endfor %} + </ul> + </div> +{% endblock %} diff --git a/puppet_compiler/tests/fixtures/exit_diff b/puppet_compiler/tests/fixtures/exit_diff new file mode 100644 index 0000000..e53eaa1 --- /dev/null +++ b/puppet_compiler/tests/fixtures/exit_diff @@ -0,0 +1,10 @@ +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 diff --git a/puppet_compiler/tests/fixtures/exit_nodiff b/puppet_compiler/tests/fixtures/exit_nodiff new file mode 100644 index 0000000..c3dd090 --- /dev/null +++ b/puppet_compiler/tests/fixtures/exit_nodiff @@ -0,0 +1,9 @@ +1 +2 +3 +4 +test.codfw.wmnet 0.0% +6 +7 +8 +9 diff --git a/puppet_compiler/tests/fixtures/modules/puppetmaster/files/production.hiera.yaml b/puppet_compiler/tests/fixtures/modules/puppetmaster/files/production.hiera.yaml new file mode 100644 index 0000000..828dfcd --- /dev/null +++ b/puppet_compiler/tests/fixtures/modules/puppetmaster/files/production.hiera.yaml @@ -0,0 +1,2 @@ +test1 = "/etc/puppet/hieradata" +test2 = "/etc/puppet/private/hieradata" diff --git a/puppet_compiler/tests/fixtures/puppet_var/ssl/test/hello b/puppet_compiler/tests/fixtures/puppet_var/ssl/test/hello new file mode 100644 index 0000000..cc628cc --- /dev/null +++ b/puppet_compiler/tests/fixtures/puppet_var/ssl/test/hello @@ -0,0 +1 @@ +world diff --git a/puppet_compiler/tests/fixtures/test_config.yaml b/puppet_compiler/tests/fixtures/test_config.yaml new file mode 100644 index 0000000..9694a9f --- /dev/null +++ b/puppet_compiler/tests/fixtures/test_config.yaml @@ -0,0 +1,4 @@ +http_url: http://www.example.com/garbagehere +test_non_existent: + - 'definitely' + - 'maybe' diff --git a/puppet_compiler/tests/test_controller.py b/puppet_compiler/tests/test_controller.py new file mode 100644 index 0000000..e9765e1 --- /dev/null +++ b/puppet_compiler/tests/test_controller.py @@ -0,0 +1,55 @@ +import unittest +import mock +import os +from puppet_compiler import controller, threads + + +class TestController(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.fixtures = os.path.join(os.path.dirname(__file__), 'fixtures') + + def test_initialize_no_configfile(self): + c = controller.Controller(None, 19, 224570, ['test.eqiad.wmnet']) + self.assertEquals(c.hosts, ['test.eqiad.wmnet']) + self.assertEquals(c.config['http_url'], + 'https://puppet-compiler.wmflabs.org/html') + self.assertEquals(c.config['base'], '/mnt/jenkins-workspace') + + def test_parse_config(self): + filename = os.path.join(self.fixtures, 'test_config.yaml') + c = controller.Controller(filename, 19, 224570, ['test.eqiad.wmnet']) + self.assertEquals(len(c.config['test_non_existent']),2) + self.assertEquals(c.config['http_url'], + 'http://www.example.com/garbagehere') + + @mock.patch('puppet_compiler.presentation.html.Index') + @mock.patch('puppet_compiler.controller.HostWorker.run_host') + def test_run_single_host(self, mocker, html_mocker): + c = controller.Controller(None, 19, 224570, ['test.eqiad.wmnet']) + mocker.return_value = 'fail' + c.m.prepare = mock.MagicMock() + c.m.prepare.return_value = True + c.m.refresh = mock.MagicMock() + c.m.refresh.return_value = True + + c.run() + c.m.prepare.assert_called_once_with() + c.m.refresh.assert_not_called() + self.assertEquals(c.state['fail'], set(['test.eqiad.wmnet'])) + + def test_node_callback(self): + c = controller.Controller(None, 19, 224570, ['test.eqiad.wmnet']) + response = threads.Msg(is_error=True, value='Something to remember', + args=None, + kwargs={'hostname': 'test.eqiad.wmnet'}) + c.count = 5 + c.on_node_compiled(response) + self.assertIn('test.eqiad.wmnet', c.state['fail']) + self.assertEquals(c.count, 6) + response = threads.Msg(is_error=False, value='noop', args=None, + kwargs={'hostname': 'test2.eqiad.wmnet'}) + c.on_node_compiled(response) + self.assertIn('test2.eqiad.wmnet', c.state['noop']) + self.assertEquals(c.count, 7) diff --git a/puppet_compiler/tests/test_hostworker.py b/puppet_compiler/tests/test_hostworker.py new file mode 100644 index 0000000..b8056a0 --- /dev/null +++ b/puppet_compiler/tests/test_hostworker.py @@ -0,0 +1,122 @@ +import unittest +import mock +import os +from puppet_compiler import controller +import subprocess + + +class TestHostWorker(unittest.TestCase): + + def setUp(self): + self.fixtures = os.path.join(os.path.dirname(__file__), 'fixtures') + self.c = controller.Controller(None, 19, 224570, ['test.eqiad.wmnet']) + self.m = self.c.m + self.hw = hw = controller.HostWorker(self.m, + 'test.codfw.wmnet', '/what/you/want') + + def test_initialize(self): + + self.assertEquals(self.hw.hostname, 'test.codfw.wmnet') + self.assertItemsEqual(['prod', 'change'], self.hw.files.keys()) + self.assertEquals( + '/mnt/jenkins-workspace/19/change/catalogs/test.codfw.wmnet.err', + self.hw.files['change']['errors'] + ) + self.assertEquals( + '/mnt/jenkins-workspace/19/production/catalogs/test.codfw.wmnet.pson', + self.hw.files['prod']['catalog'] + ) + self.assertEquals('/what/you/want/test.codfw.wmnet', + self.hw.outdir) + + @mock.patch('puppet_compiler.puppet.compile') + def test_compile_all(self, mocker): + # Verify simple calls + err = self.hw._compile_all() + calls = [ + mock.call('test.codfw.wmnet', '/mnt/jenkins-workspace/19/production', + '/var/lib/catalog-differ/puppet'), + mock.call('test.codfw.wmnet', '/mnt/jenkins-workspace/19/change', + '/var/lib/catalog-differ/puppet'), + ] + mocker.assert_has_calls(calls) + self.assertEquals(err, 0) + + # Verify all compilation is wrong + mocker.reset_mock() + mocker.side_effect = subprocess.CalledProcessError(cmd="ehehe", + returncode=30) + err = self.hw._compile_all() + self.assertEquals(err, 3) + + # Verify only the change is wrong + def complicated_side_effect(*args, **kwdargs): + if '/mnt/jenkins-workspace/19/production' in args: + return True + else: + raise subprocess.CalledProcessError(cmd="ehehe", returncode=30) + mocker.reset_mock() + mocker.side_effect = complicated_side_effect + err = self.hw._compile_all() + self.assertEquals(err, 2) + + @mock.patch('puppet_compiler.puppet.diff') + def test_make_diff(self, mocker): + mocker.return_value = True + self.hw._get_diff = mock.Mock(return_value=True) + retval = self.hw._make_diff(0) + mocker.assert_called_with('/mnt/jenkins-workspace/19', 'test.codfw.wmnet') + self.assertEquals(retval, 'diff') + self.hw._get_diff = mock.Mock(return_value=False) + self.assertEquals(self.hw._make_diff(0), 'noop') + mocker.reset_mock() + self.assertEquals(self.hw._make_diff(1), 'noop') + assert not mocker.called + mocker.reset_mock() + self.assertEquals(self.hw._make_diff(2), 'err') + assert not mocker.called + mocker.side_effect = subprocess.CalledProcessError(cmd="ehehe", returncode=30) + self.assertEquals(self.hw._make_diff(0), 'fail') + self.assertEquals(self.hw._make_diff(3), 'fail') + + @mock.patch('os.path.isfile') + @mock.patch('os.makedirs') + @mock.patch('shutil.copy') + def test_make_output(self, mock_copy, mock_makedirs, mock_isfile): + mock_isfile.return_value = False + self.hw._make_output() + mock_makedirs.assert_called_with('/what/you/want/test.codfw.wmnet', 0755) + assert not mock_copy.called + mock_isfile.return_value = True + mock_copy.assert_called_any( + '/mnt/jenkins-workspace/19/production/catalogs/test.codfw.wmnet.pson', + '/what/you/want/test.codfw.wmnet/production.test.codfw.wmnet.pson', + ) + + @mock.patch('os.path') + def test_get_diff(self, mock_path): + mock_path.join.return_value = self.fixtures + '/' + 'exit_nodiff' + mock_path.isfile.return_value = True + mock_path.getsize.return_value = 100 + self.assertEquals(self.hw._get_diff(), False) + filename = self.fixtures + '/' + 'exit_diff' + mock_path.join.return_value = filename + self.assertEquals(self.hw._get_diff(), filename) + + def test_build_html(self): + pass + + def test_run_host(self): + self.hw.m.find_yaml = mock.Mock(return_value=False) + self.assertEquals(self.hw.run_host(), 'fail') + self.hw.m.find_yaml = mock.Mock(return_value=True) + self.hw._compile_all = mock.Mock(return_value=0) + self.hw._make_diff = mock.Mock(return_value='diff') + self.hw._make_output = mock.Mock(return_value=None) + self.hw._build_html = mock.Mock(return_value=None) + self.assertEquals(self.hw.run_host(), 'diff') + assert self.hw.m.find_yaml.called + assert self.hw._compile_all.called + self.hw._make_diff.assert_called_with(0) + assert self.hw._make_output.called + assert self.hw._build_html.called diff --git a/puppet_compiler/tests/test_prepare.py b/puppet_compiler/tests/test_prepare.py new file mode 100644 index 0000000..af8a13c --- /dev/null +++ b/puppet_compiler/tests/test_prepare.py @@ -0,0 +1,90 @@ +import mock +import os +import unittest +import tempfile +import shutil +from puppet_compiler import prepare + + +class TestGit(unittest.TestCase): + + def setUp(self): + self.git = prepare.Git() + + @mock.patch('subprocess.call') + def test_call_no_args(self, mocker): + """Init a git repository""" + self.git.init() + mocker.assert_called_with(['git', 'init']) + + @mock.patch('subprocess.call') + def test_call_with_args(self, mocker): + self.git.clone('-q', '/src/orig', '/src/dest') + mocker.assert_called_with(['git', 'clone', '-q', + '/src/orig', '/src/dest']) + + +class TestManageCode(unittest.TestCase): + """ + Tests the creation of the new git trees + """ + + @classmethod + def setUpClass(cls): + cls.base = tempfile.mkdtemp(prefix='puppet-compiler') + + def setUp(self): + fixtures = os.path.join(os.path.dirname(__file__), 'fixtures', 'puppet_var') + self.m = prepare.ManageCode( + {'base': self.base, + 'puppet_src': 'https://gerrit.wikimedia.org/r/operations/puppet', + 'puppet_private': 'https://gerrit.wikimedia.org/r/labs/private', + 'puppet_var': fixtures}, + 19, + 227450) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.base) + + def test_copy_hiera(self): + """Check the hiera file gets copied""" + with prepare.pushd(os.path.join(os.path.dirname(__file__), 'fixtures')): + self.m._copy_hiera(self.base) + with open('hiera.yaml') as f: + data = f.readlines() + os.unlink('hiera.yaml') + self.assertIn(os.path.join(self.base, 'src', 'hieradata'), data[0]) + self.assertIn(os.path.join(self.base, 'private'), data[1]) + + @mock.patch('subprocess.call') + def test_fetch_change(self, mocker): + """The change can be downloaded""" + self.m._fetch_change() + calls = [ + mock.call(['git', 'fetch', '-q', 'https://gerrit.wikimedia.org/r/operations/puppet', 'refs/changes/50/227450/1']), + mock.call(['git', 'cherry-pick', 'FETCH_HEAD']) + ] + mocker.assert_has_calls(calls) + + @mock.patch('os.symlink') + @mock.patch('shutil.copytree') + def test_prepare_dir(self, mock_copy, mock_symlink): + """Changes get properly prepared""" + # pushd support + os.makedirs(os.path.join(self.base, '19', 'production', 'src')) + self.m.git = mock.MagicMock() + self.m._prepare_dir(self.m.prod_dir) + prod_src = os.path.join(self.m.prod_dir, 'src') + self.m.git.clone.assert_any_call( + '-q', + 'https://gerrit.wikimedia.org/r/operations/puppet', + prod_src) + assert 2 == self.m.git.clone.call_count + assert 2 == self.m.git.submodule.call_count + mock_copy.assert_called_with(self.m.puppet_var + '/ssl', + prod_src + '/ssl') + assert 3 == mock_symlink.call_count + exim_priv = os.path.join(self.m.prod_dir, 'private/modules/privateexim') + exim_pub = os.path.join(self.m.prod_dir, 'src/modules/privateexim') + mock_symlink.assert_any_call(exim_priv, exim_pub) diff --git a/puppet_compiler/tests/test_puppet.py b/puppet_compiler/tests/test_puppet.py new file mode 100644 index 0000000..56688f1 --- /dev/null +++ b/puppet_compiler/tests/test_puppet.py @@ -0,0 +1,58 @@ +import unittest +import mock +import subprocess +import os +from puppet_compiler import puppet + + +class TestPuppetCalls(unittest.TestCase): + + def setUp(self): + subprocess.check_call = mock.Mock() + self.fixtures = os.path.join(os.path.dirname(__file__), 'fixtures') + + @mock.patch('puppet_compiler.puppet.spoolfile') + def test_compile(self, tf_mocker): + os.environ.copy = mock.Mock(return_value={'myenv': 'ishere!'}) + env = os.environ.copy() + env['RUBYLIB'] = self.fixtures + '/src/modules/wmflib/lib/' + m = mock.mock_open(read_data='wat') + with mock.patch('__builtin__.open', m, True) as mocker: + puppet.compile('test.codfw.wmnet', self.fixtures, self.fixtures + '/puppet_var') + subprocess.check_call.assert_called_with( + ['puppet', + 'master', + '--vardir=%s' % self.fixtures + '/puppet_var', + '--modulepath=%(basedir)s/private/modules:' + '%(basedir)s/src/modules' % {'basedir': self.fixtures}, + '--confdir=%s/%s' % (self.fixtures, 'src'), + '--templatedir=%s/%s' % (self.fixtures, 'src/templates'), + '--compile=test.codfw.wmnet', + '--color=false'], + env=env, + stdout=tf_mocker.return_value, + stderr=mocker.return_value + ) + hostfile = os.path.join(self.fixtures, 'catalogs', 'test.codfw.wmnet') + calls = [ + mock.call(hostfile + '.pson', 'w'), + mock.call(hostfile + '.err', 'w'), + ] + mocker.assert_has_calls(calls, any_order=True) + + @mock.patch('puppet_compiler.puppet.spoolfile') + def test_diff(self, tf_mocker): + m = mock.mock_open(read_data='wat') + with mock.patch('__builtin__.open', m, True) as mocker: + puppet.diff(self.fixtures, 'test.codfw.wmnet') + mocker.assert_called_with(self.fixtures + '/diffs/test.codfw.wmnet.diff', 'w') + subprocess.check_call.called_with( + ['puppet', + 'catalog', + 'diff', + '--show_resource_diff', + '--content_diff', + self.fixtures + '/production/catalogs/test.codfw.wmnet.json', + self.fixtures + '/change/catalogs/test.codfw.wmnet.json'], + stdout=tf_mocker.return_value + ) diff --git a/puppet_compiler/threads.py b/puppet_compiler/threads.py new file mode 100644 index 0000000..17c7228 --- /dev/null +++ b/puppet_compiler/threads.py @@ -0,0 +1,98 @@ +import threading +import logging +import time +from collections import namedtuple +from puppet_compiler import _log +try: + # python 2.x + import Queue as queue + # Work around for pyflakes < 0.6. + # TODO: Remove when pyflakes has been updated. + queue +except ImportError: + # python 3.x + import queue + +Msg = namedtuple('Msg', ['is_error', 'value', 'args', 'kwargs']) + + +class ThreadExecutor(threading.Thread): + """ + Manages execution of payloads coming from the ThreadOrchestrator + """ + + def __init__(self, queue, out_queue): + super(ThreadExecutor, self).__init__() + self.queue = queue + self.out_queue = out_queue + + def run(self): + _log.debug('Spawning a Thread executor') + while True: + # grab data from queue + (payload, args, kwargs) = self.queue.get() + if payload == '__exit__': + _log.debug('Stopping Thread') + return + _log.debug('Executing payload %s', payload) + try: + retval = payload(*args, **kwdargs) + msg = Msg(is_error=False, value=retval, args=args, kwargs=kwargs) + _log.debug(msg) + self.out_queue.put(msg) + except Exception as e: + # TODO: log correctly + _log.error("Error in payload") + _log.debug(str(e)) + msg = Msg(is_error=True, value=e, args=args, kwargs=kwargs) + self.out_queue.put(msg) + finally: + _log.debug('Execution terminated') + self.queue.task_done() + + +class ThreadOrchestrator(object): + + def __init__(self, pool_size=4): + self.pool_size = pool_size + self._TP = [] + self._payload_queue = queue.Queue() + self._incoming_queue = queue.Queue() + for i in xrange(self.pool_size): + t = ThreadExecutor(self._payload_queue, self._incoming_queue) + # this thread will exit with the main program + self._TP.append(t) + t.start() + + def add(self, payload, *args, **kwdargs): + self._payload_queue.put((payload, args, kwdargs)) + + def _process_result(self, callback): + """ + Execute the callback, with the threads.Msg received from the + executor as an argument. + """ + res = self._incoming_queue.get(True) + try: + callback(res) + except: + _log.warn('post-exec callback failed.') + self._incoming_queue.task_done() + + def fetch(self, callback): + while not self._payload_queue.empty(): + if self._incoming_queue.empty(): + time.sleep(5) + continue + self._process_result(callback) + + # Wait for all queued jobs to terminate + self._payload_queue.join() + while not self._incoming_queue.empty(): + self._process_result(callback) + + # Now send a death signal to all slaves. + for i in xrange(len(self._TP)): + self._payload_queue.put(('__exit__', None, None)) + self._TP = [] + return diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..cdb058c --- /dev/null +++ b/setup.py @@ -0,0 +1,29 @@ +#!/usr/bin/python +import os +from setuptools import setup, find_packages + + +def all_files(cwd, path): + return reduce(lambda x, y: x + y, + [[('%s/%s' % (x[0], y))[len(cwd)+1:] + for y in x[2]] for x in os.walk(cwd + '/' + path)]) + + +setup( + name='puppet_compiler', + version='0.0.1', + description='Tools to compile puppet catalogs as a service', + author='Joe', + author_email='[email protected]', + install_requires=['jinja2', 'requests', 'pyyaml'], + test_suite='nose.collector', + tests_require=['mock', 'nose'], + zip_safe=True, + packages=find_packages(), + package_data={'puppet_compiler': all_files("puppet_compiler", "templates")}, + entry_points={ + 'console_scripts': [ + 'puppet-compiler = puppet_compiler.cli:main' + ], + }, +) -- To view, visit https://gerrit.wikimedia.org/r/228849 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Id015fa2b0ff343f3dfd5ed28297f90539ce926ad Gerrit-PatchSet: 15 Gerrit-Project: operations/software/puppet-compiler Gerrit-Branch: master Gerrit-Owner: Giuseppe Lavagetto <[email protected]> Gerrit-Reviewer: Alexandros Kosiaris <[email protected]> Gerrit-Reviewer: Filippo Giunchedi <[email protected]> Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
