Giuseppe Lavagetto has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/363808 )
Change subject: Add future parser run mode ...................................................................... Add future parser run mode Bug: T169546 Change-Id: I5fc3eabd8704518dfeb17ebeab583010ca774690 --- M puppet_compiler/cli.py M puppet_compiler/controller.py M puppet_compiler/presentation/html.py M puppet_compiler/puppet.py M puppet_compiler/state.py A puppet_compiler/templates/hostpage.future.jinja2 A puppet_compiler/templates/index.future.jinja2 M puppet_compiler/tests/test_controller.py M puppet_compiler/tests/test_hostworker.py M puppet_compiler/tests/test_puppet.py M puppet_compiler/tests/test_state.py M puppet_compiler/worker.py M setup.py 13 files changed, 332 insertions(+), 74 deletions(-) Approvals: Giuseppe Lavagetto: Looks good to me, approved jenkins-bot: Verified diff --git a/puppet_compiler/cli.py b/puppet_compiler/cli.py index d4b34d1..8f9f055 100644 --- a/puppet_compiler/cli.py +++ b/puppet_compiler/cli.py @@ -14,6 +14,7 @@ job_id = int(os.environ.get('BUILD_NUMBER')) configfile = os.environ.get('PC_CONFIG', '/etc/puppet-compiler.conf') nthreads = os.environ.get('NUM_THREADS', 2) +compiler_mode = os.environ.get('MODE', 'change').split(",") def main(): @@ -37,8 +38,9 @@ sys.exit(2) _log.info("Working on change %d", change) - c = controller.Controller(configfile, job_id, - change, host_list=nodes, nthreads=nthreads) + + c = controller.Controller(configfile, job_id, change, host_list=nodes, + nthreads=nthreads, modes=compiler_mode) success = c.run() # If the run is marked as failed, exit with a non-zero exit code if not success: diff --git a/puppet_compiler/controller.py b/puppet_compiler/controller.py index f2b61b3..88a7222 100644 --- a/puppet_compiler/controller.py +++ b/puppet_compiler/controller.py @@ -23,8 +23,13 @@ class Controller(object): + available_run_modes = { + 'change': worker.HostWorker, + 'future': worker.FutureHostWorker, + } - def __init__(self, configfile, job_id, change_id, host_list=[], nthreads=2): + def __init__(self, configfile, job_id, change_id, host_list=[], + nthreads=2, modes=['change']): self.config = { # Url under which results will be found 'http_url': 'https://puppet-compiler.wmflabs.org/html', @@ -41,7 +46,7 @@ 'puppet_var': '/var/lib/catalog-differ/puppet', 'pool_size': nthreads, } - self.run_modes = {'change': worker.HostWorker} + self.run_modes = {m: w for m, w in self.available_run_modes.items() if m in modes} try: if configfile is not None: self._parse_conf(configfile) @@ -101,28 +106,30 @@ _log.info("Creating directories under %s", self.config['base']) self.m.prepare() - threadpool = threads.ThreadOrchestrator( - self.config['pool_size']) - # For each host, the threadpool will execute # Controller._run_host with the host as the only argument. # When this main payload is executed in a thread, the presentation # work is executed in the main thread via # Controller.on_node_compiled - for host in self.hosts: - for mode, worker_class in self.run_modes.items(): - state_class = worker_class.state - html_class = worker_class.html_index + for mode, worker_class in self.run_modes.items(): + threadpool = threads.ThreadOrchestrator( + self.config['pool_size']) + _log.info('Starting run in mode %s', mode) + state_class = worker_class.state + html_class = worker_class.html_index + for host in self.hosts: h = worker_class(self.config['puppet_var'], host) threadpool.add(h.run_host, hostname=host, mode=mode, classes=(state_class, html_class)) - threadpool.fetch(self.on_node_compiled) - for mode in self.run_modes.keys(): - index = html_class(self.outdir, mode) + # Let's wait for all runs in this mode to complete + threadpool.fetch(self.on_node_compiled) + # Let's create the index for this mode + index = html_class(self.outdir) index.render(self.state.modes[mode]) - _log.info('Run finished; see your results at %s', self.index_url(index)) + _log.info('Run finished for mode %s; see your results at %s', + mode, self.index_url(index)) # Remove the source and the diffs etc, we just need the output. self.m.cleanup() return self.success diff --git a/puppet_compiler/presentation/html.py b/puppet_compiler/presentation/html.py index 2636726..5510cc7 100644 --- a/puppet_compiler/presentation/html.py +++ b/puppet_compiler/presentation/html.py @@ -8,12 +8,24 @@ class Host(object): + tpl = 'hostpage.jinja2' + page_name = 'index.html' def __init__(self, hostname, files, retcode): self.retcode = retcode self.hostname = hostname self.outdir = files.outdir self.diff_file = files.file_for('change', 'diff') + + def _retcode_to_desc(self): + if self.retcode == 'noop': + return 'no change' + elif self.retcode == 'diff': + return 'changes detected' + elif self.retcode == 'error': + return 'change fails' + else: + return 'compiler failure' def htmlpage(self): """ @@ -24,39 +36,55 @@ if self.retcode == 'diff': with open(self.diff_file, 'r') as f: data['diffs'] = f.read() - if self.retcode == 'noop': - data['desc'] = 'no change' - elif self.retcode == 'diff': - data['desc'] = 'changes detected' - elif self.retcode == 'error': - data['desc'] = 'change fails' - else: - data['desc'] = 'compiler failure' - t = env.get_template('hostpage.jinja2') + data['desc'] = self._retcode_to_desc() + t = env.get_template(self.tpl) page = t.render(host=self.hostname, jid=job_id, chid=change_id, **data) - with open(os.path.join(self.outdir, 'index.html'), 'w') as f: + with open(os.path.join(self.outdir, self.page_name), 'w') as f: f.write(page) -class Index(object): +class FutureHost(Host): + tpl = 'hostpage.future.jinja2' + page_name = 'index-future.html' - def __init__(self, outdir, mode): - # Hack! generalize somehow - if mode == 'change': - filename = "index.html" + def __init__(self, hostname, files, retcode): + super(FutureHost, self).__init__(hostname, files, retcode) + self.diff_file = files.file_for('future', 'diff') + + def _retcode_to_desc(self): + if self.retcode == 'break': + return 'change breaks the current parser' + elif self.retcode == 'error': + return 'change is not compatible with the future parser' + elif self.retcode == 'ok': + return 'change works with both parsers' + elif self.retcode == 'diff': + return 'change works with both parsers, with diffs' + + +class Index(object): + tpl = 'index.jinja2' + page_name = 'index.html' + + def __init__(self, outdir): + if self.page_name == 'index.html': self.url = "" else: - self.url = 'index%s.html' % mode - filename = self.url - self.outfile = os.path.join(outdir, filename) + self.url = self.page_name + self.outfile = os.path.join(outdir, self.page_name) def render(self, state): """ Render the index page with info coming from state """ _log.debug("Rendering the main index page") - t = env.get_template('index.jinja2') + t = env.get_template(self.tpl) # TODO: support multiple modes page = t.render(state=state, jid=job_id, chid=change_id) with open(self.outfile, 'w') as f: f.write(page) + + +class FutureIndex(Index): + tpl = 'index.future.jinja2' + page_name = 'index-future.html' diff --git a/puppet_compiler/puppet.py b/puppet_compiler/puppet.py index 7ab0f9e..dd07307 100644 --- a/puppet_compiler/puppet.py +++ b/puppet_compiler/puppet.py @@ -45,12 +45,12 @@ f.write(line) -def diff(env, hostname): +def diff(env, hostname, base='prod'): """ Compute the diffs between the two changes """ hostfiles = HostFiles(hostname) - prod_catalog = hostfiles.file_for('prod', 'catalog') + prod_catalog = hostfiles.file_for(base, 'catalog') change_catalog = hostfiles.file_for(env, 'catalog') output = hostfiles.file_for(env, 'diff') cmd = ['puppet', 'catalog', 'diff', '--show_resource_diff', diff --git a/puppet_compiler/state.py b/puppet_compiler/state.py index d46dc55..28b4d3b 100644 --- a/puppet_compiler/state.py +++ b/puppet_compiler/state.py @@ -78,3 +78,17 @@ return 'fail' else: return 'diff' + + +class FutureState(ChangeState): + + @property + def name(self): + if self.prod_error: + return 'break' + elif self.change_error or self.diff is False: + return 'error' + elif self.diff is None: + return 'ok' + else: + return 'diff' diff --git a/puppet_compiler/templates/hostpage.future.jinja2 b/puppet_compiler/templates/hostpage.future.jinja2 new file mode 100644 index 0000000..47f731a --- /dev/null +++ b/puppet_compiler/templates/hostpage.future.jinja2 @@ -0,0 +1,49 @@ +<!doctype html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>Parsers comparison for {{ host }}</title> + <style type="text/css"> + h1, h2, h3, h4 { + font-family: "HelveticaNeue-CondensedBold","Helvetica Neue","Arial Narrow",Arial,sans-serif; + } + .source { + font-family: Consolas, "Andale Mono", "Courier New", monospace + } + .err .fail { + color: red; + } + .noop { + color: darkgreen; + } + .diff { + color: darkgoldenrod; + } + </style> + </head> + <body> + <div id="main"> + {% block body %} + <h1>Compilation results for {{ host }}: <span class="{{retcode}}">{{ desc }}</span></h1> + {% if retcode == "diff" %} + <h2>Catalog differences</h2> + <textarea rows="20" cols="82" class="source">{{ diffs }}</textarea> + {% endif %} + <h2>Relevant files</h2> + <ul> + <li><a href="change.{{ host }}.pson">Old parser catalog</a></li> + <li><a href="future.{{ host }}-future.pson">Future parser catalog</a></li> + <li><a href="change.{{ host }}-future.err">Old parser errors/warnings</a></li> + <li><a href="future.{{ host }}-future.err">Future parser errors/warnings</a></li> + </ul> + {% endblock %} + {% block extnav %} + <p> + <a href="https://integration.wikimedia.org/ci/job/operations-puppet-catalog-compiler/{{ jid }}/" target="_blank">Back to jenkins job</a> + <a href="https://gerrit.wikimedia.org/r/#/c/{{ chid }}/" target="_blank">See gerrit change</a> + </p> + {% endblock %} + </div> + </body> +</html> diff --git a/puppet_compiler/templates/index.future.jinja2 b/puppet_compiler/templates/index.future.jinja2 new file mode 100644 index 0000000..6c151dd --- /dev/null +++ b/puppet_compiler/templates/index.future.jinja2 @@ -0,0 +1,30 @@ +{% extends "hostpage.jinja2" %} +{% block body %} + <h1>Results for change <a href="https://gerrit.wikimedia.org/r/#/c/{{ chid }}/" target="_blank">{{ chid }} </a> with the future parser.</h1> + <div id="list"> + <h2>Hosts that have no differences with the two parsers</h2> + <ul> + {% for host in state.ok|sort %} + <li> <a href="{{ host }}//index-future.html">{{ host }}</a> + {% endfor %} + </ul> + <h2>Hosts that compile with differences</h2> + <ul> + {% for host in state.diff|sort %} + <li> <a href="{{ host }}//index-future.html">{{ host }}</a> + {% endfor %} + </ul> + <h2>Hosts that fail to compile with the future parser</h2> + <ul> + {% for host in state.error|sort %} + <li> <a href="{{ host }}/index-future.html">{{ host }}</a> + {% endfor %} + </ul> + <h2>Hosts that break with the current parser</h2> + <ul> + {% for host in state.break|sort %} + <li> <a href="{{ host }}/index-future.html">{{ host }}</a> + {% endfor %} + </ul> + </div> +{% endblock %} diff --git a/puppet_compiler/tests/test_controller.py b/puppet_compiler/tests/test_controller.py index c3291e1..f04c68c 100644 --- a/puppet_compiler/tests/test_controller.py +++ b/puppet_compiler/tests/test_controller.py @@ -17,6 +17,8 @@ self.assertEquals(c.config['http_url'], 'https://puppet-compiler.wmflabs.org/html') self.assertEquals(c.config['base'], '/mnt/jenkins-workspace') + c = controller.Controller(None, 19, 224570, 'test.eqiad.wmnet', nthreads=2, modes=['future']) + self.assertEqual(c.run_modes.keys(), ['future']) def test_parse_config(self): filename = os.path.join(self.fixtures, 'test_config.yaml') diff --git a/puppet_compiler/tests/test_hostworker.py b/puppet_compiler/tests/test_hostworker.py index 8405ec3..77c6a81 100644 --- a/puppet_compiler/tests/test_hostworker.py +++ b/puppet_compiler/tests/test_hostworker.py @@ -127,3 +127,28 @@ # An exception writing the output doesn't make the payload fail self.hw._make_output.side_effect = Exception('Boom!') self.assertEquals(self.hw.run_host(), (True, False, None)) + + +class TestFutureHostWorker(unittest.TestCase): + def setUp(self): + self.fixtures = os.path.join(os.path.dirname(__file__), 'fixtures') + c = controller.Controller(None, 19, 224570, 'test.eqiad.wmnet') + self.hw = worker.FutureHostWorker(c.config['puppet_var'], + 'test.example.com') + + def test_initialize(self): + self.assertListEqual(self.hw._envs, ['change', 'future']) + + def test_compile_all(self): + self.hw._compile = mock.Mock(return_value=True) + self.assertEquals(self.hw._compile_all(), 0) + future_parser = [ + '--environment=future', + '--parser=future', + '--environmentpath=/mnt/jenkins-workspace/19/change/src/environments', + '--default_manifest=\$confdir/manifests/site.pp' + ] + self.hw._compile.assert_has_calls([ + mock.call('change', []), + mock.call('future', future_parser) + ]) diff --git a/puppet_compiler/tests/test_puppet.py b/puppet_compiler/tests/test_puppet.py index da4d4af..bfd858a 100644 --- a/puppet_compiler/tests/test_puppet.py +++ b/puppet_compiler/tests/test_puppet.py @@ -21,6 +21,8 @@ m = mock.mock_open(read_data='wat') with mock.patch('__builtin__.open', m, True) as mocker: puppet.compile('test.codfw.wmnet', 'prod', self.fixtures + '/puppet_var') + spool = tf_mocker.return_value + spool.return_value = ["Test ", "Info: meh"] subprocess.check_call.assert_called_with( ['puppet', 'master', @@ -33,7 +35,7 @@ '--compile=test.codfw.wmnet', '--color=false'], env=env, - stdout=tf_mocker.return_value, + stdout=spool, stderr=mocker.return_value ) hostfile = os.path.join(self.fixtures, '10/production/catalogs', 'test.codfw.wmnet') @@ -42,6 +44,29 @@ mock.call(hostfile + '.err', 'w'), ] mocker.assert_has_calls(calls, any_order=True) + with mock.patch('__builtin__.open', m, True) as mocker: + puppet.compile('test.codfw.wmnet', 'test', self.fixtures + '/puppet_var') + subprocess.check_call.assert_called_with( + ['puppet', + 'master', + '--vardir=%s' % self.fixtures + '/puppet_var', + '--modulepath=%(basedir)s/private/modules:' + '%(basedir)s/src/modules' % {'basedir': FHS.change_dir}, + '--confdir=%s/%s' % (FHS.change_dir, 'src'), + '--templatedir=%s/%s' % (FHS.change_dir, 'src/templates'), + '--trusted_node_data', + '--compile=test.codfw.wmnet', + '--color=false'], + env=env, + stdout=tf_mocker.return_value, + stderr=mocker.return_value + ) + hostfile = os.path.join(self.fixtures, '10/change/catalogs', 'test.codfw.wmnet-test') + calls = [ + mock.call(hostfile + '.pson', 'w'), + mock.call(hostfile + '.err', 'w'), + ] + mocker.assert_has_calls(calls, any_order=True) @mock.patch('puppet_compiler.puppet.spoolfile') def test_extra_args_compile(self, tf_mocker): diff --git a/puppet_compiler/tests/test_state.py b/puppet_compiler/tests/test_state.py index d7cecdd..f1acadb 100644 --- a/puppet_compiler/tests/test_state.py +++ b/puppet_compiler/tests/test_state.py @@ -48,3 +48,26 @@ collection.add(test2) self.assertEqual(collection.mode_to_str('nonexistent'), '[nonexistent] Nodes: ') self.assertRegexpMatches(collection.mode_to_str('test'), ' 1 DIFF ') + + +class TestFutureState(TestChangeState): + + def test_name(self): + # Production works, future does not + test = state.FutureState('change', 'test.example.com', False, True, None) + self.assertEqual('error', test.name) + # Prod compiled, future too, no diffs + test.change_error = False + self.assertEqual(test.name, 'ok') + # Diff failed to be created + test.diff = False + self.assertEqual(test.name, 'error') + # There are diffs + test.diff = True + self.assertEqual(test.name, 'diff') + # Both prod and future failed + test = state.FutureState('change', 'test.example.com', True, True, None) + self.assertEqual(test.name, 'break') + # Prod failed, change didn't + test = state.FutureState('change', 'test.example.com', True, False, None) + self.assertEqual(test.name, 'break') diff --git a/puppet_compiler/worker.py b/puppet_compiler/worker.py index f345bcc..192b9c3 100644 --- a/puppet_compiler/worker.py +++ b/puppet_compiler/worker.py @@ -6,13 +6,13 @@ from puppet_compiler import puppet, _log from puppet_compiler.directories import HostFiles, FHS from puppet_compiler.presentation import html -from puppet_compiler.state import ChangeState +from puppet_compiler.state import ChangeState, FutureState class HostWorker(object): E_OK = 0 - E_PROD = 1 - E_CHANGE = 2 + E_BASE = 1 + E_CHANGED = 2 state = ChangeState html_page = html.Host html_index = html.Index @@ -44,8 +44,8 @@ diff = self._make_diff() else: diff = None - base = errors & self.E_PROD - change = errors & self.E_CHANGE + base = errors & self.E_BASE + change = errors & self.E_CHANGED try: self._make_output() state = self.state('', self.hostname, base, change, diff) @@ -55,42 +55,57 @@ exc_info=True) return (base, change, diff) + def _check_if_compiled(self, env): + catalog = self._files.file_for(env, 'catalog') + err = self._files.file_for(env, 'errors') + if os.path.isfile(catalog) and os.stat(catalog).st_size > 0: + # This is already compiled and it worked. + return True + if os.path.isfile(err): + # This complied, and it just outputs an error: + return False + # Nothing is found + return None + + def _compile(self, env, args): + # this can run multiple times for the same env in different workers. + # Check if already ran, there is no need to run a second time. + check = self._check_if_compiled(env) + if check is not None: + return check + if env != 'prod' \ + and os.path.isfile(os.path.join(FHS.change_dir, 'src', '.configs')): + with open(os.path.join(FHS.change_dir, 'src', '.configs')) as f: + configs = f.readlines() + # Make sure every item has exactly 2 dashes prepended + configs = map(lambda x: "--{}".format(x.lstrip('-')), configs) + args.extend(configs) + + try: + _log.info("Compiling host %s (%s)", self.hostname, env) + puppet.compile(self.hostname, + env, + self.puppet_var, + *args) + except subprocess.CalledProcessError as e: + _log.error("Compilation failed for hostname %s " + " in environment %s.", self.hostname, env) + _log.info("Compilation exited with code %d", e.returncode) + _log.debug("Failed command: %s", e.cmd) + return False + else: + return True + def _compile_all(self): """ Does the grindwork of compiling the catalogs """ errors = self.E_OK - try: - _log.info("Compiling host %s (production)", self.hostname) - puppet.compile(self.hostname, - self._envs[0], - self.puppet_var) - except subprocess.CalledProcessError as e: - _log.error("Compilation failed for hostname %s " - " with the current tree.", self.hostname) - _log.info("Compilation exited with code %d", e.returncode) - _log.debug("Failed command: %s", e.cmd) - errors += self.E_PROD args = [] - if os.path.isfile(os.path.join(FHS.change_dir, 'src', '.configs')): - with open(os.path.join(FHS.change_dir, 'src', '.configs')) as f: - configs = f.readlines() - # Make sure every item has exactly 2 dashes prepended - configs = map(lambda x: "--{}".format(x.lstrip('-')), configs) - args.extend(configs) - try: - _log.info("Compiling host %s (change)", self.hostname) - puppet.compile(self.hostname, - self._envs[1], - self.puppet_var, - *args) - except subprocess.CalledProcessError as e: - _log.error("Compilation failed for hostname %s " - " with the change.", self.hostname) - _log.info("Compilation exited with code %d", e.returncode) - _log.debug("Failed command: %s", e.cmd) - errors += self.E_CHANGE - # Now test for regressions with the future parser + if not self._compile(self._envs[0], args): + errors += self.E_BASE + if not self._compile(self._envs[1], args): + errors += self.E_CHANGED return errors def _make_diff(self): @@ -117,7 +132,8 @@ Prepare the node output, copying the relevant files in place in the output directory """ - os.makedirs(self._files.outdir, 0o755) + if not os.path.isdir(self._files.outdir): + os.makedirs(self._files.outdir, 0o755) for env in self._envs: for label in 'catalog', 'errors': filename = self._files.file_for(env, label) @@ -157,3 +173,40 @@ # TODO: implement the actual html parsing host = self.html_page(self.hostname, self._files, retcode) host.htmlpage() + + +class FutureHostWorker(HostWorker): + """ + This worker is designed to be used when transitioning to the future parser. + It will compile the change first with the normal parser, then with the future one, + and make a diff between the two. + + Results: + "ok" => both catalogs compile, and there is no diff + "diff" => both catalogs compile, but there is a diff + "error" => normal parser works, but future parser doesn't + "break" => future parser works, but the normal one doesn't + """ + E_FUTURE = 4 + state = FutureState + html_page = html.FutureHost + html_index = html.FutureIndex + + def __init__(self, vardir, hostname): + super(FutureHostWorker, self).__init__(vardir, hostname) + self._envs = ['change', 'future'] + + def _compile_all(self): + future_args = [ + '--environment=future', + '--parser=future', + '--environmentpath=%s' % os.path.join(FHS.change_dir, 'src', 'environments'), + '--default_manifest=\$confdir/manifests/site.pp' + ] + args = [] + errors = self.E_OK + if not self._compile(self._envs[0], args): + errors += self.E_BASE + if not self._compile(self._envs[1], future_args): + errors += self.E_CHANGED + return errors diff --git a/setup.py b/setup.py index 4849bbc..8468942 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name='puppet_compiler', - version='0.1.9', + version='0.2.0', description='Tools to compile puppet catalogs as a service', author='Joe', author_email='glavage...@wikimedia.org', -- To view, visit https://gerrit.wikimedia.org/r/363808 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I5fc3eabd8704518dfeb17ebeab583010ca774690 Gerrit-PatchSet: 9 Gerrit-Project: operations/software/puppet-compiler Gerrit-Branch: master Gerrit-Owner: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits