Giuseppe Lavagetto has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/363808 )
Change subject: [WiP] Add future parser run mode
......................................................................
[WiP] Add future parser run mode
Change-Id: I5fc3eabd8704518dfeb17ebeab583010ca774690
---
M puppet_compiler/presentation/html.py
M puppet_compiler/puppet.py
M puppet_compiler/state.py
A puppet_compiler/templates/hostpage.future.jinja2
A puppet_compiler/templates/index.future.jinja2
M puppet_compiler/worker.py
6 files changed, 211 insertions(+), 58 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/software/puppet-compiler
refs/changes/08/363808/1
diff --git a/puppet_compiler/presentation/html.py
b/puppet_compiler/presentation/html.py
index a24a5ba..90f6162 100644
--- a/puppet_compiler/presentation/html.py
+++ b/puppet_compiler/presentation/html.py
@@ -8,12 +8,24 @@
class Host(object):
+ tpl = 'hostpage.jinja2'
+ page_name = 'index.html'
def __init__(self, hostname, files, retcode):
self.retcode = retcode
self.hostname = hostname
self.outdir = files.outdir
self.diff_file = files.file_for('change', 'diff')
+
+ def _retcode_to_desc(self):
+ if self.retcode == 'noop':
+ return 'no change'
+ elif self.retcode == 'diff':
+ return 'changes detected'
+ elif self.retcode == 'error':
+ return 'change fails'
+ else:
+ return 'compiler failure'
def htmlpage(self):
"""
@@ -24,39 +36,52 @@
if self.retcode == 'diff':
with open(self.diff_file, 'r') as f:
data['diffs'] = f.read()
- if self.retcode == 'noop':
- data['desc'] = 'no change'
- elif self.retcode == 'diff':
- data['desc'] = 'changes detected'
- elif self.retcode == 'error':
- data['desc'] = 'change fails'
- else:
- data['desc'] = 'compiler failure'
- t = env.get_template('hostpage.jinja2')
+ data['desc'] = self._retcode_to_desc()
+ t = env.get_template(self.tpl)
page = t.render(host=self.hostname, jid=job_id, chid=change_id, **data)
- with open(os.path.join(self.outdir, 'index.html'), 'w') as f:
+ with open(os.path.join(self.outdir, self.page_name), 'w') as f:
f.write(page)
-class Index(object):
- default_mode = 'change'
+class FutureHost(Host):
+ tpl = 'hostpage.future.jinja2'
+ page_name = 'index-future.html'
- def __init__(self, outdir, mode):
- if mode == self.default_mode:
- filename = "index.html"
- self.url = ""
- else:
- self.url = 'index%s.html' % mode
- filename = self.url
- self.outfile = os.path.join(outdir, filename)
+ def __init__(self, hostname, files, retcode):
+ super(FutureHost, self).__init__(hostname, files, retcode)
+ self.diff_file = files.file_for('future', 'diff')
+
+ def _retcode_to_desc(self):
+ if self.retcode == 'break':
+ return 'change breaks the current parser'
+ elif self.retcode == 'error':
+ return 'change is not compatible with the future parser'
+ elif self.retcode == 'ok':
+ return 'change works with both parsers'
+ elif self.retcode == 'diff':
+ return 'change works with both parsers, with diffs'
+
+
+class Index(object):
+ tpl = 'index.jinja2'
+
+ def __init__(self, outdir):
+ filename = "index.html"
+ self.url = ""
+ self.outfile = os.path.join(outdir, self.page_name)
def render(self, state):
"""
Render the index page with info coming from state
"""
_log.debug("Rendering the main index page")
- t = env.get_template('index.jinja2')
+ t = env.get_template(self.tpl)
# TODO: support multiple modes
page = t.render(state, jid=job_id, chid=change_id)
with open(self.outfile, 'w') as f:
f.write(page)
+
+
+class FutureIndex(Index):
+ tpl = 'index.future.jinja2'
+ page_name = 'index-future.html'
diff --git a/puppet_compiler/puppet.py b/puppet_compiler/puppet.py
index 7ab0f9e..dd07307 100644
--- a/puppet_compiler/puppet.py
+++ b/puppet_compiler/puppet.py
@@ -45,12 +45,12 @@
f.write(line)
-def diff(env, hostname):
+def diff(env, hostname, base='prod'):
"""
Compute the diffs between the two changes
"""
hostfiles = HostFiles(hostname)
- prod_catalog = hostfiles.file_for('prod', 'catalog')
+ prod_catalog = hostfiles.file_for(base, 'catalog')
change_catalog = hostfiles.file_for(env, 'catalog')
output = hostfiles.file_for(env, 'diff')
cmd = ['puppet', 'catalog', 'diff', '--show_resource_diff',
diff --git a/puppet_compiler/state.py b/puppet_compiler/state.py
index 51035da..bb076e5 100644
--- a/puppet_compiler/state.py
+++ b/puppet_compiler/state.py
@@ -47,3 +47,17 @@
return 'fail'
else:
return 'diff'
+
+
+class FutureState(ChangeState):
+
+ @property
+ def name(self):
+ if self.prod_error:
+ return 'break'
+ elif self.change_error or self.diff is False:
+ return 'error'
+ elif self.diff is None:
+ return 'ok'
+ else:
+ return 'diff'
diff --git a/puppet_compiler/templates/hostpage.future.jinja2
b/puppet_compiler/templates/hostpage.future.jinja2
new file mode 100644
index 0000000..9956cc4
--- /dev/null
+++ b/puppet_compiler/templates/hostpage.future.jinja2
@@ -0,0 +1,49 @@
+<!doctype html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <title>Parsers comparison for {{ host }}</title>
+ <style type="text/css">
+ h1, h2, h3, h4 {
+ font-family: "HelveticaNeue-CondensedBold","Helvetica Neue","Arial
Narrow",Arial,sans-serif;
+ }
+ .source {
+ font-family: Consolas, "Andale Mono", "Courier New", monospace
+ }
+ .err .fail {
+ color: red;
+ }
+ .noop {
+ color: darkgreen;
+ }
+ .diff {
+ color: darkgoldenrod;
+ }
+ </style>
+ </head>
+ <body>
+ <div id="main">
+ {% block body %}
+ <h1>Compilation results for {{ host }}: <span class="{{retcode}}">{{
desc }}</span></h1>
+ {% if retcode == "diff" %}
+ <h2>Catalog differences</h2>
+ <textarea rows="20" cols="82" class="source">{{ diffs }}</textarea>
+ {% endif %}
+ <h2>Relevant files</h2>
+ <ul>
+ <li><a href="change.{{ host }}.pson">Old parser catalog</a></li>
+ <li><a href="change.{{ host }}-future.pson">Future parser
catalog</a></li>
+ <li><a href="prod.{{ host }}-future.err">Old parser
errors/warnings</a></li>
+ <li><a href="change.{{ host }}-future.err">Future parser
errors/warnings</a></li>
+ </ul>
+ {% endblock %}
+ {% block extnav %}
+ <p>
+ <a
href="https://integration.wikimedia.org/ci/job/operations-puppet-catalog-compiler/{{
jid }}/" target="_blank">Back to jenkins job</a>
+ <a href="https://gerrit.wikimedia.org/r/#/c/{{ chid }}/"
target="_blank">See gerrit change</a>
+ </p>
+ {% endblock %}
+ </div>
+ </body>
+</html>
diff --git a/puppet_compiler/templates/index.future.jinja2
b/puppet_compiler/templates/index.future.jinja2
new file mode 100644
index 0000000..e5c7c39
--- /dev/null
+++ b/puppet_compiler/templates/index.future.jinja2
@@ -0,0 +1,30 @@
+{% extends "hostpage.jinja2" %}
+{% block body %}
+ <h1>Results for change <a href="https://gerrit.wikimedia.org/r/#/c/{{ chid
}}/" target="_blank">{{ chid }} with the future parser.</a></h1>
+ <div id="list">
+ <h2>Hosts that have no differences with the two parsers</h2>
+ <ul>
+ {% for host in state.ok|sort %}
+ <li> <a href="{{ host }}/">{{ host }}</a>
+ {% endfor %}
+ </ul>
+ <h2>Hosts that compile with differences</h2>
+ <ul>
+ {% for host in state.diff|sort %}
+ <li> <a href="{{ host }}/">{{ host }}</a>
+ {% endfor %}
+ </ul>
+ <h2>Hosts that fail to compile with the future parser</h2>
+ <ul>
+ {% for host in state.error|sort %}
+ <li> <a href="{{ host }}/">{{ host }}</a>
+ {% endfor %}
+ </ul>
+ <h2>Hosts that break with the current parser</h2>
+ <ul>
+ {% for host in state.break|sort %}
+ <li> <a href="{{ host }}/">{{ host }}</a>
+ {% endfor %}
+ </ul>
+ </div>
+{% endblock %}
diff --git a/puppet_compiler/worker.py b/puppet_compiler/worker.py
index f345bcc..adb59b6 100644
--- a/puppet_compiler/worker.py
+++ b/puppet_compiler/worker.py
@@ -6,13 +6,13 @@
from puppet_compiler import puppet, _log
from puppet_compiler.directories import HostFiles, FHS
from puppet_compiler.presentation import html
-from puppet_compiler.state import ChangeState
+from puppet_compiler.state import ChangeState, FutureState
class HostWorker(object):
E_OK = 0
- E_PROD = 1
- E_CHANGE = 2
+ E_BASE = 1
+ E_CHANGED = 2
state = ChangeState
html_page = html.Host
html_index = html.Index
@@ -44,8 +44,8 @@
diff = self._make_diff()
else:
diff = None
- base = errors & self.E_PROD
- change = errors & self.E_CHANGE
+ base = errors & self.E_BASE
+ change = errors & self.E_CHANGED
try:
self._make_output()
state = self.state('', self.hostname, base, change, diff)
@@ -55,42 +55,40 @@
exc_info=True)
return (base, change, diff)
+ def _compile(self, env, args):
+ if env != 'prod' \
+ and os.path.isfile(os.path.join(FHS.change_dir, 'src', '.configs')):
+ with open(os.path.join(FHS.change_dir, 'src', '.configs')) as f:
+ configs = f.readlines()
+ # Make sure every item has exactly 2 dashes prepended
+ configs = map(lambda x: "--{}".format(x.lstrip('-')), configs)
+ args.extend(configs)
+
+ try:
+ _log.info("Compiling host %s (%s)", self.hostname, env)
+ puppet.compile(self.hostname,
+ env,
+ self.puppet_var,
+ *args)
+ except subprocess.CalledProcessError as e:
+ _log.error("Compilation failed for hostname %s "
+ " in environment %s.", self.hostname, env)
+ _log.info("Compilation exited with code %d", e.returncode)
+ _log.debug("Failed command: %s", e.cmd)
+ return False
+ else:
+ return True
+
def _compile_all(self):
"""
Does the grindwork of compiling the catalogs
"""
errors = self.E_OK
- try:
- _log.info("Compiling host %s (production)", self.hostname)
- puppet.compile(self.hostname,
- self._envs[0],
- self.puppet_var)
- except subprocess.CalledProcessError as e:
- _log.error("Compilation failed for hostname %s "
- " with the current tree.", self.hostname)
- _log.info("Compilation exited with code %d", e.returncode)
- _log.debug("Failed command: %s", e.cmd)
- errors += self.E_PROD
args = []
- if os.path.isfile(os.path.join(FHS.change_dir, 'src', '.configs')):
- with open(os.path.join(FHS.change_dir, 'src', '.configs')) as f:
- configs = f.readlines()
- # Make sure every item has exactly 2 dashes prepended
- configs = map(lambda x: "--{}".format(x.lstrip('-')), configs)
- args.extend(configs)
- try:
- _log.info("Compiling host %s (change)", self.hostname)
- puppet.compile(self.hostname,
- self._envs[1],
- self.puppet_var,
- *args)
- except subprocess.CalledProcessError as e:
- _log.error("Compilation failed for hostname %s "
- " with the change.", self.hostname)
- _log.info("Compilation exited with code %d", e.returncode)
- _log.debug("Failed command: %s", e.cmd)
- errors += self.E_CHANGE
- # Now test for regressions with the future parser
+ if not self._compile(self._envs[0], args):
+ errors += self.E_BASE
+ if not self._compile(self._envs[1], args):
+ errors += self.E_CHANGED
return errors
def _make_diff(self):
@@ -157,3 +155,40 @@
# TODO: implement the actual html parsing
host = self.html_page(self.hostname, self._files, retcode)
host.htmlpage()
+
+
+class FutureHostWorker(HostWorker):
+ """
+ This worker is designed to be used when transitioning to the future parser.
+ It will compile the change first with the normal parser, then with the
future one,
+ and make a diff between the two.
+
+ Results:
+ "ok" => both catalogs compile, and there is no diff
+ "diff" => both catalogs compile, but there is a diff
+ "error" => normal parser works, but future parser doesn't
+ "break" => future parser works, but the normal one doesn't
+ """
+ E_FUTURE = 4
+ state_class = FutureState
+ html_page = html.FutureHost
+ html_index = html.FutureIndex
+
+ def __init__(self, vardir, hostname):
+ super(FutureHostWorker, self).__init__(vardir, hostname)
+ self._envs = ['change', 'future']
+
+ def _compile_all(self):
+ future_args = [
+ '--environment=future',
+ '--parser=future',
+ '--environmentpath=%s' % os.path.join(self.m.change_dir, 'src',
environments),
+ '--default_manifest=\$confdir/manifests/site.pp'
+ ]
+ args = []
+ errors = self.E_OK
+ if not self._compile(self._envs[0], args):
+ errors += self.E_BASE
+ if not self._compile(self._envs[1], future_args):
+ errors += self.E_CHANGED
+ return errors
--
To view, visit https://gerrit.wikimedia.org/r/363808
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I5fc3eabd8704518dfeb17ebeab583010ca774690
Gerrit-PatchSet: 1
Gerrit-Project: operations/software/puppet-compiler
Gerrit-Branch: master
Gerrit-Owner: Giuseppe Lavagetto <glavage...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
