Giuseppe Lavagetto has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/367678 )
Change subject: Add filters to the future parser ...................................................................... Add filters to the future parser The catalogs generated from the future parser introduce a lot of null differences with the traditional catalog, we filter out at least the differences we know to be spurious, like one-element arrays. Change-Id: I2b2e74f284c46e7ef3c99cd60e00521c26bfc418 --- A puppet_compiler/filter.py M puppet_compiler/worker.py 2 files changed, 63 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/software/puppet-compiler refs/changes/78/367678/1 diff --git a/puppet_compiler/filter.py b/puppet_compiler/filter.py new file mode 100644 index 0000000..aaeeec5 --- /dev/null +++ b/puppet_compiler/filter.py @@ -0,0 +1,58 @@ +import json +from abc import ABCMeta, abstractmethod + + +class CatalogFilter(object): + __metaclass__ = ABCMeta + + def __init__(self, filename): + self.filename = filename + self.original = '' + self.catalog = '' + self._filters = self.setup_filters() + + @abstractmethod + def setup_filters(self): + """ + Creates an array of callbacks to be used to filter the + catalog + """ + pass + + def run(self): + """ + perform the filtering, write files. + """ + self._read_json() + self.catalog = self.original + for callback in self._filters: + self.catalog = callback(self.catalog) + self._write_files() + + def _read_json(self): + """ + Reads the catalog from file + """ + with open(self.filename, 'r', encoding='latin_1') as fh: + # https://docs.puppet.com/puppet/latest/http_api/pson.html#decoding-pson-using-json-parsers + self.original = json.load(fh) + + def _write_files(self): + with open(self.filename, 'w') as fh: + json.dump(fh, self.catalog, encoding='latin_1') + with open(self.filename + '.orig', 'w') as fh: + json.dump(fh, self.original, encoding='latin_1') + + +class FilterFutureParser(CatalogFilter): + @staticmethod + def flatten_one_element_arrays(catalog): + for resource in catalog['data']['resources']: + params = resource.get('parameters', {}) + for param, val in params.items(): + if type(val) == list and len(val) == 1: + params[param] = val[0] + return catalog + + def setup_filters(self): + return [FilterFutureParser.flatten_one_element_arrays] diff --git a/puppet_compiler/worker.py b/puppet_compiler/worker.py index efb8794..662c360 100644 --- a/puppet_compiler/worker.py +++ b/puppet_compiler/worker.py @@ -4,6 +4,7 @@ import subprocess from puppet_compiler import puppet, _log +from puppet_compiler.filter import FilterFutureParser from puppet_compiler.directories import HostFiles, FHS from puppet_compiler.presentation import html from puppet_compiler.state import ChangeState, FutureState @@ -195,6 +196,7 @@ def __init__(self, vardir, hostname): super(FutureHostWorker, self).__init__(vardir, hostname) self._envs = ['change', 'future'] + self.filter_future = FilterFutureParser(self._files.file_for('future', 'catalog')) def _compile_all(self): future_args = [ @@ -209,4 +211,7 @@ errors += self.E_BASE if not self._compile(self._envs[1], future_args): errors += self.E_CHANGED + _log.info("Filtering the future catalog (%s)", self.hostname) + self.filter_future.run() + return errors -- To view, visit https://gerrit.wikimedia.org/r/367678 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I2b2e74f284c46e7ef3c99cd60e00521c26bfc418 Gerrit-PatchSet: 1 Gerrit-Project: operations/software/puppet-compiler Gerrit-Branch: master Gerrit-Owner: Giuseppe Lavagetto <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
