Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-146-Support-colorful-execution-logging 2dd9830f5 -> 81bd26a7f
reworked the entire config process. it is now possible to config the styles and formats directly from the config file Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/81bd26a7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/81bd26a7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/81bd26a7 Branch: refs/heads/ARIA-146-Support-colorful-execution-logging Commit: 81bd26a7fb453e02eab1e1b8100bb9c3057928cd Parents: 2dd9830 Author: max-orlov <ma...@gigaspaces.com> Authored: Wed Apr 26 00:24:37 2017 +0300 Committer: max-orlov <ma...@gigaspaces.com> Committed: Wed Apr 26 00:24:37 2017 +0300 ---------------------------------------------------------------------- aria/cli/color.py | 5 +- aria/cli/commands/executions.py | 6 +- aria/cli/commands/logs.py | 3 +- aria/cli/config/config_template.yaml | 40 ++++++ aria/cli/execution_logging.py | 209 ++++++++++++------------------ 5 files changed, 130 insertions(+), 133 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/81bd26a7/aria/cli/color.py ---------------------------------------------------------------------- diff --git a/aria/cli/color.py b/aria/cli/color.py index 77bfe77..49675de 100644 --- a/aria/cli/color.py +++ b/aria/cli/color.py @@ -66,8 +66,7 @@ class Color(object): @classmethod def markup(cls, str_to_stylize, matches, schema): - modified_str = str_to_stylize for group_index in xrange(len(matches.regs)): match = matches.group(group_index) - modified_str = modified_str.replace(match, schema + match + cls.Back.RESET) - return modified_str + str_to_stylize = str_to_stylize.replace(match, schema + match + cls.Back.RESET) + return str_to_stylize http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/81bd26a7/aria/cli/commands/executions.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/executions.py b/aria/cli/commands/executions.py index 3eb53ab..7f55e6a 100644 --- a/aria/cli/commands/executions.py +++ b/aria/cli/commands/executions.py @@ -149,8 +149,7 @@ def start(workflow_name, log_iterator = cli_logger.ModelLogIterator(model_storage, workflow_runner.execution_id) try: while execution_thread.is_alive(): - with execution_logging.format(mark_pattern=mark_pattern): - execution_logging.log_list(log_iterator) + execution_logging.log_list(log_iterator, mark_pattern=mark_pattern) execution_thread.join(1) except KeyboardInterrupt: @@ -158,8 +157,7 @@ def start(workflow_name, # It might be the case where some logs were written and the execution was terminated, thus we # need to drain the remaining logs. - with execution_logging.format(mark_pattern=mark_pattern): - execution_logging.log_list(log_iterator) + execution_logging.log_list(log_iterator, mark_pattern=mark_pattern) # raise any errors from the execution thread (note these are not workflow execution errors) execution_thread.raise_error_if_exists() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/81bd26a7/aria/cli/commands/logs.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/logs.py b/aria/cli/commands/logs.py index 350a035..2f7f361 100644 --- a/aria/cli/commands/logs.py +++ b/aria/cli/commands/logs.py @@ -38,8 +38,7 @@ def list(execution_id, mark_pattern, model_storage, logger): logger.info('Listing logs for execution id {0}'.format(execution_id)) log_iterator = ModelLogIterator(model_storage, execution_id) - with execution_logging.format(mark_pattern=mark_pattern): - any_logs = execution_logging.log_list(log_iterator) + any_logs = execution_logging.log_list(log_iterator, mark_pattern=mark_pattern) if not any_logs: logger.info('\tNo logs') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/81bd26a7/aria/cli/config/config_template.yaml ---------------------------------------------------------------------- diff --git a/aria/cli/config/config_template.yaml b/aria/cli/config/config_template.yaml index 66c606e..b8bad3b 100644 --- a/aria/cli/config/config_template.yaml +++ b/aria/cli/config/config_template.yaml @@ -9,3 +9,43 @@ logging: # main logger of the cli. provides basic descriptions for executed operations. aria.cli.main: info + + execution_logging: + formatting: + # According to verbosity level 0 - no verbose. 3 - high verbose + 0: '{message}' + 1: '{timestamp:%H:%M:%S} | {level[0]} | {message}' + 2: '{timestamp:%H:%M:%S} | {level[0]} | {implementation} | {message}' + 3: '{timestamp:%H:%M:%S} | {level[0]} | {implementation} | {inputs} | {message}' + + styling: + enabled: true + + level: + info: {'fore': 'lightmagenta_ex'} + debug: {'fore': 'lightmagenta_ex', 'style': 'dim'} + error: {'fore': 'red', 'style': 'bright'} + timestamp: + info: {'fore': 'lightmagenta_ex'} + debug: {'fore': 'lightmagenta_ex', 'style': 'dim'} + error: {'fore': 'red', 'style': 'bright'} + message: + info: {'fore': 'lightblue_ex'} + debug: {'fore': 'lightblue_ex', 'style': 'dim'} + error: {'fore': 'red', 'style': 'bright'} + implementation: + info: {'fore': 'lightblack_ex'} + debug: {'fore': 'lightblack_ex', 'style': 'dim'} + error: {'fore': 'red', 'style': 'bright'} + inputs: + info: {'fore': 'blue'} + debug: {'fore': 'blue', 'style': 'dim'} + error: {'fore': 'red', 'style': 'bright'} + traceback: + error: {'fore': 'red'} + + marker: 'lightyellow_ex' + final_states: + success: {'fore': 'green'} + cancel: {'fore': 'yellow'} + fail: {'fore': 'red'} http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/81bd26a7/aria/cli/execution_logging.py ---------------------------------------------------------------------- diff --git a/aria/cli/execution_logging.py b/aria/cli/execution_logging.py index ca966ec..66c7d9c 100644 --- a/aria/cli/execution_logging.py +++ b/aria/cli/execution_logging.py @@ -35,167 +35,136 @@ SUCCESS_STATE = 'success' CANCEL_STATE = 'cancel' FAIL_STATE = 'fail' - _EXECUTION_BASE_PATTERN = "\'.*\' workflow execution " _SUCCESSFUL_EXECUTION_PATTERN = _EXECUTION_BASE_PATTERN + "succeeded" _FAILED_EXECUTION_PATTERN = _EXECUTION_BASE_PATTERN + "failed" _CANCELED_EXECUTION_PATTERN = _EXECUTION_BASE_PATTERN + "canceled" -DEFAULT_FORMATTING = { - logger.NO_VERBOSE: {'message': '{message}'}, - logger.LOW_VERBOSE: { - MESSAGE: '{timestamp} | {level} | {message}', - LEVEL: '{level[0]}', - TIMESTAMP: '%H:%M:%S', - }, - logger.MEDIUM_VERBOSE: { - MESSAGE: '{timestamp} | {level} | {implementation} | {message} ', - LEVEL: '{level[0]}', - TIMESTAMP: '%H:%M:%S' - }, - logger.HIGH_VERBOSE: { - MESSAGE: '{timestamp} | {level} | {implementation} | {inputs} | {message} ', - LEVEL: '{level[0]}', - TIMESTAMP: '%H:%M:%S' - }, -} - -DEFAULT_STYLING = { - LEVEL: { - 'info': Color.Fore.LIGHTMAGENTA_EX, - 'debug': Color.Schema(fore=Color.Fore.LIGHTMAGENTA_EX, style=Color.Style.DIM), - 'error': Color.Schema(fore=Color.Fore.RED, style=Color.Style.BRIGHT), - }, - TIMESTAMP: { - 'info': Color.Fore.LIGHTMAGENTA_EX, - 'debug': Color.Schema(fore=Color.Fore.LIGHTMAGENTA_EX, style=Color.Style.DIM), - 'error': Color.Schema(fore=Color.Fore.RED, style=Color.Style.BRIGHT), - }, - MESSAGE: { - 'info': Color.Fore.LIGHTBLUE_EX, - 'debug': Color.Schema(fore=Color.Fore.LIGHTBLUE_EX, style=Color.Style.DIM), - 'error': Color.Schema(fore=Color.Fore.RED, style=Color.Style.BRIGHT), - }, - IMPLEMENTATION: { - 'info': Color.Fore.LIGHTBLACK_EX, - 'debug': Color.Schema(fore=Color.Fore.LIGHTBLACK_EX, style=Color.Style.DIM), - 'error': Color.Schema(fore=Color.Fore.RED, style=Color.Style.BRIGHT), - }, - INPUTS: { - 'info': Color.Fore.BLUE, - 'debug': Color.Schema(fore=Color.Fore.BLUE, style=Color.Style.DIM), - 'error': Color.Schema(fore=Color.Fore.RED, style=Color.Style.BRIGHT), - }, - TRACEBACK: {'error': Color.Fore.RED}, - - MARKER: Color.Back.LIGHTYELLOW_EX, - FINAL_STATES: { - SUCCESS_STATE: Color.Fore.GREEN, - CANCEL_STATE: Color.Fore.YELLOW, - FAIL_STATE: Color.Fore.RED, - } -} +_TIMESTAMP_PATTERN = '.*({timestamp.*?}).*' +_LEVEL_PATTERN = '.*({level.*?}).*' +_MESSAGE_PATTERN = '.*({message.*?}).*' +_IMPLEMENTATION_PATTERN = '.*({implementation.*?}).*' +_INPUTS_PATTERN = '.*({inputs.*?}).*' _PATTERNS = { SUCCESS_STATE: re.compile(_SUCCESSFUL_EXECUTION_PATTERN), CANCEL_STATE: re.compile(_CANCELED_EXECUTION_PATTERN), - FAIL_STATE: re.compile(_FAILED_EXECUTION_PATTERN) + FAIL_STATE: re.compile(_FAILED_EXECUTION_PATTERN), + IMPLEMENTATION: re.compile(_IMPLEMENTATION_PATTERN), + LEVEL: re.compile(_LEVEL_PATTERN), + MESSAGE: re.compile(_MESSAGE_PATTERN), + INPUTS: re.compile(_INPUTS_PATTERN), + TIMESTAMP: re.compile(_TIMESTAMP_PATTERN) } class _StylizedLogs(object): - def __init__(self): - self._formats = DEFAULT_FORMATTING - self._styles = DEFAULT_STYLING - self._mark_pattern = None + def _update_implementation(self, str_, implementation, log_item, mark_pattern=None): + str_ = self._stylize(str_, implementation, log_item, IMPLEMENTATION) + return self._markup(str_, implementation, mark_pattern) - def _push(self, styles=None, formats=None, mark_pattern=None): - self._styles = styles or self._styles - self._formats = formats or self._formats - self._mark_pattern = mark_pattern + def _update_inputs(self, str_, inputs, log_item, mark_pattern=None): + str_ = self._stylize(str_, inputs, log_item, INPUTS) + return self._markup(str_, inputs, mark_pattern) - def _implementation(self, implementation, log_item): - return self._stylize(implementation, log_item, IMPLEMENTATION) + def _update_timestamp(self, str_, timestamp, log_item): + return self._stylize(str_, timestamp, log_item, TIMESTAMP) - def _inputs(self, inputs, log_item): - return self._stylize(inputs, log_item, INPUTS) + def _update_message(self, str_, message, log_item, mark_pattern): + str_ = self._stylize(str_, message, log_item, MESSAGE) + return self._markup(str_, message, mark_pattern) - def _timestamp(self, timestamp, log_item): - return self._stylize(timestamp, log_item, TIMESTAMP) + def _get_traceback(self, traceback, log_item): + schema = Color.Schema(**self._styles[TRACEBACK].get(log_item.level.lower(), {})) + return Color.stylize(traceback, schema) - def _message(self, message, log_item): - return self._stylize(message, log_item, MESSAGE) + def _update_level(self, str_, log_item): + return self._stylize(str_, log_item.level[0], log_item, LEVEL) - def _traceback(self, traceback, log_item): - return self._stylize(traceback, log_item, TRACEBACK) + def _stylize(self, str_, msg, log_item, msg_type): + matched_str = self._find_pattern(_PATTERNS[msg_type], str_) + if not matched_str: + return str_ - def _level(self, log_item): - return self._stylize(log_item.level[0], log_item, LEVEL) + # if Styling was disabled + if not self._is_styling_enabled: + return str_.replace(matched_str, matched_str.format(**{msg_type: msg})) - def _stylize(self, msg, log_item, msg_type): - schema = (self._final_string_schema(log_item.msg) or - self._styles[msg_type].get(log_item.level.lower(), '')) - return Color.stylize(msg, schema) + schema = Color.Schema( + **(self._end_execution_schema(log_item.msg) or + self._styles[msg_type].get(log_item.level.lower()) or + {}) + ) + colored_string = Color.stylize(matched_str, schema) + return str_.replace(matched_str, colored_string.format(**{msg_type: msg})) - def _markup(self, str_, original_message): - if self._mark_pattern is None: + def _markup(self, str_, original_message, mark_pattern): + if mark_pattern is None: return str_ else: - regex_pattern = re.compile(self._mark_pattern) - matches = re.search(regex_pattern, original_message) + regex_pattern = re.compile(mark_pattern) + matches = re.search(regex_pattern, str(original_message)) if not matches: return str_ - return Color.markup(str_, matches, self._styles[MARKER]) + return Color.markup(str_, matches, Color.Schema(back=self._styles[MARKER])) - def _final_string_schema(self, original_message): + def _end_execution_schema(self, original_message): + if not self.is_workflow_log: + return for state, pattern in _PATTERNS.items(): if re.match(pattern, original_message): return self._styles[FINAL_STATES][state] - def __call__(self, item): - # If no formats are passed we revert to the default formats (per level) - formatting = self._formats.get(env.logging.verbosity_level, - DEFAULT_FORMATTING[env.logging.verbosity_level]) - msg = StringIO() - formatting_kwargs = dict(item=item) + def _find_pattern(self, pattern, field_value): + match = re.match(pattern, field_value) + if match: + return match.group(1) + + @property + def _verbosity_level(self): + return env.logging.verbosity_level + + @property + def _format(self): + return env.config._config['logging']['execution_logging']['formatting'][self._verbosity_level] + + @property + def _styles(self): + return env.config._config['logging']['execution_logging']['styling'] - # level - formatting_kwargs['level'] = self._level(item) + @property + def _is_styling_enabled(self): + return env.config._config['logging']['execution_logging']['styling'].get('enabled', False) + + def __call__(self, item, mark_pattern): # implementation if item.task: # operation task implementation = item.task.implementation inputs = dict(i.unwrap() for i in item.task.inputs.values()) + self.is_workflow_log = False else: # execution task implementation = item.execution.workflow_name inputs = dict(i.unwrap() for i in item.execution.inputs.values()) + self.is_workflow_log = True - formatting_kwargs['implementation'] = self._markup( - self._implementation(implementation, item), implementation) - formatting_kwargs['inputs'] = self._markup(self._inputs(inputs, item), str(inputs)) - - # timestamp - if 'timestamp' in formatting: - timestamp = item.created_at.strftime(formatting['timestamp']) - else: - timestamp = item.created_at - formatting_kwargs['timestamp'] = self._timestamp(timestamp, item) - - # message - formatting_kwargs['message'] = self._message(item.msg, item) - - # The message would be marked out if containing the provided pattern - msg.write(self._markup(formatting['message'].format(**formatting_kwargs), item.msg)) + # TODO: use the is_workflow_log + str_ = self._update_level(self._format, item) + str_ = self._update_timestamp(str_, item.created_at, item) + str_ = self._update_message(str_, item.msg, item, mark_pattern) + str_ = self._update_inputs(str_, inputs, item, mark_pattern) + str_ = self._update_implementation(str_, implementation, item, mark_pattern) + msg = StringIO(str_) # Add the exception and the error msg. if item.traceback and env.logging.verbosity_level >= logger.MEDIUM_VERBOSE: msg.write(os.linesep) for line in item.traceback.splitlines(True): - msg.write(self._traceback('\t' + '|' + line, item)) + msg.write(self._get_traceback('\t' + '|' + line, item)) return msg.getvalue() @@ -203,22 +172,14 @@ class _StylizedLogs(object): _stylize_log = _StylizedLogs() -@contextmanager -def format(styles=None, formats=None, mark_pattern=None): - original_styles = _stylize_log._styles - original_formats = _stylize_log._formats - _stylize_log._push(styles=styles, formats=formats, mark_pattern=mark_pattern) - yield - _stylize_log._push(styles=original_styles, formats=original_formats, mark_pattern=None) - - -def log(item, *args, **kwargs): - return getattr(env.logging.logger, item.level.lower())(_stylize_log(item), *args, **kwargs) +def log(item, mark_pattern=None, *args, **kwargs): + leveled_log = getattr(env.logging.logger, item.level.lower()) + return leveled_log(_stylize_log(item, mark_pattern), *args, **kwargs) -def log_list(iterator): +def log_list(iterator, mark_pattern=None): any_logs = False for item in iterator: - log(item) + log(item, mark_pattern) any_logs = True return any_logs
