Pylint tools package
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/2e14ee40 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/2e14ee40 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/2e14ee40 Branch: refs/heads/pylint-aria-events Commit: 2e14ee40747b4502182dcf006abba5620e6a1687 Parents: 66ed02a Author: Dan Kilman <[email protected]> Authored: Wed Oct 19 16:02:21 2016 +0300 Committer: Dan Kilman <[email protected]> Committed: Thu Oct 20 12:43:25 2016 +0300 ---------------------------------------------------------------------- aria/tools/application.py | 79 +++++++++++++++++++++++++----------------- aria/tools/lru_cache.py | 7 +++- aria/tools/module.py | 9 +++++ aria/tools/plugin.py | 13 +++---- aria/tools/process.py | 59 ++++++++++++++++++++++++------- aria/tools/validation.py | 32 +++++++++++++++-- 6 files changed, 145 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/2e14ee40/aria/tools/application.py ---------------------------------------------------------------------- diff --git a/aria/tools/application.py b/aria/tools/application.py index 0f36ca2..32feeff 100644 --- a/aria/tools/application.py +++ b/aria/tools/application.py @@ -13,6 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Convenience storage related tools. +# TODO rename module name +""" + import os import json import shutil @@ -25,6 +30,10 @@ from aria.exceptions import StorageError class StorageManager(LoggerMixin): + """ + Convenience wrapper to simplify work with the lower level storage mechanism + """ + def __init__( self, model_storage, @@ -51,11 +60,14 @@ class StorageManager(LoggerMixin): resource_storage, deployment_id, deployment_plan): + """ + Create a StorageManager from a deployment + """ return cls( - model_storage, - resource_storage, - deployment_id, - deployment_plan, + model_storage=model_storage, + resource_storage=resource_storage, + deployment_id=deployment_id, + deployment_plan=deployment_plan, blueprint_path=None, blueprint_plan=None, blueprint_id=None @@ -69,6 +81,9 @@ class StorageManager(LoggerMixin): blueprint_path, blueprint_id, blueprint_plan): + """ + Create a StorageManager from a blueprint + """ return cls( model_storage, resource_storage, @@ -248,32 +263,32 @@ class StorageManager(LoggerMixin): def _load_plugin_from_archive(tar_source): - if not tarfile.is_tarfile(tar_source): - # TODO: go over the exceptions - raise StorageError( - 'the provided tar archive can not be read.') + if not tarfile.is_tarfile(tar_source): + # TODO: go over the exceptions + raise StorageError( + 'the provided tar archive can not be read.') - with tarfile.open(tar_source) as tar: - tar_members = tar.getmembers() - # a wheel plugin will contain exactly one sub directory - if not tar_members: - raise StorageError( - 'archive file structure malformed. expecting exactly one ' - 'sub directory; got none.') - package_json_path = os.path.join(tar_members[0].name, - 'package.json') - try: - package_member = tar.getmember(package_json_path) - except KeyError: - raise StorageError("'package.json' was not found under {0}" - .format(package_json_path)) - try: - package_json = tar.extractfile(package_member) - except tarfile.ExtractError as e: - raise StorageError(str(e)) - try: - return json.load(package_json) - except ValueError as e: - raise StorageError("'package.json' is not a valid json: " - "{json_str}. error is {error}" - .format(json_str=package_json.read(), error=str(e))) + with tarfile.open(tar_source) as tar: + tar_members = tar.getmembers() + # a wheel plugin will contain exactly one sub directory + if not tar_members: + raise StorageError( + 'archive file structure malformed. expecting exactly one ' + 'sub directory; got none.') + package_json_path = os.path.join(tar_members[0].name, + 'package.json') + try: + package_member = tar.getmember(package_json_path) + except KeyError: + raise StorageError("'package.json' was not found under {0}" + .format(package_json_path)) + try: + package_json = tar.extractfile(package_member) + except tarfile.ExtractError as e: + raise StorageError(str(e)) + try: + return json.load(package_json) + except ValueError as e: + raise StorageError("'package.json' is not a valid json: " + "{json_str}. error is {error}" + .format(json_str=package_json.read(), error=str(e))) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/2e14ee40/aria/tools/lru_cache.py ---------------------------------------------------------------------- diff --git a/aria/tools/lru_cache.py b/aria/tools/lru_cache.py index 2cd2864..5863376 100755 --- a/aria/tools/lru_cache.py +++ b/aria/tools/lru_cache.py @@ -25,6 +25,7 @@ from collections import OrderedDict class _LRUCache(object): + def __init__(self, input_func, max_size, timeout): self._input_func = input_func self._max_size = max_size @@ -38,13 +39,17 @@ class _LRUCache(object): # in case called from a regular function - the caller is None. self._caches_dict = {} - def _prepare_key(self, *args, **kwargs): + @staticmethod + def _prepare_key(*args, **kwargs): kwargs_key = "".join( imap(lambda x: str(x) + str(type(kwargs[x])) + str(kwargs[x]), sorted(kwargs))) return "".join(imap(lambda x: str(type(x)) + str(x), args)) + kwargs_key def cache_clear(self, caller=None): + """ + Clears the cache, optionally, only for a specific caller + """ # Remove the cache for the caller, only if exists: if caller in self._caches_dict: del self._caches_dict[caller] http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/2e14ee40/aria/tools/module.py ---------------------------------------------------------------------- diff --git a/aria/tools/module.py b/aria/tools/module.py index 535f7aa..3afc0ff 100644 --- a/aria/tools/module.py +++ b/aria/tools/module.py @@ -13,10 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Utility methods for dynamically loading python code +""" + import importlib def load_attribute(attribute_path): + """ + Dynamically load an attribute based on the path to it. + e.g. some_package.some_module.some_attribute, will load the some_attribute from the + some_package.some_module module + """ module_name, attribute_name = attribute_path.rsplit('.', 1) try: module = importlib.import_module(module_name) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/2e14ee40/aria/tools/plugin.py ---------------------------------------------------------------------- diff --git a/aria/tools/plugin.py b/aria/tools/plugin.py index 976ac93..bb2b974 100644 --- a/aria/tools/plugin.py +++ b/aria/tools/plugin.py @@ -13,18 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Contains utility methods that enable dynamic python code loading +# TODO: merge with tools.module +""" + import os from importlib import import_module def plugin_installer(path, plugin_suffix, package=None, callback=None): """ - - :param path: - :param plugin_suffix: - :param package: - :param callback: - :return: + Load each module under ``path`` that ends with ``plugin_suffix``. If ``callback`` is supplied, + call it with each loaded module. """ assert callback is None or callable(callback) plugin_suffix = '{0}.py'.format(plugin_suffix) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/2e14ee40/aria/tools/process.py ---------------------------------------------------------------------- diff --git a/aria/tools/process.py b/aria/tools/process.py index 7df0026..5a3d8a0 100644 --- a/aria/tools/process.py +++ b/aria/tools/process.py @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Subprocess utility methods +""" + import os import subprocess from signal import SIGKILL @@ -23,6 +27,10 @@ from aria.exceptions import ExecutorException, ProcessException class Process(LoggerMixin): + """ + Subprocess wrapper + """ + def __init__( self, args, @@ -32,12 +40,7 @@ class Process(LoggerMixin): env=None, **kwargs): """ - Process class - subprocess wrapper - :param args: - :param stdout: - :param stderr: - :param cwd: - :param env: + Subprocess wrapper """ super(Process, self).__init__(**kwargs) self.args = args @@ -48,7 +51,7 @@ class Process(LoggerMixin): self._stderr = stderr def __repr__(self): - return '{cls.__name__}(args=self.args, cwd=self.cwd)'.format( + return '{cls.__name__}(args={self.args}, cwd={self.cwd})'.format( cls=self.__class__, self=self) def __getattr__(self, item): @@ -56,25 +59,40 @@ class Process(LoggerMixin): @property def name(self): + """ + The process name + """ return self.args[0] @property def pid(self): + """ + The process pid (if running) + """ if self.is_running(): return self.process.pid @property def stdout(self): - assert self.process, 'Need to run before calling thie method' + """ + The process stdout + """ + assert self.process, 'Need to run before calling this method' return self.process.stdout @property def stderr(self): - assert self.process, 'Need to run before calling thie method' + """ + The process stderr + """ + assert self.process, 'Need to run before calling this method' return self.process.stderr @property def return_code(self): + """ + The process return code. Will wait for process to end if it hasn't already + """ if self.process is None: return None if self.is_running(): @@ -85,6 +103,10 @@ class Process(LoggerMixin): return self.process.returncode def terminate(self): + """ + Terminates the process by sending a SIGTERM to it. If the process did not stop after that, + sends a SIGKILL with 1 second interval for a maximum of 10 times. + """ if self.process is not None and self.process.poll() is None: self.logger.debug('terminating process {0:d} ({1})'.format(self.process.pid, self.name)) self.process.terminate() @@ -96,16 +118,22 @@ class Process(LoggerMixin): sleep(1) kill_attempts += 1 - def killpg(self): + def kill(self): + """ + Kill the process by sending a SIGKILL to it + """ if self.is_running(): os.killpg(os.getpgid(self.pid), SIGKILL) def is_running(self): + """ + Returns ``True`` if the process is currently running + """ return self.process.poll() is None if self.process else False def wait(self): """ - Block till child process finishes + Block until process finishes """ assert self.process, 'Need to run before calling thie method' self.process.wait() @@ -113,8 +141,6 @@ class Process(LoggerMixin): def run(self, nice=None, universal_newlines=True): """ Run the child process. This call does not block. - :param int nice: nice on the child process run - :param bool universal_newlines: """ self.logger.debug('Running child process: {0}'.format(' '.join(self.args))) self.process = subprocess.Popen( @@ -128,6 +154,9 @@ class Process(LoggerMixin): universal_newlines=universal_newlines) def run_in_shell(self, nice=None, universal_newlines=True): + """ + Run the child process in a shell. This call does not block. + """ command = ' '.join(self.args) self.logger.debug('Running child process in shell: {0}'.format(command)) self.process = subprocess.Popen( @@ -142,6 +171,10 @@ class Process(LoggerMixin): universal_newlines=universal_newlines) def raise_failure(self): + """ + Raise a ProcessException if the process terminated with a non zero return code. Will wait + for the process to finish if it hasn't already + """ if self.is_running(): self.wait() if self.return_code == 0: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/2e14ee40/aria/tools/validation.py ---------------------------------------------------------------------- diff --git a/aria/tools/validation.py b/aria/tools/validation.py index 1de995b..ea1dae7 100644 --- a/aria/tools/validation.py +++ b/aria/tools/validation.py @@ -13,14 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Contains validation related utilities +""" + class ValidatorMixin(object): + """ + A mixin that should be added to classes that require validating user input + """ + _ARGUMENT_TYPE_MESSAGE = '{name} argument must be {type} based, got {arg!r}' _ACTION_MESSAGE = 'action arg options: {actions}, got {action}' _ARGUMENT_CHOICE_MESSAGE = '{name} argument must be in {choices}, got {arg!r}' @classmethod def validate_actions(cls, action): + """ + Validate action is defined in the class ``ACTIONS`` attribute + """ # todo: remove this and use validate choice if action not in cls.ACTIONS: raise TypeError(cls._ACTION_MESSAGE.format( @@ -28,33 +39,50 @@ class ValidatorMixin(object): @classmethod def validate_in_choice(cls, name, argument, choices): + """ + Validate ``argument`` is in ``choices`` + """ if argument not in choices: raise TypeError(cls._ARGUMENT_CHOICE_MESSAGE.format( name=name, choices=choices, arg=argument)) @classmethod def validate_type(cls, argument_name, argument, expected_type): + """ + Validate ``argument`` is a subclass of ``expected_type`` + """ if not issubclass(argument, expected_type): raise TypeError(cls._ARGUMENT_TYPE_MESSAGE.format( name=argument_name, type=expected_type, arg=argument)) @classmethod def validate_instance(cls, argument_name, argument, expected_type): + """ + Validate ``argument`` is a instance of ``expected_type`` + """ if not isinstance(argument, expected_type): raise TypeError(cls._ARGUMENT_TYPE_MESSAGE.format( name=argument_name, type=expected_type, arg=argument)) @classmethod def validate_callable(cls, argument_name, argument): + """ + Validate ``argument`` is callable + """ if not callable(argument): raise TypeError(cls._ARGUMENT_TYPE_MESSAGE.format( name=argument_name, type='callable', arg=argument)) def validate_function_arguments(func, func_kwargs): - _KWARGS_FLAG = 8 + """ + Validates all required arguments are supplied to ``func`` and that no additional arguments are + supplied + """ + + _kwargs_flags = 8 - has_kwargs = func.func_code.co_flags & _KWARGS_FLAG != 0 + has_kwargs = func.func_code.co_flags & _kwargs_flags != 0 args_count = func.func_code.co_argcount # all args without the ones with default values
