AMBARI-3656. Resource Manager. Params subtitution: allow dictionaries in dictionaries, allow default prefixes (Andrew Onischuk via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/cefc54a6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/cefc54a6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/cefc54a6 Branch: refs/heads/trunk Commit: cefc54a63c55974cdd568f19387a3f0ff6b44075 Parents: 451222f Author: Lisnichenko Dmitro <[email protected]> Authored: Fri Nov 1 17:12:47 2013 +0200 Committer: Lisnichenko Dmitro <[email protected]> Committed: Fri Nov 1 17:12:47 2013 +0200 ---------------------------------------------------------------------- .../src/main/python/resource_management/base.py | 34 ++------ .../python/resource_management/environment.py | 84 +++++++++++++++++++- .../main/python/resource_management/utils.py | 69 +++++++++++++++- 3 files changed, 156 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/cefc54a6/ambari-agent/src/main/python/resource_management/base.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/resource_management/base.py b/ambari-agent/src/main/python/resource_management/base.py index dae745a..35aeefe 100644 --- a/ambari-agent/src/main/python/resource_management/base.py +++ b/ambari-agent/src/main/python/resource_management/base.py @@ -4,10 +4,8 @@ __all__ = ["Resource", "ResourceArgument", "ForcedListArgument", "BooleanArgument"] import logging -from string import Template -from resource_management.environment import Environment from resource_management.exceptions import Fail, InvalidArgument - +from resource_management.environment import Environment, Substitutor class ResourceArgument(object): def __init__(self, default=None, required=False, allow_override=False): @@ -71,8 +69,8 @@ class ResourceMetaclass(type): value.name = key mcs._arguments[key] = value setattr(mcs, key, Accessor(key)) - - + + class Resource(object): __metaclass__ = ResourceMetaclass @@ -96,8 +94,8 @@ class Resource(object): name = name[0] - name = Resource.subsitute_params(name) env = env or Environment.get_instance() + name = Substitutor.substitute(name) provider = provider or getattr(cls, 'provider', None) r_type = cls.__name__ @@ -124,8 +122,9 @@ class Resource(object): if hasattr(self, 'name'): return - self.name = Resource.subsitute_params(name) self.env = env or Environment.get_instance() + self.name = Substitutor.substitute(name) + self.provider = provider or getattr(self, 'provider', None) self.arguments = {} @@ -136,7 +135,7 @@ class Resource(object): raise Fail("%s received unsupported argument %s" % (self, key)) else: try: - self.arguments[key] = Resource.subsitute_params(arg.validate(value)) + self.arguments[key] = Substitutor.substitute(arg.validate(value)) except InvalidArgument, exc: raise InvalidArgument("%s %s" % (self, exc)) @@ -156,25 +155,6 @@ class Resource(object): self.subscribe(*sub) self.validate() - - @staticmethod - def subsitute_params(val): - env = Environment.get_instance() - - if env.config.params and isinstance(val, str): - try: - # use 'safe_substitute' to ignore failures - result = Template(val).substitute(env.config.params) - if '$' in val: - Resource.log.debug("%s after substitution is %s", val, result) - return result - except KeyError as ex: - key_name = '$'+str(ex).strip("'") - raise Fail("Configuration %s not found" % key_name) - - return val - - def validate(self): pass http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/cefc54a6/ambari-agent/src/main/python/resource_management/environment.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/resource_management/environment.py b/ambari-agent/src/main/python/resource_management/environment.py index c1f84c5..c9cb6e0 100644 --- a/ambari-agent/src/main/python/resource_management/environment.py +++ b/ambari-agent/src/main/python/resource_management/environment.py @@ -11,8 +11,9 @@ from datetime import datetime from resource_management import shell from resource_management.exceptions import Fail from resource_management.providers import find_provider -from resource_management.utils import AttributeDictionary +from resource_management.utils import AttributeDictionary, ParamsAttributeDictionary from resource_management.system import System +from string import Template class Environment(object): @@ -32,6 +33,7 @@ class Environment(object): self.config = AttributeDictionary() self.resources = {} self.resource_list = [] + Substitutor.default_prefixes = [] self.delayed_actions = set() self.update_config({ # current time @@ -43,7 +45,7 @@ class Environment(object): # dir where templates,failes dirs are 'basedir': basedir, # variables, which can be used in templates - 'params': params, + 'params': ParamsAttributeDictionary(Substitutor, params), }) def backup_file(self, path): @@ -87,6 +89,9 @@ class Environment(object): self.log.info( "%s sending %s action to %s (delayed)" % (resource, action, res)) self.delayed_actions |= resource.subscriptions['delayed'] + + def set_default_prefixes(self, dict): + Substitutor.default_prefixes = dict def _check_condition(self, cond): if hasattr(cond, '__call__'): @@ -157,3 +162,78 @@ class Environment(object): self.resources = state['resources'] self.resource_list = state['resource_list'] self.delayed_actions = state['delayed_actions'] + + +class Substitutor(): + log = logging.getLogger("resource_management.resource") + default_prefixes = [] + + class ExtendedTemplate(Template): + """ + This is done to support substitution of dictionaries in dictionaries + ( ':' sign) + + default is: + idpattern = r'[_a-z][_a-z0-9]*' + """ + idpattern = r'[_a-z][_a-z0-9:]*' + + @staticmethod + def _get_subdict(name, dic): + """ + "a:b:c" => a[b][c] + + doesn't use prefixes + """ + name_parts = name.split(':') + curr = dic + + for x in name_parts: + curr = curr[x] + return curr + + @staticmethod + def get_subdict(name, dic): + """ + "a:b:c" => a[b][c] + + can use prefixes + """ + prefixes = list(Substitutor.default_prefixes) + prefixes.insert(0, None) # for not prefixed case + name_parts = name.split(':') + is_found = False + result = None + + for prefix in prefixes: + curr = Substitutor._get_subdict(prefix,dic) if prefix else dic + + try: + for x in name_parts: + curr = curr[x] + except (KeyError, TypeError): + continue + + if is_found: + raise Fail("Variable ${%s} found more than one time, please check your default prefixes!" % name) + + is_found = True + result = curr + + if not result: + raise Fail("Configuration on ${%s} cannot be resolved" % name) + + return result + + @staticmethod + def substitute(val): + env = Environment.get_instance() + dic = env.config.params + + if dic and isinstance(val, str): + result = Substitutor.ExtendedTemplate(val).substitute(dic) + if '$' in val: + Substitutor.log.debug("%s after substitution is %s", val, result) + return result + + return val http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/cefc54a6/ambari-agent/src/main/python/resource_management/utils.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/resource_management/utils.py b/ambari-agent/src/main/python/resource_management/utils.py index ff99e1a..4a00576 100644 --- a/ambari-agent/src/main/python/resource_management/utils.py +++ b/ambari-agent/src/main/python/resource_management/utils.py @@ -14,8 +14,7 @@ class AttributeDictionary(object): try: return self[name] except KeyError: - raise AttributeError( - "'%s' object has no attribute '%s'" % (self.__class__.__name__, name)) + raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, name)) def __setitem__(self, name, value): self._dict[name] = self._convert_value(value) @@ -66,3 +65,69 @@ class AttributeDictionary(object): def __setstate__(self, state): super(AttributeDictionary, self).__setattr__("_dict", state) + +class ParamsAttributeDictionary(object): + """ + This class can store user parameters + and support some features necessary for substitution to work. + """ + def __init__(self, substitutor, *args, **kwargs): + d = kwargs + if len(args)==1: + d = args[0] + super(ParamsAttributeDictionary, self).__setattr__("_dict", d) + super(ParamsAttributeDictionary, self).__setattr__("substitutor", substitutor) + + def __setattr__(self, name, value): + self[name] = value + + def __setitem__(self, name, value): + self._dict[name] = self._convert_value(value) + + def __getitem__(self, name): + val = self.substitutor.get_subdict(name, self._dict) + return self._convert_value(val) + + def _convert_value(self, value): + if isinstance(value, dict) and not isinstance(value, ParamsAttributeDictionary): + return ParamsAttributeDictionary(self.substitutor, value) + return value + + def copy(self): + return self.__class__(self._dict.copy()) + + def update(self, *args, **kwargs): + self._dict.update(*args, **kwargs) + + def items(self): + return self._dict.items() + + def values(self): + return self._dict.values() + + def keys(self): + return self._dict.keys() + + def pop(self, *args, **kwargs): + return self._dict.pop(*args, **kwargs) + + def get(self, *args, **kwargs): + return self._dict.get(*args, **kwargs) + + def __repr__(self): + return self._dict.__repr__() + + def __unicode__(self): + return self._dict.__unicode__() + + def __str__(self): + return self._dict.__str__() + + def __iter__(self): + return self._dict.__iter__() + + def __getstate__(self): + return self._dict + + def __setstate__(self, state): + super(ParamsAttributeDictionary, self).__setattr__("_dict", state) \ No newline at end of file
