http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/core/system.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/system.py b/ambari-common/src/main/python/resource_management/core/system.py new file mode 100644 index 0000000..d7dab7d --- /dev/null +++ b/ambari-common/src/main/python/resource_management/core/system.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +__all__ = ["System"] + +import os +import sys +import platform +from resource_management.core import shell +from resource_management.core.exceptions import Fail +from functools import wraps +from ambari_commons import OSCheck + +def lazy_property(undecorated): + name = '_' + undecorated.__name__ + + @property + @wraps(undecorated) + def decorated(self): + try: + return getattr(self, name) + except AttributeError: + v = undecorated(self) + setattr(self, name, v) + return v + + return decorated + +class System(object): + @lazy_property + def os(self): + """ + Return values: + linux, unknown + + In case cannot detect raises 'unknown' + """ + platform = sys.platform + if platform.startswith('linux'): + return "linux" + else: + return "unknown" + + @lazy_property + def os_version(self): + """ + Example return value: + "6.3" for "Centos 6.3" + + In case cannot detect --> Fail + """ + return OSCheck.get_os_version() + + @lazy_property + def os_major_version(self): + """ + Example return value: + "6" for "Centos 6.3" + + In case cannot detect --> Fail + """ + return OSCheck.get_os_major_version() + + @lazy_property + def os_release_name(self): + """ + For Ubuntu 12.04: + precise + """ + return OSCheck.get_os_release_name() + + @lazy_property + def os_type(self): + """ + Return values: + redhat, fedora, centos, oraclelinux, ascendos, + amazon, xenserver, oel, ovs, cloudlinux, slc, scientific, psbm, + ubuntu, debian, sles, sled, opensuse, suse ... and others + + In case cannot detect raises exception. + """ + return OSCheck.get_os_type() + + @lazy_property + def os_family(self): + """ + Return values: + redhat, debian, suse + + In case cannot detect raises exception + """ + return OSCheck.get_os_family() + + @lazy_property + def ec2(self): + if not os.path.exists("/proc/xen"): + return False + if os.path.exists("/etc/ec2_version"): + return True + return False + + @lazy_property + def vm(self): + if os.path.exists("/usr/bin/VBoxControl"): + return "vbox" + elif os.path.exists("/usr/bin/vmware-toolbox-cmd") or os.path.exists( + "/usr/sbin/vmware-toolbox-cmd"): + return "vmware" + elif os.path.exists("/proc/xen"): + return "xen" + return None + + @lazy_property + def arch(self): + machine = self.machine + if machine in ("i386", "i486", "i686"): + return "x86_32" + return machine + + @lazy_property + def machine(self): + code, out = shell.call(["/bin/uname", "-m"]) + return out.strip() + + @lazy_property + def locales(self): + code, out = shell.call("locale -a") + return out.strip().split("\n") + + @classmethod + def get_instance(cls): + try: + return cls._instance + except AttributeError: + cls._instance = cls() + return cls._instance + + def unquote(self, val): + if val[0] == '"': + val = val[1:-1] + return val
http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/core/utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/utils.py b/ambari-common/src/main/python/resource_management/core/utils.py new file mode 100644 index 0000000..52a12b3 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/core/utils.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management.core.exceptions import Fail + +class AttributeDictionary(object): + def __init__(self, *args, **kwargs): + d = kwargs + if args: + d = args[0] + super(AttributeDictionary, self).__setattr__("_dict", d) + + def __setattr__(self, name, value): + self[name] = value + + def __getattr__(self, name): + if name in self.__dict__: + return self.__dict__[name] + try: + return self[name] + except KeyError: + raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, name)) + + def __setitem__(self, name, value): + self._dict[name] = self._convert_value(value) + + def __getitem__(self, name): + return self._convert_value(self._dict[name]) + + def _convert_value(self, value): + if isinstance(value, dict) and not isinstance(value, AttributeDictionary): + return AttributeDictionary(value) + return value + + def copy(self): + return self.__class__(self._dict.copy()) + + def update(self, *args, **kwargs): + self._dict.update(*args, **kwargs) + + def items(self): + return self._dict.items() + + def iteritems(self): + return self._dict.iteritems() + + def values(self): + return self._dict.values() + + def keys(self): + return self._dict.keys() + + def pop(self, *args, **kwargs): + return self._dict.pop(*args, **kwargs) + + def get(self, *args, **kwargs): + return self._dict.get(*args, **kwargs) + + def __repr__(self): + return self._dict.__repr__() + + def __unicode__(self): + if isinstance(self._dict, str): + return self._dict.__unicode__() + else: + return str(self._dict) + + def __str__(self): + return self._dict.__str__() + + def __iter__(self): + return self._dict.__iter__() + + def __getstate__(self): + return self._dict + + def __setstate__(self, state): + super(AttributeDictionary, self).__setattr__("_dict", state) + +def checked_unite(dict1, dict2): + for key in dict1: + if key in dict2: + if not dict2[key] is dict1[key]: # it's not a big deal if this is the same variable + raise Fail("Variable '%s' already exists more than once as a variable/configuration/kwarg parameter. Cannot evaluate it." % key) + + result = dict1.copy() + result.update(dict2) + + return result http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/__init__.py b/ambari-common/src/main/python/resource_management/libraries/__init__.py new file mode 100644 index 0000000..b1d3a36 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/__init__.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management.libraries.functions import * +from resource_management.libraries.resources import * +from resource_management.libraries.providers import * +from resource_management.libraries.script import * http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py new file mode 100644 index 0000000..ec2fdeb --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management.libraries.functions.default import * +from resource_management.libraries.functions.format import * +from resource_management.libraries.functions.get_kinit_path import * +from resource_management.libraries.functions.get_unique_id_and_date import * +from resource_management.libraries.functions.check_process_status import * +from resource_management.libraries.functions.is_empty import * +from resource_management.libraries.functions.substitute_vars import * +from resource_management.libraries.functions.get_port_from_url import * http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py b/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py new file mode 100644 index 0000000..7fdecdc --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.core.logger import Logger +__all__ = ["check_process_status"] + +import os + +def check_process_status(pid_file): + """ + Function checks whether process is running. + Process is considered running, if pid file exists, and process with + a pid, mentioned in pid file is running + If process is not running, will throw ComponentIsNotRunning exception + + @param pid_file: path to service pid file + """ + if not pid_file or not os.path.isfile(pid_file): + raise ComponentIsNotRunning() + with open(pid_file, "r") as f: + try: + pid = int(f.read()) + except: + Logger.debug("Pid file {0} does not exist".format(pid_file)) + raise ComponentIsNotRunning() + try: + # Kill will not actually kill the process + # From the doc: + # If sig is 0, then no signal is sent, but error checking is still + # performed; this can be used to check for the existence of a + # process ID or process group ID. + os.kill(pid, 0) + except OSError: + Logger.debug("Process with pid {0} is not running. Stale pid file" + " at {1}".format(pid, pid_file)) + raise ComponentIsNotRunning() + pass http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/functions/default.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/default.py b/ambari-common/src/main/python/resource_management/libraries/functions/default.py new file mode 100644 index 0000000..733c03a --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/default.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +__all__ = ["default"] +from resource_management.libraries.script import Script +from resource_management.libraries.script.config_dictionary import UnknownConfiguration +from resource_management.core.logger import Logger + +def default(name, default_value): + subdicts = filter(None, name.split('/')) + + curr_dict = Script.get_config() + for x in subdicts: + if x in curr_dict: + curr_dict = curr_dict[x] + else: + if not isinstance(default_value, UnknownConfiguration): + Logger.debug("Cannot find configuration: '%s'. Using '%s' value as default" % (name, default_value)) + return default_value + + return curr_dict \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/functions/format.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/format.py b/ambari-common/src/main/python/resource_management/libraries/functions/format.py new file mode 100644 index 0000000..efbacd7 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/format.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +__all__ = ["format"] +import sys +from string import Formatter +from resource_management.core.exceptions import Fail +from resource_management.core.utils import checked_unite +from resource_management.core.environment import Environment +from resource_management.core.logger import Logger +from resource_management.core.shell import quote_bash_args + + +class ConfigurationFormatter(Formatter): + """ + Flags: + !e - escape bash properties flag + !h - hide sensitive information from the logs + !p - password flag, !p=!s+!e. Has both !e, !h effect + """ + def format(self, format_string, *args, **kwargs): + env = Environment.get_instance() + variables = kwargs + params = env.config.params + all_params = checked_unite(variables, params) + + self.convert_field = self.convert_field_protected + result_protected = self.vformat(format_string, args, all_params) + + self.convert_field = self.convert_field_unprotected + result_unprotected = self.vformat(format_string, args, all_params) + + if result_protected != result_unprotected: + Logger.sensitive_strings[result_unprotected] = result_protected + + return result_unprotected + + def convert_field_unprotected(self, value, conversion): + return self._convert_field(value, conversion, False) + + def convert_field_protected(self, value, conversion): + """ + Enable masking sensitive information like + passwords from logs via !p (password) format flag. + """ + return self._convert_field(value, conversion, True) + + def _convert_field(self, value, conversion, is_protected): + if conversion == 'e': + return quote_bash_args(str(value)) + elif conversion == 'h': + return "[PROTECTED]" if is_protected else value + elif conversion == 'p': + return "[PROTECTED]" if is_protected else self._convert_field(value, 'e', is_protected) + + return super(ConfigurationFormatter, self).convert_field(value, conversion) + + +def format(format_string, *args, **kwargs): + variables = sys._getframe(1).f_locals + + result = checked_unite(kwargs, variables) + result.pop("self", None) # self kwarg would result in an error + return ConfigurationFormatter().format(format_string, args, **result) http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/functions/get_kinit_path.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_kinit_path.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_kinit_path.py new file mode 100644 index 0000000..74d331d --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_kinit_path.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +__all__ = ["get_kinit_path"] +import os + +def get_kinit_path(pathes_list): + """ + @param pathes: comma separated list + """ + kinit_path = "" + + for x in pathes_list: + if not x: + continue + + path = os.path.join(x,"kinit") + + if os.path.isfile(path): + kinit_path = path + break + + return kinit_path http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/functions/get_port_from_url.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_port_from_url.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_port_from_url.py new file mode 100644 index 0000000..70bd2d7 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_port_from_url.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * +from resource_management.libraries.functions.is_empty import * +from resource_management.core.exceptions import Fail +import re + +def get_port_from_url(address): + """ + Return port from URL. If address is UnknownConfiguration, + UnknownConfiguration will be returned. If no port was found, Fail will be + raised. + """ + if not is_empty(address): + port = re.findall(":([\d]{1,5})(?=/|$)", address) + if port: + return port[0] + raise Fail("No port in URL:{0}".format(address)) + else: + return address \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py new file mode 100644 index 0000000..a79a1e5 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +__all__ = ["get_unique_id_and_date"] +import datetime +from resource_management.core import shell + +def get_unique_id_and_date(): + out = shell.checked_call("hostid")[1].split('\n')[-1] # bugfix: take the lastline (stdin is not tty part cut) + id = out.strip() + + now = datetime.datetime.now() + date = now.strftime("%M%d%y") + + return "id{id}_date{date}".format(id=id, date=date) http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/functions/is_empty.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/is_empty.py b/ambari-common/src/main/python/resource_management/libraries/functions/is_empty.py new file mode 100644 index 0000000..f920d02 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/is_empty.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +from resource_management.libraries.script.config_dictionary import UnknownConfiguration + +def is_empty(var): + """ + Check if certain configuration sent from the server has been received. + """ + return isinstance(var, UnknownConfiguration) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/functions/substitute_vars.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/substitute_vars.py b/ambari-common/src/main/python/resource_management/libraries/functions/substitute_vars.py new file mode 100644 index 0000000..2036208 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/substitute_vars.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +import re + +_MAX_SUBST = 20 + +def substitute_vars(raw, config): + """ + @param raw: str (e.g '${hbase.tmp.dir}/local') + @param config: dict (e.g {'hbase.tmp.dir': '/hadoop/hbase'}) + """ + result = raw + + pattern = re.compile("\$\{[^\}\$\x0020]+\}") + + for depth in range(0, _MAX_SUBST - 1): + match = pattern.search(result) + + if match: + start = match.start() + end = match.end() + + name = result[start + 2 : end - 1] + + try: + value = config[name] + except KeyError: + return result + + result = result[:start] + value + result[end:] + else: + break + + return result http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py b/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py new file mode 100644 index 0000000..6d8a8f7 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +PROVIDERS = dict( + redhat=dict( + Repository="resource_management.libraries.providers.repository.RhelSuseRepositoryProvider", + ), + suse=dict( + Repository="resource_management.libraries.providers.repository.RhelSuseRepositoryProvider", + ), + debian=dict( + Repository="resource_management.libraries.providers.repository.DebianRepositoryProvider", + ), + default=dict( + ExecuteHadoop="resource_management.libraries.providers.execute_hadoop.ExecuteHadoopProvider", + TemplateConfig="resource_management.libraries.providers.template_config.TemplateConfigProvider", + XmlConfig="resource_management.libraries.providers.xml_config.XmlConfigProvider", + PropertiesFile="resource_management.libraries.providers.properties_file.PropertiesFileProvider", + MonitorWebserver="resource_management.libraries.providers.monitor_webserver.MonitorWebserverProvider", + HdfsDirectory="resource_management.libraries.providers.hdfs_directory.HdfsDirectoryProvider", + CopyFromLocal="resource_management.libraries.providers.copy_from_local.CopyFromLocalProvider" + ), +) http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py b/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py new file mode 100644 index 0000000..9031a77 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import os +from resource_management import * + +class CopyFromLocalProvider(Provider): + def action_run(self): + path = self.resource.path + dest_dir = self.resource.dest_dir + kinnit_if_needed = self.resource.kinnit_if_needed + owner = self.resource.owner + group = self.resource.group + mode = self.resource.mode + hdfs_usr=self.resource.hdfs_user + hadoop_conf_path = self.resource.hadoop_conf_dir + + copy_cmd = format("fs -copyFromLocal {path} {dest_dir}") + dest_file_name = os.path.split(path)[1] + dest_path = dest_dir + dest_file_name if dest_dir.endswith(os.sep) else dest_dir + os.sep + dest_file_name + # Need to run unless as resource user + su_cmd = 'su - {0} -c'.format(owner) + unless_cmd = format("{su_cmd} '{kinnit_if_needed} hadoop fs -ls {dest_path}' >/dev/null 2>&1") + + ExecuteHadoop(copy_cmd, + not_if=unless_cmd, + user=owner, + conf_dir=hadoop_conf_path + ) + + if not owner: + chown = None + else: + if not group: + chown = owner + else: + chown = format('{owner}:{group}') + + if chown: + chown_cmd = format("fs -chown {chown} {dest_path}") + + ExecuteHadoop(chown_cmd, + user=hdfs_usr, + conf_dir=hadoop_conf_path) + pass + + if mode: + dir_mode = oct(mode)[1:] + chmod_cmd = format('fs -chmod {dir_mode} {dest_path}') + + ExecuteHadoop(chmod_cmd, + user=hdfs_usr, + conf_dir=hadoop_conf_path) + pass http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/providers/execute_hadoop.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/execute_hadoop.py b/ambari-common/src/main/python/resource_management/libraries/providers/execute_hadoop.py new file mode 100644 index 0000000..8ab71ff --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/providers/execute_hadoop.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * + +class ExecuteHadoopProvider(Provider): + def action_run(self): + kinit__path_local = self.resource.kinit_path_local + keytab = self.resource.keytab + conf_dir = self.resource.conf_dir + command = self.resource.command + principal = self.resource.principal + + if isinstance(command, (list, tuple)): + command = ' '.join(quote_bash_args(x) for x in command) + + with Environment.get_instance_copy() as env: + if self.resource.security_enabled and not self.resource.kinit_override: + Execute (format("{kinit__path_local} -kt {keytab} {principal}"), + path = ['/bin'], + user = self.resource.user + ) + + Execute (format("hadoop --config {conf_dir} {command}"), + user = self.resource.user, + tries = self.resource.tries, + try_sleep = self.resource.try_sleep, + logoutput = self.resource.logoutput, + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_directory.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_directory.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_directory.py new file mode 100644 index 0000000..08ac9cd --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_directory.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * +directories_list = [] #direcotries list for mkdir +chmod_map = {} #(mode,recursive):dir_list map +chown_map = {} #(owner,group,recursive):dir_list map +class HdfsDirectoryProvider(Provider): + def action_create_delayed(self): + global delayed_directories + global chmod_map + global chown_map + + if not self.resource.dir_name: + return + + dir_name = self.resource.dir_name + dir_owner = self.resource.owner + dir_group = self.resource.group + dir_mode = oct(self.resource.mode)[1:] if self.resource.mode else None + directories_list.append(self.resource.dir_name) + + recursive_chown_str = "-R" if self.resource.recursive_chown else "" + recursive_chmod_str = "-R" if self.resource.recursive_chmod else "" + # grouping directories by mode/owner/group to modify them in one 'chXXX' call + if dir_mode: + chmod_key = (dir_mode,recursive_chmod_str) + if chmod_map.has_key(chmod_key): + chmod_map[chmod_key].append(dir_name) + else: + chmod_map[chmod_key] = [dir_name] + + if dir_owner: + owner_key = (dir_owner,dir_group,recursive_chown_str) + if chown_map.has_key(owner_key): + chown_map[owner_key].append(dir_name) + else: + chown_map[owner_key] = [dir_name] + + def action_create(self): + global delayed_directories + global chmod_map + global chown_map + + self.action_create_delayed() + + hdp_conf_dir = self.resource.conf_dir + hdp_hdfs_user = self.resource.hdfs_user + secured = self.resource.security_enabled + keytab_file = self.resource.keytab + kinit_path = self.resource.kinit_path_local + + chmod_commands = [] + chown_commands = [] + + for chmod_key, chmod_dirs in chmod_map.items(): + mode = chmod_key[0] + recursive = chmod_key[1] + chmod_dirs_str = ' '.join(chmod_dirs) + chmod_commands.append(format("hadoop fs -chmod {recursive} {mode} {chmod_dirs_str}")) + + for chown_key, chown_dirs in chown_map.items(): + owner = chown_key[0] + group = chown_key[1] + recursive = chown_key[2] + chown_dirs_str = ' '.join(chown_dirs) + if owner: + chown = owner + if group: + chown = format("{owner}:{group}") + chown_commands.append(format("hadoop fs -chown {recursive} {chown} {chown_dirs_str}")) + + if secured: + Execute(format("{kinit_path} -kt {keytab_file} {hdp_hdfs_user}"), + user=hdp_hdfs_user) + #create all directories in one 'mkdir' call + dir_list_str = ' '.join(directories_list) + #for hadoop 2 we need to specify -p to create directories recursively + parent_flag = '`rpm -q hadoop | grep -q "hadoop-1" || echo "-p"`' + + Execute(format('hadoop fs -mkdir {parent_flag} {dir_list_str} && {chmod_cmd} && {chown_cmd}', + chmod_cmd=' && '.join(chmod_commands), + chown_cmd=' && '.join(chown_commands)), + user=hdp_hdfs_user, + not_if=format("su - {hdp_hdfs_user} -c 'hadoop fs -ls {dir_list_str}'") + ) + + directories_list[:] = [] + chmod_map.clear() + chown_map.clear() http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/providers/monitor_webserver.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/monitor_webserver.py b/ambari-common/src/main/python/resource_management/libraries/providers/monitor_webserver.py new file mode 100644 index 0000000..3aef22e --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/providers/monitor_webserver.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * +from resource_management.core.system import System + + +class MonitorWebserverProvider(Provider): + def action_start(self): + self.get_serivice_params() + self.enable_keep_alive() + service_name = self.service_name + Execute(format("/etc/init.d/{service_name} start")) + + def action_stop(self): + self.get_serivice_params() + service_name = self.service_name + Execute(format("/etc/init.d/{service_name} stop")) + + def action_restart(self): + self.action_stop() + self.action_start() + + def get_serivice_params(self): + self.system = System.get_instance() + if self.system.os_family in ["suse","debian"]: + self.service_name = "apache2" + self.httpd_conf_dir = '/etc/apache2' + else: + self.service_name = "httpd" + self.httpd_conf_dir = '/etc/httpd/conf' + + def enable_keep_alive(self): + httpd_conf_dir = self.httpd_conf_dir + Execute(format( + "grep -E 'KeepAlive (On|Off)' {httpd_conf_dir}/httpd.conf && sed -i 's/KeepAlive Off/KeepAlive On/' {httpd_conf_dir}/httpd.conf || echo 'KeepAlive On' >> {httpd_conf_dir}/httpd.conf")) http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/providers/properties_file.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/properties_file.py b/ambari-common/src/main/python/resource_management/libraries/providers/properties_file.py new file mode 100644 index 0000000..94a51ff --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/providers/properties_file.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import time +import os +from resource_management import * + +class PropertiesFileProvider(Provider): + def action_create(self): + filename = self.resource.filename + dir = self.resource.dir + if dir == None: + filepath = filename + else: + filepath = os.path.join(dir, filename) + + config_content = InlineTemplate('''# Generated by Apache Ambari. {{time.asctime(time.localtime())}} + {% for key, value in properties_dict|dictsort %} +{{key}}={{value}}{% endfor %} + ''', extra_imports=[time], properties_dict=self.resource.properties) + + Logger.info(format("Generating properties file: {filepath}")) + + with Environment.get_instance_copy() as env: + File (format("{filepath}"), + content = config_content, + owner = self.resource.owner, + group = self.resource.group, + mode = self.resource.mode + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/providers/repository.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/repository.py b/ambari-common/src/main/python/resource_management/libraries/providers/repository.py new file mode 100644 index 0000000..c670ac1 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/providers/repository.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import os +import filecmp +import tempfile +from ambari_commons import OSCheck +from resource_management import * + +class RhelSuseRepositoryProvider(Provider): + def action_create(self): + with Environment.get_instance_copy() as env: + repo_file_name = self.resource.repo_file_name + repo_dir = repos_dirs[env.system.os_family] + repo_template = self.resource.repo_template + File(format("{repo_dir}/{repo_file_name}.repo"), + content = Template(repo_template, repo_id=self.resource.repo_id, repo_file_name=self.resource.repo_file_name, base_url=self.resource.base_url, mirror_list=self.resource.mirror_list) + ) + + def action_remove(self): + with Environment.get_instance_copy() as env: + repo_file_name = self.resource.repo_file_name + repo_dir = repos_dirs[env.system.os_family] + + File(format("{repo_dir}/{repo_file_name}.repo"), + action = "delete") + + +repos_dirs = { + 'redhat': '/etc/yum.repos.d', + 'suse': '/etc/zypp/repos.d' +} + + +class DebianRepositoryProvider(Provider): + package_type = "deb" + repo_dir = "/etc/apt/sources.list.d" + update_cmd = 'apt-get update -qq -o Dir::Etc::sourcelist="sources.list.d/{repo_file_name}" -o APT::Get::List-Cleanup="0"' + missing_pkey_regex = "The following signatures couldn't be verified because the public key is not available: NO_PUBKEY (.+)" + add_pkey_cmd = "apt-key adv --recv-keys --keyserver keyserver.ubuntu.com {pkey}" + + def action_create(self): + with Environment.get_instance_copy() as env: + with tempfile.NamedTemporaryFile() as tmpf: + File(tmpf.name, + content = Template(self.resource.repo_template, + package_type=self.package_type, base_url=self.resource.base_url, components=' '.join(self.resource.components)) + ) + + repo_file_name = format("{repo_file_name}.list",repo_file_name = self.resource.repo_file_name) + repo_file_path = format("{repo_dir}/{repo_file_name}", repo_dir = self.repo_dir) + + if not os.path.isfile(repo_file_path) or not filecmp.cmp(tmpf.name, repo_file_path): + File(repo_file_path, + content = StaticFile(tmpf.name) + ) + + # this is time expensive + retcode, out = checked_call(format(self.update_cmd)) + + # add public keys for new repos + missing_pkeys = set(re.findall(self.missing_pkey_regex, out)) + for pkey in missing_pkeys: + Execute(format(self.add_pkey_cmd), + timeout = 15, # in case we are on the host w/o internet (using localrepo), we should ignore hanging + ignore_failures = True + ) + + def action_remove(self): + with Environment.get_instance_copy() as env: + repo_file_name = format("{repo_file_name}.list",repo_file_name = self.resource.repo_file_name) + repo_file_path = format("{repo_dir}/{repo_file_name}", repo_dir = self.repo_dir) + + if os.path.isfile(repo_file_path): + File(repo_file_path, + action = "delete") + + # this is time expensive + Execute(format(self.update_cmd)) http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/providers/template_config.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/template_config.py b/ambari-common/src/main/python/resource_management/libraries/providers/template_config.py new file mode 100644 index 0000000..4972797 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/providers/template_config.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import os +from resource_management import * + +class TemplateConfigProvider(Provider): + def action_create(self): + template_tag = self.resource.template_tag + qualified_file_name = self.resource.name + file_name = os.path.basename(qualified_file_name) + + if not template_tag: + template_name = format("{file_name}.j2") + else: + template_name = format("{file_name}-{template_tag}.j2") + + with Environment.get_instance_copy() as env: + File( qualified_file_name, + owner = self.resource.owner, + group = self.resource.group, + mode = self.resource.mode, + content = Template(template_name, extra_imports=self.resource.extra_imports) + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py b/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py new file mode 100644 index 0000000..c962847 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import time +from resource_management import * + +class XmlConfigProvider(Provider): + def action_create(self): + filename = self.resource.filename + conf_dir = self.resource.conf_dir + + # |e - for html-like escaping of <,>,'," + config_content = InlineTemplate('''<!--{{time.asctime(time.localtime())}}--> + <configuration> + {% for key, value in configurations_dict|dictsort %} + <property> + <name>{{ key|e }}</name> + <value>{{ value|e }}</value> + {%- if not configuration_attrs is none -%} + {%- for attrib_name, attrib_occurances in configuration_attrs.items() -%} + {%- for property_name, attrib_value in attrib_occurances.items() -%} + {% if property_name == key and attrib_name %} + <{{attrib_name|e}}>{{attrib_value|e}}</{{attrib_name|e}}> + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {%- endif %} + </property> + {% endfor %} + </configuration>''', extra_imports=[time], configurations_dict=self.resource.configurations, + configuration_attrs=self.resource.configuration_attributes) + + + Logger.info(format("Generating config: {conf_dir}/{filename}")) + + with Environment.get_instance_copy() as env: + File (format("{conf_dir}/{filename}"), + content = config_content, + owner = self.resource.owner, + group = self.resource.group, + mode = self.resource.mode + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/resources/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/__init__.py b/ambari-common/src/main/python/resource_management/libraries/resources/__init__.py new file mode 100644 index 0000000..24b497c --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/resources/__init__.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management.libraries.resources.execute_hadoop import * +from resource_management.libraries.resources.template_config import * +from resource_management.libraries.resources.xml_config import * +from resource_management.libraries.resources.properties_file import * +from resource_management.libraries.resources.repository import * +from resource_management.libraries.resources.monitor_webserver import * +from resource_management.libraries.resources.hdfs_directory import * +from resource_management.libraries.resources.copy_from_local import * \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py b/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py new file mode 100644 index 0000000..328d9c2 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +_all__ = ["CopyFromLocal"] +from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument + +class CopyFromLocal(Resource): + action = ForcedListArgument(default="run") + + path = ResourceArgument(default=lambda obj: obj.name) + dest_dir = ResourceArgument(required=True) + owner = ResourceArgument(required=True) + group = ResourceArgument() + mode = ResourceArgument() + kinnit_if_needed = ResourceArgument(default='') + hadoop_conf_dir = ResourceArgument(default='/etc/hadoop/conf') + hdfs_user = ResourceArgument(default='hdfs') + + actions = Resource.actions + ["run"] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/resources/execute_hadoop.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/execute_hadoop.py b/ambari-common/src/main/python/resource_management/libraries/resources/execute_hadoop.py new file mode 100644 index 0000000..94daf5b --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/resources/execute_hadoop.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +_all__ = ["ExecuteHadoop"] +from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument + +class ExecuteHadoop(Resource): + action = ForcedListArgument(default="run") + command = ResourceArgument(default=lambda obj: obj.name) + kinit_override = BooleanArgument(default=False) + tries = ResourceArgument(default=1) + try_sleep = ResourceArgument(default=0) # seconds + user = ResourceArgument() + logoutput = BooleanArgument(default=False) + principal = ResourceArgument(default=lambda obj: obj.user) + + conf_dir = ResourceArgument() + + security_enabled = BooleanArgument(default=False) + keytab = ResourceArgument() + kinit_path_local = ResourceArgument() + + actions = Resource.actions + ["run"] + http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_directory.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_directory.py b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_directory.py new file mode 100644 index 0000000..63d9cc2 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_directory.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +_all__ = ["HdfsDirectory"] +from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument + +class HdfsDirectory(Resource): + action = ForcedListArgument() + + dir_name = ResourceArgument(default=lambda obj: obj.name) + owner = ResourceArgument() + group = ResourceArgument() + mode = ResourceArgument() + recursive_chown = BooleanArgument(default=False) + recursive_chmod = BooleanArgument(default=False) + + conf_dir = ResourceArgument() + security_enabled = BooleanArgument(default=False) + keytab = ResourceArgument() + kinit_path_local = ResourceArgument() + hdfs_user = ResourceArgument() + + #action 'create' immediately creates all pending directory in efficient manner + #action 'create_delayed' add directory to list of pending directories + actions = Resource.actions + ["create","create_delayed"] http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/resources/monitor_webserver.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/monitor_webserver.py b/ambari-common/src/main/python/resource_management/libraries/resources/monitor_webserver.py new file mode 100644 index 0000000..dfd1174 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/resources/monitor_webserver.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +_all__ = ["MonitorWebserver"] +from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument + + +class MonitorWebserver(Resource): + action = ForcedListArgument(default=lambda obj: [obj.name]) + actions = Resource.actions + ["start", "stop", "restart"] http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/resources/properties_file.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/properties_file.py b/ambari-common/src/main/python/resource_management/libraries/resources/properties_file.py new file mode 100644 index 0000000..0e5afb4 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/resources/properties_file.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +_all__ = ["PropertiesFile"] +from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument + +class PropertiesFile(Resource): + action = ForcedListArgument(default="create") + filename = ResourceArgument(default=lambda obj: obj.name) + + properties = ResourceArgument() + dir = ResourceArgument() + + mode = ResourceArgument() + owner = ResourceArgument() + group = ResourceArgument() + + actions = Resource.actions + ["create"] http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/resources/repository.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/repository.py b/ambari-common/src/main/python/resource_management/libraries/resources/repository.py new file mode 100644 index 0000000..48a347a --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/resources/repository.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +_all__ = ["Repository"] + +from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument + +class Repository(Resource): + action = ForcedListArgument(default="create") + repo_id = ResourceArgument(default=lambda obj: obj.name) + base_url = ResourceArgument() + mirror_list = ResourceArgument() + repo_file_name = ResourceArgument() + repo_template = ResourceArgument() + components = ForcedListArgument(default=[]) # ubuntu specific + + actions = Resource.actions + ["create","remove"] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/resources/template_config.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/template_config.py b/ambari-common/src/main/python/resource_management/libraries/resources/template_config.py new file mode 100644 index 0000000..8ce2a00 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/resources/template_config.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +_all__ = ["TemplateConfig"] +from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument + +class TemplateConfig(Resource): + action = ForcedListArgument(default="create") + path = ResourceArgument(default=lambda obj: obj.name) + mode = ResourceArgument() + owner = ResourceArgument() + group = ResourceArgument() + template_tag = ResourceArgument() + extra_imports = ResourceArgument(default=[]) + + actions = Resource.actions + ["create"] http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py b/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py new file mode 100644 index 0000000..37eb511 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +_all__ = ["XmlConfig"] +from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument + +class XmlConfig(Resource): + action = ForcedListArgument(default="create") + filename = ResourceArgument(default=lambda obj: obj.name) + + configurations = ResourceArgument() + configuration_attributes = ResourceArgument() + conf_dir = ResourceArgument() + + mode = ResourceArgument() + owner = ResourceArgument() + group = ResourceArgument() + + actions = Resource.actions + ["create"] http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/script/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/__init__.py b/ambari-common/src/main/python/resource_management/libraries/script/__init__.py new file mode 100644 index 0000000..72d3aaf --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/script/__init__.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management.libraries.script.script import * +from resource_management.libraries.script.hook import * +from resource_management.libraries.script.config_dictionary import * \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/script/config_dictionary.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/config_dictionary.py b/ambari-common/src/main/python/resource_management/libraries/script/config_dictionary.py new file mode 100644 index 0000000..453c546 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/script/config_dictionary.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +from resource_management.core.exceptions import Fail + +class ConfigDictionary(dict): + """ + Immutable config dictionary + """ + + def __init__(self, dictionary): + """ + Recursively turn dict to ConfigDictionary + """ + for k, v in dictionary.iteritems(): + if isinstance(v, dict): + dictionary[k] = ConfigDictionary(v) + + super(ConfigDictionary, self).__init__(dictionary) + + def __setitem__(self, name, value): + raise Fail("Configuration dictionary is immutable!") + + def __getitem__(self, name): + """ + - use Python types + - enable lazy failure for unknown configs. + """ + try: + value = super(ConfigDictionary, self).__getitem__(name) + except KeyError: + return UnknownConfiguration(name) + + + if value == "true": + value = True + elif value == "false": + value = False + else: + try: + value = int(value) + except (ValueError, TypeError): + try: + value = float(value) + except (ValueError, TypeError): + pass + + return value + + +class UnknownConfiguration(): + """ + Lazy failing for unknown configs. + """ + def __init__(self, name): + self.name = name + + def __getattr__(self, name): + raise Fail("Configuration parameter '"+self.name+"' was not found in configurations dictionary!") + + def __getitem__(self, name): + """ + Allow [] + """ + return self \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/script/hook.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/hook.py b/ambari-common/src/main/python/resource_management/libraries/script/hook.py new file mode 100644 index 0000000..5c8eafc --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/script/hook.py @@ -0,0 +1,66 @@ +#!/usr/bin/env ambari-python-wrap + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +__all__ = ["Hook"] + +from resource_management.libraries.script import Script +import subprocess +import sys + +class Hook(Script): + """ + Executes a hook for acommand for custom service. stdout and stderr are written to + tmpoutfile and to tmperrfile respectively. + """ + + HOOK_METHOD_NAME = "hook" # This method is always executed at hooks + + + def choose_method_to_execute(self, command_name): + """ + Changes logics of resolving method name + """ + return super(Hook, self).choose_method_to_execute(self.HOOK_METHOD_NAME) + + + def run_custom_hook(self, command): + """ + Runs custom hook + """ + args = sys.argv + + #Hook script to run + args[0] = args[0].replace('before-'+args[1], command) + args[0] = args[0].replace('after-'+args[1], command) + + #Hook script base directory + args[3] = args[3].replace('before-'+args[1], command) + args[3] = args[3].replace('after-'+args[1], command) + + args[1] = command.split("-")[1] + + + cmd = [sys.executable] + cmd.extend(args) + + if subprocess.call(cmd) != 0: + self.fail_with_error("Error: Unable to run the custom hook script " + + cmd.__str__()) +
