Repository: ambari Updated Branches: refs/heads/trunk 79f3d96e6 -> 3839cd8f8
AMBARI-11199. Use Internal calls instead of sudo calls when running as root (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3839cd8f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3839cd8f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3839cd8f Branch: refs/heads/trunk Commit: 3839cd8f889d3e7da5e86e5853d44858bab0b64c Parents: 79f3d96 Author: Andrew Onishuk <[email protected]> Authored: Sun May 17 21:02:19 2015 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Sun May 17 21:02:19 2015 +0300 ---------------------------------------------------------------------- .../core/providers/system.py | 27 +- .../python/resource_management/core/shell.py | 2 +- .../python/resource_management/core/source.py | 9 - .../python/resource_management/core/sudo.py | 266 ++++++++++++------- .../python/resource_management/core/utils.py | 23 ++ 5 files changed, 199 insertions(+), 128 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/3839cd8f/ambari-common/src/main/python/resource_management/core/providers/system.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/system.py b/ambari-common/src/main/python/resource_management/core/providers/system.py index 6a41bc1..07155f5 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/system.py +++ b/ambari-common/src/main/python/resource_management/core/providers/system.py @@ -23,9 +23,7 @@ Ambari Agent from __future__ import with_statement import re -import grp import os -import pwd import time from resource_management.core import shell from resource_management.core import sudo @@ -33,29 +31,8 @@ from resource_management.core.base import Fail from resource_management.core import ExecuteTimeoutException from resource_management.core.providers import Provider from resource_management.core.logger import Logger - - -def _coerce_uid(user): - try: - uid = int(user) - except ValueError: - try: - uid = pwd.getpwnam(user).pw_uid - except KeyError: - raise Fail("User %s doesn't exist." % user) - return uid - - -def _coerce_gid(group): - try: - gid = int(group) - except ValueError: - try: - gid = grp.getgrnam(group).gr_gid - except KeyError: - raise Fail("Group %s doesn't exist." % group) - return gid - +from resource_management.core.utils import _coerce_uid +from resource_management.core.utils import _coerce_gid def _ensure_metadata(path, user, group, mode=None, cd_access=None): stat = sudo.stat(path) http://git-wip-us.apache.org/repos/asf/ambari/blob/3839cd8f/ambari-common/src/main/python/resource_management/core/shell.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/shell.py b/ambari-common/src/main/python/resource_management/core/shell.py index 4fcb04b..f2b5000 100644 --- a/ambari-common/src/main/python/resource_management/core/shell.py +++ b/ambari-common/src/main/python/resource_management/core/shell.py @@ -162,7 +162,7 @@ def _call(command, logoutput=None, throw_on_failure=True, # in case logoutput==False, never log. logoutput = logoutput==True and Logger.logger.isEnabledFor(logging.INFO) or logoutput==None and Logger.logger.isEnabledFor(logging.DEBUG) out = "" - read_timeout = .04 # seconds + read_timeout = .001 # seconds try: while True: http://git-wip-us.apache.org/repos/asf/ambari/blob/3839cd8f/ambari-common/src/main/python/resource_management/core/source.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/source.py b/ambari-common/src/main/python/resource_management/core/source.py index 4eecc7d..7317492 100644 --- a/ambari-common/src/main/python/resource_management/core/source.py +++ b/ambari-common/src/main/python/resource_management/core/source.py @@ -25,7 +25,6 @@ from resource_management.core.environment import Environment from resource_management.core.logger import Logger from resource_management.core.exceptions import Fail from resource_management.core.utils import checked_unite -from resource_management.core import sudo __all__ = ["Source", "Template", "InlineTemplate", "StaticFile", "DownloadSource"] @@ -33,8 +32,6 @@ import os import time import urllib2 import urlparse -from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl -from ambari_commons import OSConst class Source(object): @@ -77,16 +74,10 @@ class StaticFile(Source): return self.read_file(path) - @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def read_file(self, path): from resource_management.core import sudo return sudo.read_file(path) - @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) - def read_file(self, path): - with open(path, "rb") as fp: - return fp.read() - try: from ambari_jinja2 import Environment as JinjaEnvironment, BaseLoader, TemplateNotFound, FunctionLoader, StrictUndefined http://git-wip-us.apache.org/repos/asf/ambari/blob/3839cd8f/ambari-common/src/main/python/resource_management/core/sudo.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/sudo.py b/ambari-common/src/main/python/resource_management/core/sudo.py index 38af963..b5dfa47 100644 --- a/ambari-common/src/main/python/resource_management/core/sudo.py +++ b/ambari-common/src/main/python/resource_management/core/sudo.py @@ -21,111 +21,191 @@ Ambari Agent import os import tempfile +import shutil +import stat from resource_management.core import shell from resource_management.core.logger import Logger +from resource_management.core.utils import _coerce_uid +from resource_management.core.utils import _coerce_gid +from ambari_commons.os_check import OSCheck -# os.chown replacement -def chown(path, owner, group): - if owner: - shell.checked_call(["chown", owner, path], sudo=True) - if group: - shell.checked_call(["chgrp", group, path], sudo=True) - -# os.chmod replacement -def chmod(path, mode): - shell.checked_call(["chmod", oct(mode), path], sudo=True) +if os.geteuid() == 0 or OSCheck.is_windows_family(): + def chown(path, owner, group): + return os.chown(path, _coerce_uid(owner) if owner else -1, _coerce_gid(group) if group else -1) -def chmod_extended(path, mode): - shell.checked_call(["chmod", mode, path], sudo=True) + def chmod(path, mode): + return os.chmod(path, mode) -# os.makedirs replacement -def makedirs(path, mode): - shell.checked_call(["mkdir", "-p", path], sudo=True) - chmod(path, mode) - -# os.makedir replacement -def makedir(path, mode): - shell.checked_call(["mkdir", path], sudo=True) - chmod(path, mode) - -# os.symlink replacement -def symlink(source, link_name): - shell.checked_call(["ln","-sf", source, link_name], sudo=True) - -# os.link replacement -def link(source, link_name): - shell.checked_call(["ln", "-f", source, link_name], sudo=True) + mode_to_stat = {"a+x": stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, "u+x": stat.S_IXUSR, "g+x": stat.S_IXGRP, "o+x": stat.S_IXOTH} + def chmod_extended(path, mode): + if mode in mode_to_stat: + st = os.stat(path) + os.chmod(path, st.st_mode | mode_to_stat[mode]) + else: + shell.checked_call(["chmod", mode, path]) + + def makedirs(path, mode): + os.makedirs(path, mode) -# os unlink -def unlink(path): - shell.checked_call(["rm","-f", path], sudo=True) + def makedir(path, mode): + os.mkdir(path) + + def symlink(source, link_name): + os.symlink(source, link_name) + + def link(source, link_name): + os.link(source, link_name) + + def unlink(path): + os.unlink(path) + + def rmtree(path): + shutil.rmtree(path) + + def create_file(filename, content, encoding=None): + """ + if content is None, create empty file + """ + with open(filename, "wb") as fp: + if content: + content = content.encode(encoding) if encoding else content + fp.write(content) + + def read_file(filename, encoding=None): + with open(filename, "rb") as fp: + content = fp.read() + + content = content.decode(encoding) if encoding else content + return content + + def path_exists(path): + return os.path.exists(path) -# shutil.rmtree -def rmtree(path): - shell.checked_call(["rm","-rf", path], sudo=True) + def path_isdir(path): + return os.path.isdir(path) -# fp.write replacement -def create_file(filename, content, encoding=None): - """ - if content is None, create empty file - """ - tmpf = tempfile.NamedTemporaryFile() + def path_lexists(path): + return os.path.lexists(path) - if content: - content = content.encode(encoding) if encoding else content - with open(tmpf.name, "wb") as fp: - fp.write(content) + def path_isfile(path): + return os.path.isfile(path) + + def stat(path): + class Stat: + def __init__(self, path): + stat_val = os.stat(path) + self.st_uid, self.st_gid, self.st_mode = stat_val.st_uid, stat_val.st_gid, stat_val.st_mode & 07777 + return Stat(path) - with tmpf: - shell.checked_call(["cp", "-f", tmpf.name, filename], sudo=True) +else: + + # os.chown replacement + def chown(path, owner, group): + if owner: + shell.checked_call(["chown", owner, path], sudo=True) + if group: + shell.checked_call(["chgrp", group, path], sudo=True) + + # os.chmod replacement + def chmod(path, mode): + shell.checked_call(["chmod", oct(mode), path], sudo=True) - # set default files mode - chmod(filename, 0644) + def chmod_extended(path, mode): + shell.checked_call(["chmod", mode, path], sudo=True) -# fp.read replacement -def read_file(filename, encoding=None): - tmpf = tempfile.NamedTemporaryFile() - shell.checked_call(["cp", "-f", filename, tmpf.name], sudo=True) - - with tmpf: - with open(tmpf.name, "rb") as fp: - content = fp.read() - - content = content.decode(encoding) if encoding else content - return content + # os.makedirs replacement + def makedirs(path, mode): + shell.checked_call(["mkdir", "-p", path], sudo=True) + chmod(path, mode) -# os.path.exists -def path_exists(path): - return (shell.call(["test", "-e", path], sudo=True)[0] == 0) - -# os.path.isdir -def path_isdir(path): - return (shell.call(["test", "-d", path], sudo=True)[0] == 0) - -# os.path.lexists -def path_lexists(path): - return (shell.call(["test", "-L", path], sudo=True)[0] == 0) - -# os.path.isfile -def path_isfile(path): - return (shell.call(["test", "-f", path], sudo=True)[0] == 0) - -# os.stat -def stat(path): - class Stat: - RETRY_COUNT = 5 - def __init__(self, path): - # Sometimes (on heavy load) stat call returns an empty output with zero return code - for i in range(0, self.RETRY_COUNT): - out = shell.checked_call(["stat", "-c", "%u %g %a", path], sudo=True)[1] - values = out.split(' ') - if len(values) == 3: - uid_str, gid_str, mode_str = values - self.st_uid, self.st_gid, self.st_mode = int(uid_str), int(gid_str), int(mode_str, 8) - break - else: - warning_message = "Can not parse a sudo stat call output: \"{0}\"".format(out) - Logger.warning(warning_message) + # os.makedir replacement + def makedir(path, mode): + shell.checked_call(["mkdir", path], sudo=True) + chmod(path, mode) + + # os.symlink replacement + def symlink(source, link_name): + shell.checked_call(["ln","-sf", source, link_name], sudo=True) + + # os.link replacement + def link(source, link_name): + shell.checked_call(["ln", "-f", source, link_name], sudo=True) + + # os unlink + def unlink(path): + shell.checked_call(["rm","-f", path], sudo=True) + + # shutil.rmtree + def rmtree(path): + shell.checked_call(["rm","-rf", path], sudo=True) + + def stat(path): + class Stat: + def __init__(self, path): stat_val = os.stat(path) self.st_uid, self.st_gid, self.st_mode = stat_val.st_uid, stat_val.st_gid, stat_val.st_mode & 07777 - return Stat(path) + + return Stat(path) + + # fp.write replacement + def create_file(filename, content, encoding=None): + """ + if content is None, create empty file + """ + tmpf = tempfile.NamedTemporaryFile() + + if content: + content = content.encode(encoding) if encoding else content + with open(tmpf.name, "wb") as fp: + fp.write(content) + + with tmpf: + shell.checked_call(["cp", "-f", tmpf.name, filename], sudo=True) + + # fp.read replacement + def read_file(filename, encoding=None): + tmpf = tempfile.NamedTemporaryFile() + shell.checked_call(["cp", "-f", filename, tmpf.name], sudo=True) + + with tmpf: + with open(tmpf.name, "rb") as fp: + content = fp.read() + + content = content.decode(encoding) if encoding else content + return content + + # os.path.exists + def path_exists(path): + return (shell.call(["test", "-e", path], sudo=True)[0] == 0) + + # os.path.isdir + def path_isdir(path): + return (shell.call(["test", "-d", path], sudo=True)[0] == 0) + + # os.path.lexists + def path_lexists(path): + return (shell.call(["test", "-L", path], sudo=True)[0] == 0) + + # os.path.isfile + def path_isfile(path): + return (shell.call(["test", "-f", path], sudo=True)[0] == 0) + + # os.stat + def stat(path): + class Stat: + RETRY_COUNT = 5 + def __init__(self, path): + # Sometimes (on heavy load) stat call returns an empty output with zero return code + for i in range(0, self.RETRY_COUNT): + out = shell.checked_call(["stat", "-c", "%u %g %a", path], sudo=True)[1] + values = out.split(' ') + if len(values) == 3: + uid_str, gid_str, mode_str = values + self.st_uid, self.st_gid, self.st_mode = int(uid_str), int(gid_str), int(mode_str, 8) + break + else: + warning_message = "Can not parse a sudo stat call output: \"{0}\"".format(out) + Logger.warning(warning_message) + stat_val = os.stat(path) + self.st_uid, self.st_gid, self.st_mode = stat_val.st_uid, stat_val.st_gid, stat_val.st_mode & 07777 + return Stat(path) http://git-wip-us.apache.org/repos/asf/ambari/blob/3839cd8f/ambari-common/src/main/python/resource_management/core/utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/utils.py b/ambari-common/src/main/python/resource_management/core/utils.py index 52a12b3..f9ad193 100644 --- a/ambari-common/src/main/python/resource_management/core/utils.py +++ b/ambari-common/src/main/python/resource_management/core/utils.py @@ -20,6 +20,8 @@ Ambari Agent """ +import grp +import pwd from resource_management.core.exceptions import Fail class AttributeDictionary(object): @@ -106,3 +108,24 @@ def checked_unite(dict1, dict2): result.update(dict2) return result + +def _coerce_uid(user): + try: + uid = int(user) + except ValueError: + try: + uid = pwd.getpwnam(user).pw_uid + except KeyError: + raise Fail("User %s doesn't exist." % user) + return uid + + +def _coerce_gid(group): + try: + gid = int(group) + except ValueError: + try: + gid = grp.getgrnam(group).gr_gid + except KeyError: + raise Fail("Group %s doesn't exist." % group) + return gid
