AMBARI-20670 Node manager start extremely slow when YARN NM local dirs are very large (dgrinenko via dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/28238ae8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/28238ae8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/28238ae8 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 28238ae84e24aa72613e59b6af341f1c2452a0aa Parents: c11d004 Author: Dmytro Sen <[email protected]> Authored: Thu Apr 6 15:22:10 2017 +0300 Committer: Dmytro Sen <[email protected]> Committed: Thu Apr 6 15:22:10 2017 +0300 ---------------------------------------------------------------------- .../python/resource_management/TestUtils.py | 39 +++++++++++++ .../python/resource_management/core/sudo.py | 58 +++++++++++++++----- .../python/resource_management/core/utils.py | 56 ++++++++++++++++++- .../YARN/2.1.0.2.0/package/scripts/yarn.py | 16 ++++-- .../YARN/3.0.0.3.0/package/scripts/yarn.py | 14 +++-- 5 files changed, 159 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/28238ae8/ambari-agent/src/test/python/resource_management/TestUtils.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestUtils.py b/ambari-agent/src/test/python/resource_management/TestUtils.py new file mode 100644 index 0000000..0fbae3e --- /dev/null +++ b/ambari-agent/src/test/python/resource_management/TestUtils.py @@ -0,0 +1,39 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import stat +from unittest import TestCase +from resource_management.core.utils import attr_to_bitmask + + +class TestUtils(TestCase): + + def test_attr_to_bitmask(self): + test_set = [ + ["+r", stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH, 0], + ["u+w", stat.S_IWUSR, 0], + ["uo+x", stat.S_IXUSR | stat.S_IXOTH, 0], + ["-x", stat.S_IRUSR, stat.S_IXUSR | stat.S_IXOTH | stat.S_IRUSR], + ["=x", stat.S_IXUSR | stat.S_IXOTH | stat.S_IXGRP, stat.S_IRUSR | stat.S_IRGRP] + ] + + for test in test_set: + test_pattern, expected, initial_val = test + bitmask = attr_to_bitmask(test_pattern, initial_bitmask= initial_val) + self.assertEquals(expected, bitmask, "Test set \"{0}\" failed, expected: {1} but got {2}".format( + test_pattern, expected, bitmask)) http://git-wip-us.apache.org/repos/asf/ambari/blob/28238ae8/ambari-common/src/main/python/resource_management/core/sudo.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/sudo.py b/ambari-common/src/main/python/resource_management/core/sudo.py index d6fd71f..c350161 100644 --- a/ambari-common/src/main/python/resource_management/core/sudo.py +++ b/ambari-common/src/main/python/resource_management/core/sudo.py @@ -30,6 +30,7 @@ from resource_management.core import shell from resource_management.core.exceptions import Fail import subprocess +from resource_management.core.utils import attr_to_bitmask if os.geteuid() == 0: def chown(path, owner, group): @@ -54,16 +55,45 @@ if os.geteuid() == 0: def chmod(path, mode): + """ + Wrapper around python function + + :type path str + :type mode int + """ return os.chmod(path, mode) - mode_to_stat = {"a+x": stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, "a+rx": stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH, "u+x": stat.S_IXUSR, "g+x": stat.S_IXGRP, "o+x": stat.S_IXOTH} + def chmod_extended(path, mode): - if mode in mode_to_stat: - st = os.stat(path) - os.chmod(path, st.st_mode | mode_to_stat[mode]) - else: - shell.checked_call(["chmod", mode, path]) - + """ + :type path str + :type mode str + """ + st = os.stat(path) + os.chmod(path, attr_to_bitmask(mode, initial_bitmask=st.st_mode)) + + def chmod_recursive(path, recursive_mode_flags, recursion_follow_links=False): + """ + Change recursively permissions on directories or files + + :type path str + :type recursive_mode_flags + :type recursion_follow_links bool + """ + dir_attrib = recursive_mode_flags["d"] if "d" in recursive_mode_flags else None + files_attrib = recursive_mode_flags["f"] if "d" in recursive_mode_flags else None + + for root, dirs, files in os.walk(path, followlinks=recursion_follow_links): + if dir_attrib is not None: + for dir_name in dirs: + full_dir_path = os.path.join(root, dir_name) + chmod(full_dir_path, attr_to_bitmask(dir_attrib, initial_bitmask=os.stat(full_dir_path).st_mode)) + + if files_attrib is not None: + for file_name in files: + full_file_path = os.path.join(root, file_name) + chmod(full_file_path, attr_to_bitmask(files_attrib, initial_bitmask=os.stat(full_file_path).st_mode)) + def copy(src, dst): shutil.copy(src, dst) @@ -278,10 +308,10 @@ else: return files -def chmod_recursive(path, recursive_mode_flags, recursion_follow_links): - find_flags = [] - if recursion_follow_links: - find_flags.append('-L') - - for key, flags in recursive_mode_flags.iteritems(): - shell.checked_call(["find"] + find_flags + [path, "-type", key, "-exec" , "chmod", flags ,"{}" ,";"]) + def chmod_recursive(path, recursive_mode_flags, recursion_follow_links): + find_flags = [] + if recursion_follow_links: + find_flags.append('-L') + + for key, flags in recursive_mode_flags.iteritems(): + shell.checked_call(["find"] + find_flags + [path, "-type", key, "-exec" , "chmod", flags ,"{}" ,";"]) http://git-wip-us.apache.org/repos/asf/ambari/blob/28238ae8/ambari-common/src/main/python/resource_management/core/utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/utils.py b/ambari-common/src/main/python/resource_management/core/utils.py index 265b2f2..53f6b8b 100644 --- a/ambari-common/src/main/python/resource_management/core/utils.py +++ b/ambari-common/src/main/python/resource_management/core/utils.py @@ -27,10 +27,16 @@ import sys import signal import cStringIO from functools import wraps + +import re + from resource_management.core.exceptions import Fail from itertools import chain, repeat, islice PASSWORDS_HIDE_STRING = "[PROTECTED]" +PERM_STRING_REGEXP = re.compile("(?P<scope>[ugoa]*)(?P<direction>[-+=])(?P<attr>[rwx]*)") +PERM_REGISTER = {"u": 0o100, "g": 0o010, "o": 0o001} +PERM_BITS = {"r": 0o004, "w": 0o002, "x": 0o001} class AttributeDictionary(object): def __init__(self, *args, **kwargs): @@ -157,4 +163,52 @@ def pad_infinite(iterable, padding=None): return chain(iterable, repeat(padding)) def pad(iterable, size, padding=None): - return islice(pad_infinite(iterable, padding), size) \ No newline at end of file + return islice(pad_infinite(iterable, padding), size) + + +def attr_to_bitmask(attr, initial_bitmask=0o0): + """ + Function able to generate permission bits from passed named permission string (chmod like style) + + Supports: + - scope modifications: u,g,o or a + - setting mode: +,-,- + - attributes: r,x,w + + Samples: + uo+rw, a+x, u-w, o=r + + :type attr str + :type initial_bitmask int + """ + attr_dict = {"scope": "", "direction": "", "attr": ""} + re_match_result = PERM_STRING_REGEXP.match(attr) + + if re_match_result: + attr_dict = re_match_result.groupdict(default=attr_dict) + + if attr_dict["scope"] == "": + attr_dict["scope"] = "a" + + if "a" in attr_dict["scope"]: + attr_dict["scope"] = "ugo" + + attr_dict["scope"] = list(attr_dict["scope"]) + attr_dict["attr"] = list(attr_dict["attr"]) + + if attr_dict["direction"] == "=": + clear_mask = 0o0 + for scope in attr_dict["scope"]: + clear_mask = clear_mask | 0o007 * PERM_REGISTER[scope] + + initial_bitmask = initial_bitmask ^ (initial_bitmask & clear_mask) + attr_dict["direction"] = "+" + + for scope in attr_dict["scope"]: + for attr in attr_dict["attr"]: + if attr_dict["direction"] == "-" and (initial_bitmask & (PERM_BITS[attr] * PERM_REGISTER[scope])) > 0: + initial_bitmask = initial_bitmask ^ (PERM_BITS[attr] * PERM_REGISTER[scope]) + elif attr_dict["direction"] == "+": + initial_bitmask = initial_bitmask | (PERM_BITS[attr] * PERM_REGISTER[scope]) + + return initial_bitmask http://git-wip-us.apache.org/repos/asf/ambari/blob/28238ae8/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py index 204ab56..52338df 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py @@ -347,7 +347,6 @@ def setup_nodemanager(): elif not params.security_enabled: File(params.nm_security_marker, action="delete") - if not params.security_enabled or params.toggle_nm_security: # handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount. nm_log_dir_to_mount_file_content = handle_mounted_dirs(create_log_dir, params.nm_log_dirs, params.nm_log_dir_to_mount_file, params) @@ -459,17 +458,24 @@ def create_log_dir(dir_name): ignore_failures=True, ) + def create_local_dir(dir_name): import params + + directory_args = {} + + if params.toggle_nm_security: + directory_args["recursive_mode_flags"] = {'f': 'a+rw', 'd': 'a+rwx'} + Directory(dir_name, - create_parents = True, + create_parents=True, cd_access="a", mode=0755, owner=params.yarn_user, group=params.user_group, ignore_failures=True, - recursive_mode_flags = {'f': 'a+rw', 'd': 'a+rwx'}, - ) + **directory_args + ) @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) def yarn(name = None): @@ -500,4 +506,4 @@ def yarn(name = None): ServiceConfig(service_name, action="change_user", username = params.yarn_user, - password = Script.get_password(params.yarn_user)) \ No newline at end of file + password = Script.get_password(params.yarn_user)) http://git-wip-us.apache.org/repos/asf/ambari/blob/28238ae8/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py index ae1b425..d601f8f 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py @@ -347,7 +347,6 @@ def setup_nodemanager(): elif not params.security_enabled: File(params.nm_security_marker, action="delete") - if not params.security_enabled or params.toggle_nm_security: # handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount. nm_log_dir_to_mount_file_content = handle_mounted_dirs(create_log_dir, params.nm_log_dirs, params.nm_log_dir_to_mount_file, params) @@ -459,16 +458,23 @@ def create_log_dir(dir_name): ignore_failures=True, ) + def create_local_dir(dir_name): import params + + directory_args = {} + + if params.toggle_nm_security: + directory_args["recursive_mode_flags"] = {'f': 'a+rw', 'd': 'a+rwx'} + Directory(dir_name, - create_parents = True, + create_parents=True, cd_access="a", mode=0755, owner=params.yarn_user, group=params.user_group, ignore_failures=True, - recursive_mode_flags = {'f': 'a+rw', 'd': 'a+rwx'}, + **directory_args ) @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) @@ -500,4 +506,4 @@ def yarn(name = None): ServiceConfig(service_name, action="change_user", username = params.yarn_user, - password = Script.get_password(params.yarn_user)) \ No newline at end of file + password = Script.get_password(params.yarn_user))
