AMBARI-19121. Add ability for users to provide mount exclusion list before cluster deployment (Dmytro Grinenko via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/51ec956b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/51ec956b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/51ec956b Branch: refs/heads/branch-feature-AMBARI-18456 Commit: 51ec956b14378d8d0fdab03fa8d2e60559772d1f Parents: d56576d Author: Sumit Mohanty <[email protected]> Authored: Wed Dec 7 10:35:05 2016 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Wed Dec 7 10:38:52 2016 -0800 ---------------------------------------------------------------------- ambari-agent/conf/unix/ambari-agent.ini | 1 + .../src/main/python/ambari_agent/Hardware.py | 52 ++++++++++++++- .../TestCustomServiceOrchestrator.py | 4 +- .../test/python/ambari_agent/TestHardware.py | 70 ++++++++++++++++++++ 4 files changed, 122 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/51ec956b/ambari-agent/conf/unix/ambari-agent.ini ---------------------------------------------------------------------- diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini index d6fcf5f..56fa605 100644 --- a/ambari-agent/conf/unix/ambari-agent.ini +++ b/ambari-agent/conf/unix/ambari-agent.ini @@ -40,6 +40,7 @@ alert_kinit_timeout=14400000 system_resource_overrides=/etc/resource_overrides ; memory_threshold_soft_mb=400 ; memory_threshold_hard_mb=1000 +; ignore_mount_points=/mnt/custom1,/mnt/custom2 [security] keysdir=/var/lib/ambari-agent/keys http://git-wip-us.apache.org/repos/asf/ambari/blob/51ec956b/ambari-agent/src/main/python/ambari_agent/Hardware.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/Hardware.py b/ambari-agent/src/main/python/ambari_agent/Hardware.py index 3c94d28..0d431a3 100644 --- a/ambari-agent/src/main/python/ambari_agent/Hardware.py +++ b/ambari-agent/src/main/python/ambari_agent/Hardware.py @@ -41,6 +41,7 @@ class Hardware: CHECK_REMOTE_MOUNTS_TIMEOUT_DEFAULT = '10' IGNORE_ROOT_MOUNTS = ["proc", "dev", "sys"] IGNORE_DEVICES = ["proc", "tmpfs", "cgroup", "mqueue", "shm"] + LINUX_PATH_SEP = "/" def __init__(self, config): self.hardware = { @@ -88,6 +89,37 @@ class Hardware: return True @classmethod + def _is_mount_blacklisted(cls, blacklist, mount_point): + """ + Verify if particular mount point is in the black list. + + :return True if mount_point or a part of mount point is in the blacklist, otherwise return False + + Example: + Mounts: /, /mnt/my_mount, /mnt/my_mount/sub_mount + Blacklist: /mnt/my_mount + Result: / + + :type blacklist list + :type mount_point str + :rtype bool + """ + + if not blacklist or not mount_point: + return False + + mount_point_elements = mount_point.split(cls.LINUX_PATH_SEP) + + for el in blacklist: + el_list = el.split(cls.LINUX_PATH_SEP) + # making patch elements comparision + if el_list == mount_point_elements[:len(el_list)]: + return True + + return False + + + @classmethod @OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) def osdisks(cls, config=None): """ Run df to find out the disks on the host. Only works on linux @@ -95,6 +127,11 @@ class Hardware: and any mounts with spaces. """ timeout = cls._get_mount_check_timeout(config) command = ["timeout", timeout, "df", "-kPT"] + blacklisted_mount_points = [] + + if config: + ignore_mount_value = config.get("agent", "ignore_mount_points", default="") + blacklisted_mount_points = [item.strip() for item in ignore_mount_value.split(",")] if not cls._check_remote_mounts(config): command.append("-l") @@ -103,6 +140,7 @@ class Hardware: dfdata = df.communicate()[0] mounts = [cls._parse_df_line(line) for line in dfdata.splitlines() if line] result_mounts = [] + ignored_mounts = [] for mount in mounts: if not mount: @@ -113,13 +151,21 @@ class Hardware: - mounted device is not in the ignored list - is accessible to user under which current process running - it is not file-mount (docker environment) + - mount path or a part of mount path is not in the blacklist """ - if mount["device"] not in cls.IGNORE_DEVICES and \ + if mount["device"] not in cls.IGNORE_DEVICES and\ mount["mountpoint"].split("/")[0] not in cls.IGNORE_ROOT_MOUNTS and\ - cls._chk_writable_mount(mount['mountpoint']) and \ - not path_isfile(mount["mountpoint"]): + cls._chk_writable_mount(mount['mountpoint']) and\ + not path_isfile(mount["mountpoint"]) and\ + not cls._is_mount_blacklisted(blacklisted_mount_points, mount["mountpoint"]): result_mounts.append(mount) + else: + ignored_mounts.append(mount) + + if len(ignored_mounts) > 0: + ignore_list = [el["mountpoint"] for el in ignored_mounts] + logger.info("Some mount points was ignored: {0}".format(', '.join(ignore_list))) return result_mounts http://git-wip-us.apache.org/repos/asf/ambari/blob/51ec956b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py index 5323d9a..563d250 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py +++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py @@ -68,7 +68,7 @@ class TestCustomServiceOrchestrator(TestCase): def test_add_reg_listener_to_controller(self, FileCache_mock): FileCache_mock.return_value = None dummy_controller = MagicMock() - config = AmbariConfig().getConfig() + config = AmbariConfig() tempdir = tempfile.gettempdir() config.set('agent', 'prefix', tempdir) CustomServiceOrchestrator(config, dummy_controller) @@ -204,7 +204,7 @@ class TestCustomServiceOrchestrator(TestCase): def test_resolve_script_path(self, FileCache_mock, exists_mock): FileCache_mock.return_value = None dummy_controller = MagicMock() - config = AmbariConfig().getConfig() + config = AmbariConfig() orchestrator = CustomServiceOrchestrator(config, dummy_controller) # Testing existing path exists_mock.return_value = True http://git-wip-us.apache.org/repos/asf/ambari/blob/51ec956b/ambari-agent/src/test/python/ambari_agent/TestHardware.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestHardware.py b/ambari-agent/src/test/python/ambari_agent/TestHardware.py index 038b2f8..ff3b40b 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestHardware.py +++ b/ambari-agent/src/test/python/ambari_agent/TestHardware.py @@ -25,6 +25,7 @@ from mock.mock import patch, MagicMock, Mock import unittest import platform import socket +import os from only_for_platform import not_for_platform, PLATFORM_WINDOWS from ambari_agent import hostname from ambari_agent.Hardware import Hardware @@ -373,6 +374,75 @@ SwapFree: 1598676 kB self.assertEquals(2, json_mock.call_count) self.assertEquals('value', result['key']) + @patch.object(Hardware, "_chk_writable_mount") + @patch("ambari_agent.Hardware.path_isfile") + def test_osdisks_blacklist(self, isfile_mock, chk_writable_mount_mock): + df_output = \ + """Filesystem Type 1024-blocks Used Available Capacity Mounted on + /dev/mapper/docker-253:0-4980899-d45c264d37ab18c8ed14f890f4d59ac2b81e1c52919eb36a79419787209515f3 xfs 31447040 1282384 30164656 5% / + tmpfs tmpfs 32938336 4 32938332 1% /dev + tmpfs tmpfs 32938336 0 32938336 0% /sys/fs/cgroup + /dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /etc/resolv.conf + /dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /etc/hostname + /dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /etc/hosts + shm tmpfs 65536 0 65536 0% /dev/shm + /dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /run/secrets + /dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /mnt/blacklisted_mount + /dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /mnt/blacklisted_mount/sub-dir + """ + + def isfile_side_effect(path): + assume_files = ["/etc/resolv.conf", "/etc/hostname", "/etc/hosts"] + return path in assume_files + + def chk_writable_mount_side_effect(path): + assume_read_only = ["/run/secrets"] + return path not in assume_read_only + + isfile_mock.side_effect = isfile_side_effect + chk_writable_mount_mock.side_effect = chk_writable_mount_side_effect + + config_dict = { + "agent": { + "ignore_mount_points": "/mnt/blacklisted_mount" + } + } + + with patch("subprocess.Popen") as open_mock: + proc_mock = Mock() + attr = { + 'communicate.return_value': [ + df_output + ] + } + proc_mock.configure_mock(**attr) + open_mock.return_value = proc_mock + + def conf_get(section, key, default=""): + if section in config_dict and key in config_dict[section]: + return config_dict[section][key] + + return default + + def has_option(section, key): + return section in config_dict and key in config_dict[section] + + conf = Mock() + attr = { + 'get.side_effect': conf_get, + 'has_option.side_effect': has_option + } + conf.configure_mock(**attr) + + result = Hardware.osdisks(conf) + + self.assertEquals(1, len(result)) + + expected_mounts_left = ["/"] + mounts_left = [item["mountpoint"] for item in result] + + self.assertEquals(expected_mounts_left, mounts_left) + if __name__ == "__main__": unittest.main()
