Repository: ambari Updated Branches: refs/heads/branch-2.4 0ad16dfd2 -> 281d61e2b refs/heads/trunk 5ce987a5c -> e0f3b534b
AMBARI-17550. While changing NN, DN directories from UI, proper warning should be present for invalid values (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e0f3b534 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e0f3b534 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e0f3b534 Branch: refs/heads/trunk Commit: e0f3b534b180281842f9a141b13a4b4de7d90696 Parents: 5ce987a Author: Andrew Onishuk <[email protected]> Authored: Thu Jul 7 12:45:09 2016 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Thu Jul 7 12:45:09 2016 +0300 ---------------------------------------------------------------------- .../resource_management/TestDatanodeHelper.py | 4 + .../resource_management/TestFileSystem.py | 19 +++++ .../libraries/functions/file_system.py | 12 +-- .../libraries/functions/mounted_dirs_helper.py | 19 ++++- .../stacks/HDP/2.0.6/services/stack_advisor.py | 44 +++++++++- .../stacks/2.0.6/common/test_stack_advisor.py | 84 +++++++++++++++++++- .../stacks/2.2/common/test_stack_advisor.py | 37 +++++++-- .../stacks/2.3/common/test_stack_advisor.py | 2 +- 8 files changed, 200 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e0f3b534/ambari-agent/src/test/python/resource_management/TestDatanodeHelper.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestDatanodeHelper.py b/ambari-agent/src/test/python/resource_management/TestDatanodeHelper.py index c33a295..9fa986b 100644 --- a/ambari-agent/src/test/python/resource_management/TestDatanodeHelper.py +++ b/ambari-agent/src/test/python/resource_management/TestDatanodeHelper.py @@ -162,3 +162,7 @@ class TestDatanodeHelper(TestCase): print args[0] self.assertEquals(0, log_error.call_count) + + def test_get_mounts_with_multiple_data_dirs(self): + self.assertEquals([], mounted_dirs_helper.get_mounts_with_multiple_data_dirs(["/", "/hodoop", "/tmp"], "/hadoop/data,/tmp")) + self.assertEquals([("/", ["/hadoop/data", "/tmp"])], mounted_dirs_helper.get_mounts_with_multiple_data_dirs(["/"], "/hadoop/data,/tmp")) http://git-wip-us.apache.org/repos/asf/ambari/blob/e0f3b534/ambari-agent/src/test/python/resource_management/TestFileSystem.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestFileSystem.py b/ambari-agent/src/test/python/resource_management/TestFileSystem.py index 925758c..e2f7ed8 100644 --- a/ambari-agent/src/test/python/resource_management/TestFileSystem.py +++ b/ambari-agent/src/test/python/resource_management/TestFileSystem.py @@ -165,3 +165,22 @@ class TestFileSystem(TestCase): mount_point = file_system.get_mount_point_for_dir("/hadoop/hdfs/data1") self.assertEqual(mount_point, "/hadoop/hdfs/data1") + + @patch.object(Logger, "info") + @patch.object(Logger, "error") + @patch('resource_management.core.providers.mount.get_mounted') + def test_get_mount_point_for_dir_with_mounsts(self, mounted_mock, log_error, log_info): + """ + Testing get_mount_point_for_dir when mounsts are provided. + """ + mounted_mock.return_value = self._get_mount(self.MOUNT_TYPE.SAME_PREFIX_MOUNTS) + + # refresh cached mounts + file_system.get_and_cache_mount_points(True) + + mount_point = file_system.get_mount_point_for_dir("/hadoop/hdfs/data1") + self.assertEqual(mount_point, "/hadoop/hdfs/data1") + + # Should use provided mounts, not fetch via get_and_cache_mount_points + mount_point = file_system.get_mount_point_for_dir("/hadoop/hdfs/data1", ["/", "/hadoop"]) + self.assertEqual(mount_point, "/hadoop") http://git-wip-us.apache.org/repos/asf/ambari/blob/e0f3b534/ambari-common/src/main/python/resource_management/libraries/functions/file_system.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/file_system.py b/ambari-common/src/main/python/resource_management/libraries/functions/file_system.py index 2a859ed..cd48836 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/file_system.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/file_system.py @@ -47,7 +47,7 @@ def get_and_cache_mount_points(refresh=False): return mounts -def get_mount_point_for_dir(dir): +def get_mount_point_for_dir(dir, mount_points = None): """ :param dir: Directory to check, even if it doesn't exist. :return: Returns the closest mount point as a string for the directory. if the "dir" variable is None, will return None. @@ -57,7 +57,7 @@ def get_mount_point_for_dir(dir): if dir: dir = dir.strip() - cached_mounts = get_and_cache_mount_points() + cached_mounts = [m['mount_point'] for m in get_and_cache_mount_points()] if mount_points is None else mount_points # If the path is "/hadoop/hdfs/data", then possible matches for mounts could be # "/", "/hadoop/hdfs", and "/hadoop/hdfs/data". @@ -65,11 +65,11 @@ def get_mount_point_for_dir(dir): for m in cached_mounts: # Ensure that the mount path and the dir path ends with "/" # The mount point "/hadoop" should not match the path "/hadoop1" - if os.path.join(dir, "").startswith(os.path.join(m['mount_point'], "")): + if os.path.join(dir, "").startswith(os.path.join(m, "")): if best_mount_found is None: - best_mount_found = m["mount_point"] - elif os.path.join(best_mount_found, "").count(os.path.sep) < os.path.join(m["mount_point"], "").count(os.path.sep): - best_mount_found = m["mount_point"] + best_mount_found = m + elif os.path.join(best_mount_found, "").count(os.path.sep) < os.path.join(m, "").count(os.path.sep): + best_mount_found = m Logger.info("Mount point for directory %s is %s" % (str(dir), str(best_mount_found))) return best_mount_found http://git-wip-us.apache.org/repos/asf/ambari/blob/e0f3b534/ambari-common/src/main/python/resource_management/libraries/functions/mounted_dirs_helper.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/mounted_dirs_helper.py b/ambari-common/src/main/python/resource_management/libraries/functions/mounted_dirs_helper.py index 9574ce5..ba59a92 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/mounted_dirs_helper.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/mounted_dirs_helper.py @@ -19,9 +19,10 @@ limitations under the License. Ambari Agent """ -__all__ = ["handle_mounted_dirs", ] +__all__ = ["handle_mounted_dirs", "get_mounts_with_multiple_data_dirs"] import os import re +from collections import defaultdict from resource_management.libraries.functions.file_system import get_mount_point_for_dir, get_and_cache_mount_points from resource_management.core.logger import Logger @@ -198,3 +199,19 @@ def handle_mounted_dirs(func, dirs_string, history_filename, update_cache=True): return dir_to_mount +def get_mounts_with_multiple_data_dirs(mount_points, dirs): + """ + Returns a list with (mount, dir_list) for mounts with multiple dirs. + Currently is used in the stack_advisor. + """ + mount_dirs = defaultdict(list) + for dir in [raw_dir.strip() for raw_dir in dirs.split(",")]: + mount_point = get_mount_point_for_dir(dir, mount_points) + mount_dirs[mount_point].append(dir) + + partition_mounts_list = [] + for mount_point, dir_list in mount_dirs.iteritems(): + if len(dir_list) > 1: + partition_mounts_list.append((mount_point, dir_list)) + + return partition_mounts_list http://git-wip-us.apache.org/repos/asf/ambari/blob/e0f3b534/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 06f7cfe..8358438 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -25,6 +25,7 @@ import socket from math import ceil, floor from resource_management.core.logger import Logger +from resource_management.libraries.functions.mounted_dirs_helper import get_mounts_with_multiple_data_dirs from stack_advisor import DefaultStackAdvisor @@ -1438,7 +1439,9 @@ class HDP206StackAdvisor(DefaultStackAdvisor): return self.toConfigurationValidationProblems(validationItems, "hbase-env") def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): - validationItems = [{"config-name": 'dfs.datanode.du.reserved', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'dfs.datanode.du.reserved')}] + clusterEnv = getSiteProperties(configurations, "cluster-env") + validationItems = [{"config-name": 'dfs.datanode.du.reserved', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'dfs.datanode.du.reserved')}, + {"config-name": 'dfs.datanode.data.dir', "item": self.validatorOneDataDirPerPartition(properties, 'dfs.datanode.data.dir', services, hosts, clusterEnv)}] return self.toConfigurationValidationProblems(validationItems, "hdfs-site") def validateHDFSConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts): @@ -1447,6 +1450,45 @@ class HDP206StackAdvisor(DefaultStackAdvisor): {"config-name": 'namenode_opt_maxnewsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_opt_maxnewsize')}] return self.toConfigurationValidationProblems(validationItems, "hadoop-env") + def validatorOneDataDirPerPartition(self, properties, propertyName, services, hosts, clusterEnv): + if not propertyName in properties: + return self.getErrorItem("Value should be set") + dirs = properties[propertyName] + + if not (clusterEnv and "one_dir_per_partition" in clusterEnv and clusterEnv["one_dir_per_partition"].lower() == "true"): + return None + + dataNodeHosts = self.getDataNodeHosts(services, hosts) + + warnings = set() + for host in dataNodeHosts: + hostName = host["Hosts"]["host_name"] + + mountPoints = [] + for diskInfo in host["Hosts"]["disk_info"]: + mountPoints.append(diskInfo["mountpoint"]) + + if get_mounts_with_multiple_data_dirs(mountPoints, dirs): + # A detailed message can be too long on large clusters: + # warnings.append("Host: " + hostName + "; Mount: " + mountPoint + "; Data directories: " + ", ".join(dirList)) + warnings.add(hostName) + break; + + if len(warnings) > 0: + return self.getWarnItem("cluster-env/one_dir_per_partition is enabled but there are multiple data directories on the same mount. Affected hosts: {0}".format(", ".join(sorted(warnings)))) + + return None + + """ + Returns the list of Data Node hosts. + """ + def getDataNodeHosts(self, services, hosts): + if len(hosts["items"]) > 0: + dataNodeHosts = self.getHostsWithComponent("HDFS", "DATANODE", services, hosts) + if dataNodeHosts is not None: + return dataNodeHosts + return [] + def getMastersWithMultipleInstances(self): return ['ZOOKEEPER_SERVER', 'HBASE_MASTER'] http://git-wip-us.apache.org/repos/asf/ambari/blob/e0f3b534/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 7a092fc..931d59b 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -1261,23 +1261,26 @@ class TestHDP206StackAdvisor(TestCase): def test_validateHDFSConfigurations(self): configurations = {} - services = '' + services = {'configurations': {}} hosts = '' #Default configuration recommendedDefaults = {'dfs.datanode.du.reserved': '1024'} - properties = {'dfs.datanode.du.reserved': '1024'} + properties = {'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data'} res = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts) self.assertFalse(res) #Value is less then expected recommendedDefaults = {'dfs.datanode.du.reserved': '1024'} - properties = {'dfs.datanode.du.reserved': '512'} + properties = {'dfs.datanode.du.reserved': '512', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data'} res = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts) self.assertTrue(res) #Value is begger then expected recommendedDefaults = {'dfs.datanode.du.reserved': '1024'} - properties = {'dfs.datanode.du.reserved': '2048'} + properties = {'dfs.datanode.du.reserved': '2048', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data'} res = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts) self.assertFalse(res) @@ -1314,6 +1317,79 @@ class TestHDP206StackAdvisor(TestCase): res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '') self.assertEquals(res, res_expected) + def test_validateOneDataDirPerPartition(self): + recommendedDefaults = { + 'dfs.datanode.du.reserved': '1024' + } + + properties = { + 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data,/hadoop/hdfs/data2', + } + configurations = { + 'hdfs-site': { + 'properties': properties, + }, + 'cluster-env': { + 'properties': {'one_dir_per_partition': 'true'} + }, + } + services = {"services": + [{"StackServices": + {"service_name" : "HDFS", + "service_version" : "2.6.0.2.2", + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "DATANODE", + "hostnames": ["host1", "host2"] + } + } + ] + }], + "configurations": configurations, + } + host1 = { + "Hosts" : { + "host_name" : "host1", + "disk_info": [ + {"mountpoint" : "/hadoop/hdfs/data"}, + {"mountpoint" : "/hadoop/hdfs/data2"} + ] + } + } + host2 = { + "Hosts" : { + "host_name" : "host2", + "disk_info": [ + {"mountpoint": "/"}, + {"mountpoint" : "/hadoop"} + ] + } + } + hosts = { + "items" : [ + host1, + host2 + ] + } + + # Multiple data directories on the same mount. A warning is expected. + expected = [{'config-name': 'dfs.datanode.data.dir', + 'config-type': 'hdfs-site', + 'level': 'WARN', + 'message': "cluster-env/one_dir_per_partition is enabled but there are multiple data directories on the same mount. Affected hosts: host2", + 'type': 'configuration'}] + validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts) + self.assertEquals(validation_problems, expected) + + # One data directory. + properties['dfs.datanode.data.dir'] = '/hadoop/hdfs/data' + expected = [] + validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts) + self.assertEquals(validation_problems, expected) + def test_validateAmsHbaseSiteConfigurations(self): configurations = { "hdfs-site": { http://git-wip-us.apache.org/repos/asf/ambari/blob/e0f3b534/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 52b7e0f..9443042 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -358,6 +358,7 @@ class TestHDP22StackAdvisor(TestCase): # TEST CASE: Unsecured cluster, secure ports properties = { # hdfs-site 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', 'dfs.datanode.address': '0.0.0.0:1019', 'dfs.datanode.http.address': '0.0.0.0:1022', } @@ -377,7 +378,8 @@ class TestHDP22StackAdvisor(TestCase): {"service_name" : "HDFS", "service_version" : "2.6.0.2.2", } - }] + }], + "configurations": {} } expected = [] # No warnings validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, None) @@ -386,6 +388,7 @@ class TestHDP22StackAdvisor(TestCase): # TEST CASE: Unsecured cluster, unsecure ports properties = { # hdfs-site 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', 'dfs.datanode.address': '0.0.0.0:55555', 'dfs.datanode.http.address': '0.0.0.0:55555', } @@ -407,8 +410,9 @@ class TestHDP22StackAdvisor(TestCase): [{"StackServices": {"service_name" : "HDFS", "service_version" : "2.6.0.2.2", - } - }] + }, + }], + "configurations": {} } validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, None) self.assertEquals(validation_problems, expected) @@ -416,6 +420,7 @@ class TestHDP22StackAdvisor(TestCase): # TEST CASE: Secure cluster, invalid dfs.http.policy value properties = { # hdfs-site 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', 'dfs.http.policy': 'WRONG_VALUE', 'dfs.datanode.address': '0.0.0.0:1019', 'dfs.datanode.http.address': '0.0.0.0:1022', @@ -442,8 +447,9 @@ class TestHDP22StackAdvisor(TestCase): [{"StackServices": {"service_name" : "HDFS", "service_version" : "2.6.0.2.2", - } - }] + }, + }], + "configurations": {} } validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, None) self.assertEquals(validation_problems, expected) @@ -451,6 +457,7 @@ class TestHDP22StackAdvisor(TestCase): # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, https address not defined properties = { # hdfs-site 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', 'dfs.http.policy': 'HTTPS_ONLY', 'dfs.datanode.address': '0.0.0.0:1019', } @@ -473,7 +480,8 @@ class TestHDP22StackAdvisor(TestCase): {"service_name" : "HDFS", "service_version" : "2.6.0.2.2", } - }] + }], + "configurations": {} } validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, None) self.assertEquals(validation_problems, expected) @@ -481,6 +489,7 @@ class TestHDP22StackAdvisor(TestCase): # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, https address defined and secure properties = { # hdfs-site 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', 'dfs.http.policy': 'HTTPS_ONLY', 'dfs.datanode.address': '0.0.0.0:1019', 'dfs.datanode.https.address': '0.0.0.0:1022', @@ -504,7 +513,8 @@ class TestHDP22StackAdvisor(TestCase): {"service_name" : "HDFS", "service_version" : "2.6.0.2.2", } - }] + }], + "configurations": {} } validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, None) self.assertEquals(validation_problems, expected) @@ -512,6 +522,7 @@ class TestHDP22StackAdvisor(TestCase): # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, https address defined and non secure properties = { # hdfs-site 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', 'dfs.http.policy': 'HTTPS_ONLY', 'dfs.datanode.address': '0.0.0.0:1019', 'dfs.datanode.https.address': '0.0.0.0:50475', @@ -535,7 +546,8 @@ class TestHDP22StackAdvisor(TestCase): {"service_name" : "HDFS", "service_version" : "2.6.0.2.2", } - }] + }], + "configurations": {} } validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, None) self.assertEquals(validation_problems, expected) @@ -543,6 +555,7 @@ class TestHDP22StackAdvisor(TestCase): # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, non secure dfs port, https property not defined properties = { # hdfs-site 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', 'dfs.http.policy': 'HTTPS_ONLY', 'dfs.datanode.address': '0.0.0.0:50010', } @@ -595,6 +608,7 @@ class TestHDP22StackAdvisor(TestCase): # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, non secure dfs port, https defined and secure properties = { # hdfs-site 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', 'dfs.http.policy': 'HTTPS_ONLY', 'dfs.datanode.address': '0.0.0.0:50010', 'dfs.datanode.https.address': '0.0.0.0:1022', @@ -645,6 +659,7 @@ class TestHDP22StackAdvisor(TestCase): # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, valid non-root configuration properties = { # hdfs-site 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', 'dfs.http.policy': 'HTTPS_ONLY', 'dfs.datanode.address': '0.0.0.0:50010', 'dfs.datanode.https.address': '0.0.0.0:50475', @@ -670,6 +685,7 @@ class TestHDP22StackAdvisor(TestCase): # TEST CASE: Secure cluster, dfs.http.policy=HTTP_ONLY, insecure port properties = { # hdfs-site 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', 'dfs.http.policy': 'HTTP_ONLY', 'dfs.datanode.address': '0.0.0.0:1019', 'dfs.datanode.http.address': '0.0.0.0:50475', @@ -712,6 +728,7 @@ class TestHDP22StackAdvisor(TestCase): # TEST CASE: Secure cluster, dfs.http.policy=HTTP_ONLY, valid configuration properties = { # hdfs-site 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', 'dfs.http.policy': 'HTTP_ONLY', 'dfs.datanode.address': '0.0.0.0:1019', 'dfs.datanode.http.address': '0.0.0.0:1022', @@ -736,6 +753,7 @@ class TestHDP22StackAdvisor(TestCase): # TEST CASE: Secure cluster, absent dfs.http.policy (typical situation) properties = { # hdfs-site 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', 'dfs.datanode.address': '0.0.0.0:1019', 'dfs.datanode.http.address': '0.0.0.0:1022', } @@ -759,6 +777,7 @@ class TestHDP22StackAdvisor(TestCase): # TEST CASE: Secure cluster, dfs.http.policy=HTTP_ONLY, misusage of dfs.data.transfer.protection warning properties = { # hdfs-site 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', 'dfs.http.policy': 'HTTP_ONLY', 'dfs.datanode.address': '0.0.0.0:1019', 'dfs.datanode.http.address': '0.0.0.0:1022', @@ -789,6 +808,7 @@ class TestHDP22StackAdvisor(TestCase): # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, wrong dfs.data.transfer.protection value properties = { # hdfs-site 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', 'dfs.http.policy': 'HTTPS_ONLY', 'dfs.datanode.address': '0.0.0.0:50010', 'dfs.datanode.https.address': '0.0.0.0:50475', @@ -819,6 +839,7 @@ class TestHDP22StackAdvisor(TestCase): properties = { # hdfs-site 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', 'dfs.encrypt.data.transfer': 'true', # Wire encryption 'dfs.datanode.address': '0.0.0.0:1019', 'dfs.datanode.http.address': '0.0.0.0:1022', http://git-wip-us.apache.org/repos/asf/ambari/blob/e0f3b534/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index da0a704..784b271 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -1512,7 +1512,7 @@ class TestHDP23StackAdvisor(TestCase): self.assertEquals(recommendedConfigurations['ranger-admin-site']['properties']['ranger.audit.solr.zookeepers'], 'NONE') def test_validateHDFSConfigurations(self): - properties = {} + properties = {'dfs.datanode.data.dir': '/hadoop/hdfs/data'} recommendedDefaults = {} configurations = { "core-site": {"properties": {}},
