Repository: ambari Updated Branches: refs/heads/trunk f707cb50f -> c0ecec05c
AMBARI-15215: Update warning messages for HAWQ and PXF in stack advisor (mithmatt) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c0ecec05 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c0ecec05 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c0ecec05 Branch: refs/heads/trunk Commit: c0ecec05c5f0d0efa6a4ceadca5bcd6d4dc56461 Parents: f707cb5 Author: Matt <mmat...@pivotal.io> Authored: Wed Mar 2 16:36:21 2016 -0800 Committer: Matt <mmat...@pivotal.io> Committed: Wed Mar 2 16:36:21 2016 -0800 ---------------------------------------------------------------------- .../stacks/HDP/2.3/services/stack_advisor.py | 41 +++----- .../stacks/2.3/common/test_stack_advisor.py | 105 ++++++++++++++----- 2 files changed, 93 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c0ecec05/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index ae7c36e..53b46be 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -69,54 +69,41 @@ class HDP23StackAdvisor(HDP22StackAdvisor): hawqSegmentHosts = self.__getHosts(componentsList, "HAWQSEGMENT") datanodeHosts = self.__getHosts(componentsList, "DATANODE") - # Generate WARNING if any HAWQSEGMENT is not co-located with a DATANODE - mismatchHosts = set(hawqSegmentHosts).symmetric_difference(set(datanodeHosts)) + # Generate WARNING if any HAWQSEGMENT is not colocated with a DATANODE + mismatchHosts = sorted(set(hawqSegmentHosts).symmetric_difference(set(datanodeHosts))) if len(mismatchHosts) > 0: hostsString = ', '.join(mismatchHosts) - message = "HAWQSEGMENT is not co-located with DATANODE on the following {0} host(s): {1}".format(len(mismatchHosts), hostsString) + message = "HAWQ Segment must be installed on all DataNodes. " \ + "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format(len(mismatchHosts), hostsString) childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQSEGMENT' } ) # single node case is not analyzed because HAWQ Standby Master will not be present in single node topology due to logic in createComponentLayoutRecommendations() if len(hawqMasterHosts) == 1 and len(hawqStandbyHosts) == 1 and hawqMasterHosts == hawqStandbyHosts: - message = "HAWQ Standby Master and HAWQ Master should not be deployed on the same host." + message = "HAWQ Master and HAWQ Standby Master cannot be deployed on the same host." childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } ) if len(hawqMasterHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqMasterHosts[0]): message = "HAWQ Master and Ambari Server should not be deployed on the same host. " \ - "If you leave them collocated, make sure to set HAWQ Master Port property " \ + "If you leave them colocated, make sure to set HAWQ Master Port property " \ "to a value different from the port number used by Ambari Server database." childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQMASTER', "host": hawqMasterHosts[0] } ) if len(hawqStandbyHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqStandbyHosts[0]): message = "HAWQ Standby Master and Ambari Server should not be deployed on the same host. " \ - "If you leave them collocated, make sure to set HAWQ Master Port property " \ + "If you leave them colocated, make sure to set HAWQ Master Port property " \ "to a value different from the port number used by Ambari Server database." childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } ) if "PXF" in servicesList: pxfHosts = self.__getHosts(componentsList, "PXF") - namenodeHosts = self.__getHosts(componentsList, "NAMENODE") - datanodeHosts = self.__getHosts(componentsList, "DATANODE") - - # Generate WARNING if any PXF is not co-located with a NAMENODE - mismatchHosts = sorted(set(namenodeHosts).difference(set(pxfHosts))) - if len(mismatchHosts) > 0: - hostsString = ', '.join(mismatchHosts) - message = "PXF is not co-located with NAMENODE on the following {0} host(s): {1}".format(len(mismatchHosts), hostsString) - childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PXF' } ) - - # Generate WARNING if any PXF is not co-located with a DATANODE - mismatchHosts = sorted(set(datanodeHosts).difference(set(pxfHosts))) - if len(mismatchHosts) > 0: - hostsString = ', '.join(mismatchHosts) - message = "PXF is not co-located with DATANODE on the following {0} host(s): {1}".format(len(mismatchHosts), hostsString) - childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PXF' } ) + expectedPxfHosts = set(self.__getHosts(componentsList, "NAMENODE") + self.__getHosts(componentsList, "DATANODE")) - # Generate WARNING if any PXF is not co-located with a NAMENODE or a DATANODE - mismatchHosts = sorted(set(pxfHosts).difference(set(namenodeHosts).union(set(datanodeHosts)))) + # Generate WARNING if any PXF is not colocated with NAMENODE or DATANODE + mismatchHosts = sorted(expectedPxfHosts.symmetric_difference(set(pxfHosts))) if len(mismatchHosts) > 0: hostsString = ', '.join(mismatchHosts) - message = "PXF is not co-located with NAMENODE or DATANODE on the following {0} host(s): {1}".format(len(mismatchHosts), hostsString) + message = "PXF must be installed on the NameNode, Standby NameNode and all DataNodes. " \ + "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format(len(mismatchHosts), hostsString) childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PXF' } ) if "SPARK" in servicesList: @@ -947,7 +934,9 @@ class HDP23StackAdvisor(HDP22StackAdvisor): prop_name = 'hawq_master_address_port' validationItems.append({"config-name": prop_name, "item": self.getWarnItem( - "HAWQ Master or Standby Master cannot use the port 5432 when installed on the same host as the Ambari Server. Ambari Postgres DB uses the same port. Please choose a different value (e.g. 10432)")}) + "The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Master port. " + "If you are using port 5432 for Postgres, you must either deploy the HAWQ Master on a different host " + "or configure a different port for the HAWQ Master in the HAWQ Configuration page.")}) # 2. Check if any data directories are pointing to root dir '/' directories = { http://git-wip-us.apache.org/repos/asf/ambari/blob/c0ecec05/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 8932bde..cf1f606 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -170,9 +170,11 @@ class TestHDP23StackAdvisor(TestCase): component_names = [component["name"] for component in hostgroup["components"]] if "NAMENODE" in component_names or "DATANODE" in component_names: self.assertTrue("PXF" in component_names) + if "NAMENODE" not in component_names and "DATANODE" not in component_names: + self.assertTrue("PXF" not in component_names) - def test_hawqsegmentDatanode(self): + def test_createComponentLayoutRecommendations_hawqsegment_co_locate_datanode(self): """ Test that HAWQSegment gets recommended on same host group which has DATANODE""" # Case 1: HDFS is already installed, HAWQ is being added during Add Service Wizard @@ -275,6 +277,7 @@ class TestHDP23StackAdvisor(TestCase): self.assertTrue('HAWQSEGMENT' in component_names) self.assertTrue('DATANODE' in component_names) + def fqdn_mock_result(value=None): return 'c6401.ambari.apache.org' if value is None else value @@ -283,7 +286,7 @@ class TestHDP23StackAdvisor(TestCase): return [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"] == componentName][0] - def test_validations_hawqsegment(self): + def test_getComponentLayoutValidations_hawqsegment_not_co_located_with_datanode(self): """ Test validation warning for HAWQ segment not colocated with DATANODE """ services = self.load_json("services-normal-hawq-3-hosts.json") @@ -297,7 +300,12 @@ class TestHDP23StackAdvisor(TestCase): datanodeComponent["hostnames"] = ['c6402.ambari.apache.org'] validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) - expected = {'component-name': 'HAWQSEGMENT', 'message': 'HAWQSEGMENT is not co-located with DATANODE on the following 2 host(s): c6402.ambari.apache.org, c6401.ambari.apache.org', 'type': 'host-component', 'level': 'WARN'} + expected = { + 'type': 'host-component', + 'level': 'WARN', + 'component-name': 'HAWQSEGMENT', + 'message': 'HAWQ Segment must be installed on all DataNodes. The following 2 host(s) do not satisfy the colocation recommendation: c6401.ambari.apache.org, c6402.ambari.apache.org', + } self.assertEquals(validations[0], expected) datanodeComponent["hostnames"] = ['c6401.ambari.apache.org'] @@ -325,11 +333,11 @@ class TestHDP23StackAdvisor(TestCase): validations = [validation for validation in self.stackAdvisor.getComponentLayoutValidations(services, hosts) if validation["component-name"] == "PXF"] self.assertEquals(len(validations), 1) expected = { - "type": 'host-component', - "level": 'WARN', - "message": 'PXF is not co-located with NAMENODE on the following 1 host(s): c6401.ambari.apache.org', - "component-name": 'PXF' - } + "type": 'host-component', + "level": 'WARN', + "component-name": 'PXF', + "message": 'PXF must be installed on the NameNode, Standby NameNode and all DataNodes. The following 1 host(s) do not satisfy the colocation recommendation: c6401.ambari.apache.org' + } self.assertEquals(validations[0], expected) @@ -353,11 +361,11 @@ class TestHDP23StackAdvisor(TestCase): validations = [validation for validation in self.stackAdvisor.getComponentLayoutValidations(services, hosts) if validation["component-name"] == "PXF"] self.assertEquals(len(validations), 1) expected = { - "type": 'host-component', - "level": 'WARN', - "message": 'PXF is not co-located with DATANODE on the following 2 host(s): c6402.ambari.apache.org, c6403.ambari.apache.org', - "component-name": 'PXF' - } + "type": 'host-component', + "level": 'WARN', + "component-name": 'PXF', + "message": 'PXF must be installed on the NameNode, Standby NameNode and all DataNodes. The following 2 host(s) do not satisfy the colocation recommendation: c6402.ambari.apache.org, c6403.ambari.apache.org' + } self.assertEquals(validations[0], expected) @@ -381,11 +389,11 @@ class TestHDP23StackAdvisor(TestCase): validations = [validation for validation in self.stackAdvisor.getComponentLayoutValidations(services, hosts) if validation["component-name"] == "PXF"] self.assertEquals(len(validations), 1) expected = { - "type": 'host-component', - "level": 'WARN', - "message": 'PXF is not co-located with NAMENODE or DATANODE on the following 1 host(s): c6403.ambari.apache.org', - "component-name": 'PXF' - } + "type": 'host-component', + "level": 'WARN', + "component-name": 'PXF', + "message": 'PXF must be installed on the NameNode, Standby NameNode and all DataNodes. The following 1 host(s) do not satisfy the colocation recommendation: c6403.ambari.apache.org' + } self.assertEquals(validations[0], expected) @@ -443,7 +451,7 @@ class TestHDP23StackAdvisor(TestCase): validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) self.assertEquals(len(validations), 1) - expected={'component-name': 'HAWQSTANDBY', 'message': 'HAWQ Standby Master and HAWQ Master should not be deployed on the same host.', 'type': 'host-component', 'host': 'c6403.ambari.apache.org', 'level': 'ERROR'} + expected={'component-name': 'HAWQSTANDBY', 'message': 'HAWQ Master and HAWQ Standby Master cannot be deployed on the same host.', 'type': 'host-component', 'host': 'c6403.ambari.apache.org', 'level': 'ERROR'} self.assertEquals(validations[0], expected) # case-3: HAWQ Master and Ambari Server are collocated @@ -459,7 +467,7 @@ class TestHDP23StackAdvisor(TestCase): validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) self.assertEquals(len(validations), 1) - expected={'component-name': 'HAWQMASTER', 'message': 'HAWQ Master and Ambari Server should not be deployed on the same host. If you leave them collocated, make sure to set HAWQ Master Port property to a value different from the port number used by Ambari Server database.', 'type': 'host-component', 'host': 'c6401.ambari.apache.org', 'level': 'WARN'} + expected={'component-name': 'HAWQMASTER', 'message': 'HAWQ Master and Ambari Server should not be deployed on the same host. If you leave them colocated, make sure to set HAWQ Master Port property to a value different from the port number used by Ambari Server database.', 'type': 'host-component', 'host': 'c6401.ambari.apache.org', 'level': 'WARN'} self.assertEquals(validations[0], expected) # case-4: HAWQ Standby and Ambari Server are collocated @@ -475,7 +483,7 @@ class TestHDP23StackAdvisor(TestCase): validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) self.assertEquals(len(validations), 1) - expected={'component-name': 'HAWQSTANDBY', 'message': 'HAWQ Standby Master and Ambari Server should not be deployed on the same host. If you leave them collocated, make sure to set HAWQ Master Port property to a value different from the port number used by Ambari Server database.', 'type': 'host-component', 'host': 'c6401.ambari.apache.org', 'level': 'WARN'} + expected={'component-name': 'HAWQSTANDBY', 'message': 'HAWQ Standby Master and Ambari Server should not be deployed on the same host. If you leave them colocated, make sure to set HAWQ Master Port property to a value different from the port number used by Ambari Server database.', 'type': 'host-component', 'host': 'c6401.ambari.apache.org', 'level': 'WARN'} self.assertEquals(validations[0], expected) @@ -1744,6 +1752,7 @@ class TestHDP23StackAdvisor(TestCase): problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts) self.assertEqual(len(problems), 0) + def test_validateHAWQConfigurations(self): services = self.load_json("services-hawq-3-hosts.json") # setup default configuration values @@ -1815,15 +1824,57 @@ class TestHDP23StackAdvisor(TestCase): """ problems = self.stackAdvisor.validateHAWQConfigurations(properties, defaults, services["configurations"], services, hosts) self.assertEqual(len(problems), 1) - self.assertEqual(problems[0]["config-type"], "hawq-site") - self.assertEqual(problems[0]["message"], "hawq_global_rm_type must be set to none if YARN service is not installed") - self.assertEqual(problems[0]["type"], "configuration") - self.assertEqual(problems[0]["config-name"], "hawq_global_rm_type") - self.assertEqual(problems[0]["level"], "ERROR") - + expected = { + "config-type": "hawq-site", + "message": "hawq_global_rm_type must be set to none if YARN service is not installed", + "type": "configuration", + "config-name": "hawq_global_rm_type", + "level": "ERROR" + } + self.assertEqual(problems[0], expected) # case 2: hawq_global_rm_type is set as yarn, and YARN service is installed. No validation errors expected. services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]}) problems = self.stackAdvisor.validateHAWQConfigurations(properties, defaults, services["configurations"], services, hosts) self.assertEqual(len(problems), 0) + + # Test HAWQ Master port conflict with Ambari Server Postgres port + + # case 1: HAWQ Master is placed on Ambari Server and HAWQ Master port is same as Ambari Server Postgres Port + self.stackAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=True) + configurations = { + "hawq-site": { + "properties": + {"hawq_master_address_port": "5432"} + } + } + problems = self.stackAdvisor.validateHAWQConfigurations(properties, defaults, configurations, services, hosts) + self.assertEqual(len(problems), 1) + expected = { + "config-name": "hawq_master_address_port", + "config-type": "hawq-site", + "level": "WARN", + "message": "The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Master port. " + "If you are using port 5432 for Postgres, you must either deploy the HAWQ Master on a different host " + "or configure a different port for the HAWQ Master in the HAWQ Configuration page.", + "type": "configuration"} + self.assertEqual(problems[0], expected) + + # case 2: HAWQ Master is placed on Ambari Server and HAWQ Master port is different from Ambari Server Postgres Port + self.stackAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=True) + configurations["hawq-site"]["properties"]["hawq_master_address_port"] = "10432" + problems = self.stackAdvisor.validateHAWQConfigurations(properties, defaults, configurations, services, hosts) + self.assertEqual(len(problems), 0) + + # case 3: HAWQ Master is not placed on Ambari Server and HAWQ Master port is same as Ambari Server Postgres Port + self.stackAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=False) + configurations["hawq-site"]["properties"]["hawq_master_address_port"] = "5432" + problems = self.stackAdvisor.validateHAWQConfigurations(properties, defaults, configurations, services, hosts) + self.assertEqual(len(problems), 0) + + # case 4: HAWQ Master is not placed on Ambari Server and HAWQ Master port is different from Ambari Server Postgres Port + self.stackAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=False) + configurations["hawq-site"]["properties"]["hawq_master_address_port"] = "10432" + problems = self.stackAdvisor.validateHAWQConfigurations(properties, defaults, configurations, services, hosts) + self.assertEqual(len(problems), 0) \ No newline at end of file