http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git
a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 2944f6f..4dfb8af 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -20,7 +20,7 @@ import json
import os
import socket
from unittest import TestCase
-from mock.mock import patch, MagicMock
+from mock.mock import patch
class TestHDP23StackAdvisor(TestCase):
@@ -86,694 +86,10 @@ class TestHDP23StackAdvisor(TestCase):
return self.get_system_min_uid_real()
- def test_createComponentLayoutRecommendations_hawq_1_Host(self):
-
- services = self.load_json("services-hawq-1-host.json")
- componentsListList = [service["components"] for service in
services["services"]]
- componentsList = [item for sublist in componentsListList for item in
sublist]
- componentNames = [component["StackServiceComponents"]["component_name"]
for component in componentsList]
- self.assertTrue('HAWQSTANDBY' in componentNames)
-
- hosts = self.load_json("hosts-1-host.json")
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- self.assertEquals(len(hostsList), 1)
-
- self.insertHAWQServiceAdvisorInfo(services)
- recommendations =
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
-
- recommendedComponentsListList = [hostgroup["components"] for hostgroup in
recommendations["blueprint"]["host_groups"]]
- recommendedComponents = [item["name"] for sublist in
recommendedComponentsListList for item in sublist]
- self.assertTrue('HAWQMASTER' in recommendedComponents)
- self.assertFalse('HAWQSTANDBY' in recommendedComponents)
- self.assertTrue('HAWQSEGMENT' in recommendedComponents)
-
- def insertHAWQServiceAdvisorInfo(self, services):
- for service in services["services"]:
- if service["StackServices"]["service_name"] == 'HAWQ':
- service["StackServices"]["advisor_name"] = "HAWQ200ServiceAdvisor"
- path = os.path.join(self.testDirectory,
'../../../../../main/resources/common-services/HAWQ/2.0.0/service_advisor.py')
- service["StackServices"]["advisor_path"] = path
-
- def insertPXFServiceAdvisorInfo(self, services):
- for service in services["services"]:
- if service["StackServices"]["service_name"] == 'PXF':
- service["StackServices"]["advisor_name"] = "PXF300ServiceAdvisor"
- path = os.path.join(self.testDirectory,
'../../../../../main/resources/common-services/PXF/3.0.0/service_advisor.py')
- service["StackServices"]["advisor_path"] = path
-
- def test_createComponentLayoutRecommendations_hawq_3_Hosts(self):
- """ Test that HAWQSTANDBY is recommended on a 3-node cluster """
-
- services = self.load_json("services-hawq-3-hosts.json")
- componentsListList = [service["components"] for service in
services["services"]]
- componentsList = [item for sublist in componentsListList for item in
sublist]
- componentNames = [component["StackServiceComponents"]["component_name"]
for component in componentsList]
- self.assertTrue('HAWQSTANDBY' in componentNames)
-
- hosts = self.load_json("hosts-3-hosts.json")
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- self.assertEquals(len(hostsList), 3)
-
- self.insertHAWQServiceAdvisorInfo(services)
- recommendations = self.stackAdvisor.recommendComponentLayout(services,
hosts)
- layoutRecommendations = recommendations["recommendations"]
-
- recommendedComponentsListList = [hostgroup["components"] for hostgroup in
layoutRecommendations["blueprint"]["host_groups"]]
- recommendedComponents = [item["name"] for sublist in
recommendedComponentsListList for item in sublist]
- self.assertTrue('HAWQMASTER' in recommendedComponents)
- self.assertTrue('HAWQSTANDBY' in recommendedComponents)
- self.assertTrue('HAWQSEGMENT' in recommendedComponents)
-
- # make sure master components are not collocated
- for sublist in recommendedComponentsListList:
- hostComponents = [item["name"] for item in sublist]
- self.assertFalse(set(['HAWQMASTER',
'HAWQSTANDBY']).issubset(hostComponents))
-
-
- def test_createComponentLayoutRecommendations_no_hawq_3_Hosts(self):
- """ Test no failures when there are no HAWQ components """
-
- services = self.load_json("services-nohawq-3-hosts.json")
- componentsListList = [service["components"] for service in
services["services"]]
- componentsList = [item for sublist in componentsListList for item in
sublist]
- componentNames = [component["StackServiceComponents"]["component_name"]
for component in componentsList]
- self.assertFalse('HAWQMASTER' in componentNames)
- self.assertFalse('HAWQSTANDBY' in componentNames)
- self.assertFalse('HAWQSEGMENT' in componentNames)
-
- hosts = self.load_json("hosts-3-hosts.json")
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- self.assertEquals(len(hostsList), 3)
-
- self.insertHAWQServiceAdvisorInfo(services)
- recommendations = self.stackAdvisor.recommendComponentLayout(services,
hosts)
- layoutRecommendations = recommendations["recommendations"]
-
- recommendedComponentsListList = [hostgroup["components"] for hostgroup in
layoutRecommendations["blueprint"]["host_groups"]]
- recommendedComponents = [item["name"] for sublist in
recommendedComponentsListList for item in sublist]
- self.assertFalse('HAWQMASTER' in recommendedComponents)
- self.assertFalse('HAWQSTANDBY' in recommendedComponents)
- self.assertFalse('HAWQSEGMENT' in recommendedComponents)
-
- def
test_createComponentLayoutRecommendations_hawqsegment_cluster_install(self):
- """ Test that HAWQSEGMENT gets recommended correctly during Cluster
Install Wizard, when HAWQ is selected for installation """
-
- hosts = self.prepareHosts(["c6401.ambari.apache.org",
"c6402.ambari.apache.org", "c6403.ambari.apache.org"])
- services = {
- "services" : [
- {
- "StackServices" : {
- "service_name" : "HDFS"
- },
- "components" : [
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "SLAVE",
- "component_name" : "DATANODE",
- "hostnames" : []
- }
- }
- ]
- },
- {
- "StackServices" : {
- "service_name" : "HAWQ"
- },
- "components" : [
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "SLAVE",
- "component_name" : "HAWQSEGMENT",
- "hostnames" : []
- }
- }
- ]
- }
- ]
- }
-
- hawqSegmentHosts = set(["c6401.ambari.apache.org",
"c6402.ambari.apache.org", "c6403.ambari.apache.org"])
- self.insertHAWQServiceAdvisorInfo(services)
- recommendations =
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
- hostGroups = [ hostgroup["name"] for hostgroup in
recommendations["blueprint"]["host_groups"] if {"name": "HAWQSEGMENT"} in
hostgroup["components"] ]
- hostNames = [ host["fqdn"] for hostgroup in
recommendations["blueprint_cluster_binding"]["host_groups"] if
hostgroup["name"] in hostGroups for host in hostgroup["hosts"] ]
- self.assertEquals(set(hostNames), hawqSegmentHosts)
-
- def
test_createComponentLayoutRecommendations_hawqsegment_add_service_wizard_to_be_installed(self):
- """ Test that HAWQSEGMENT gets recommended correctly during Add Service
Wizard, when HAWQ is selected for installation """
-
- hosts = self.prepareHosts(["c6401.ambari.apache.org",
"c6402.ambari.apache.org", "c6403.ambari.apache.org"])
- services = {
- "services" : [
- {
- "StackServices" : {
- "service_name" : "HDFS"
- },
- "components" : [
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "SLAVE",
- "component_name" : "DATANODE",
- "hostnames" : ["c6401.ambari.apache.org",
"c6403.ambari.apache.org"]
- }
- }
- ]
- },
- {
- "StackServices" : {
- "service_name" : "HAWQ"
- },
- "components" : [
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "SLAVE",
- "component_name" : "HAWQSEGMENT",
- "hostnames" : []
- }
- }
- ]
- }
- ]
- }
-
- hawqSegmentHosts = set(["c6401.ambari.apache.org",
"c6403.ambari.apache.org"])
- self.insertHAWQServiceAdvisorInfo(services)
- recommendations =
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
- hostGroups = [ hostgroup["name"] for hostgroup in
recommendations["blueprint"]["host_groups"] if {"name": "HAWQSEGMENT"} in
hostgroup["components"] ]
- hostNames = [ host["fqdn"] for hostgroup in
recommendations["blueprint_cluster_binding"]["host_groups"] if
hostgroup["name"] in hostGroups for host in hostgroup["hosts"] ]
- self.assertEquals(set(hostNames), hawqSegmentHosts)
-
- def
test_createComponentLayoutRecommendations_hawqsegment_add_service_wizard_already_installed(self):
- """ Test that HAWQSEGMENT does not get recommended during Add Service
Wizard, when HAWQ has already been installed """
-
- hosts = self.prepareHosts(["c6401.ambari.apache.org",
"c6402.ambari.apache.org", "c6403.ambari.apache.org"])
- services = {
- "services" : [
- {
- "StackServices" : {
- "service_name" : "HDFS"
- },
- "components" : [
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "SLAVE",
- "component_name" : "DATANODE",
- "hostnames" : ["c6401.ambari.apache.org",
"c6403.ambari.apache.org"]
- }
- }
- ]
- },
- {
- "StackServices" : {
- "service_name" : "HAWQ"
- },
- "components" : [
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "SLAVE",
- "component_name" : "HAWQSEGMENT",
- "hostnames" : ["c6402.ambari.apache.org"]
- }
- }
- ]
- },
- {
- "StackServices" : {
- "service_name" : "PXF"
- },
- "components" : [
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "SLAVE",
- "component_name" : "PXF",
- "hostnames" : []
- }
- }
- ]
- }
- ]
- }
-
- hawqSegmentHosts = set(["c6402.ambari.apache.org"])
- self.insertHAWQServiceAdvisorInfo(services)
- recommendations =
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
- hostGroups = [ hostgroup["name"] for hostgroup in
recommendations["blueprint"]["host_groups"] if {"name": "HAWQSEGMENT"} in
hostgroup["components"] ]
- hostNames = [ host["fqdn"] for hostgroup in
recommendations["blueprint_cluster_binding"]["host_groups"] if
hostgroup["name"] in hostGroups for host in hostgroup["hosts"] ]
- self.assertEquals(set(hostNames), hawqSegmentHosts)
-
- def test_createComponentLayoutRecommendations_pxf_cluster_install(self):
- """ Test that PXF gets recommended correctly during Cluster Install
Wizard, when PXF is selected for installation """
-
- hosts = self.prepareHosts(["c6401.ambari.apache.org",
"c6402.ambari.apache.org", "c6403.ambari.apache.org"])
- services = {
- "services" : [
- {
- "StackServices" : {
- "service_name" : "HDFS"
- },
- "components" : [
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "MASTER",
- "component_name" : "NAMENODE",
- "is_master": "true",
- "hostnames" : []
- }
- },
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "SLAVE",
- "component_name" : "DATANODE",
- "hostnames" : []
- }
- }
- ]
- },
- {
- "StackServices" : {
- "service_name" : "PXF"
- },
- "components" : [
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "SLAVE",
- "component_name" : "PXF",
- "hostnames" : []
- }
- }
- ]
- }
- ]
- }
-
- pxfHosts = set(["c6401.ambari.apache.org", "c6402.ambari.apache.org",
"c6403.ambari.apache.org"])
-
- self.insertPXFServiceAdvisorInfo(services)
- recommendations =
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
- hostGroups = [ hostgroup["name"] for hostgroup in
recommendations["blueprint"]["host_groups"] if {"name": "PXF"} in
hostgroup["components"] ]
- hostNames = [ host["fqdn"] for hostgroup in
recommendations["blueprint_cluster_binding"]["host_groups"] if
hostgroup["name"] in hostGroups for host in hostgroup["hosts"] ]
- self.assertEquals(set(hostNames), pxfHosts)
-
- def
test_createComponentLayoutRecommendations_pxf_add_service_wizard_to_be_installed(self):
- """ Test that PXF gets recommended correctly during Add Service Wizard,
when PXF is selected for installation """
-
- hosts = self.prepareHosts(["c6401.ambari.apache.org",
"c6402.ambari.apache.org", "c6403.ambari.apache.org"])
- services = {
- "services" : [
- {
- "StackServices" : {
- "service_name" : "HDFS"
- },
- "components" : [
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "MASTER",
- "component_name" : "NAMENODE",
- "is_master": "true",
- "hostnames" : ["c6401.ambari.apache.org"]
- }
- },
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "SLAVE",
- "component_name" : "DATANODE",
- "hostnames" : ["c6402.ambari.apache.org",
"c6403.ambari.apache.org"]
- }
- }
- ]
- },
- {
- "StackServices" : {
- "service_name" : "PXF"
- },
- "components" : [
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "SLAVE",
- "component_name" : "PXF",
- "hostnames" : []
- }
- }
- ]
- }
- ]
- }
-
- pxfHosts = set(["c6401.ambari.apache.org", "c6402.ambari.apache.org",
"c6403.ambari.apache.org"])
-
- self.insertPXFServiceAdvisorInfo(services)
- recommendations =
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
- hostGroups = [ hostgroup["name"] for hostgroup in
recommendations["blueprint"]["host_groups"] if {"name": "PXF"} in
hostgroup["components"] ]
- hostNames = [ host["fqdn"] for hostgroup in
recommendations["blueprint_cluster_binding"]["host_groups"] if
hostgroup["name"] in hostGroups for host in hostgroup["hosts"] ]
- self.assertEquals(set(hostNames), pxfHosts)
-
- def
test_createComponentLayoutRecommendations_pxf_add_service_wizard_already_installed(self):
- """ Test that PXF does not get recommended during Add Service Wizard, when
PXF has already been installed """
-
- hosts = self.prepareHosts(["c6401.ambari.apache.org",
"c6402.ambari.apache.org", "c6403.ambari.apache.org"])
- services = {
- "services" : [
- {
- "StackServices" : {
- "service_name" : "HDFS"
- },
- "components" : [
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "MASTER",
- "component_name" : "NAMENODE",
- "is_master": "true",
- "hostnames" : ["c6401.ambari.apache.org"]
- }
- },
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "SLAVE",
- "component_name" : "DATANODE",
- "hostnames" : ["c6402.ambari.apache.org",
"c6403.ambari.apache.org"]
- }
- }
- ]
- },
- {
- "StackServices" : {
- "service_name" : "PXF"
- },
- "components" : [
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "SLAVE",
- "component_name" : "PXF",
- "hostnames" : ["c6402.ambari.apache.org"]
- }
- }
- ]
- },
- {
- "StackServices" : {
- "service_name" : "HBASE"
- },
- "components" : [
- {
- "StackServiceComponents" : {
- "cardinality" : "1+",
- "component_category" : "MASTER",
- "component_name" : "HBASE_MASTER",
- "is_master": "true",
- "hostnames" : []
- }
- }
- ]
- }
- ]
- }
-
- pxfHosts = set(["c6402.ambari.apache.org"])
-
- self.insertPXFServiceAdvisorInfo(services)
- recommendations =
self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
- hostGroups = [ hostgroup["name"] for hostgroup in
recommendations["blueprint"]["host_groups"] if {"name": "PXF"} in
hostgroup["components"] ]
- hostNames = [ host["fqdn"] for hostgroup in
recommendations["blueprint_cluster_binding"]["host_groups"] if
hostgroup["name"] in hostGroups for host in hostgroup["hosts"] ]
- self.assertEquals(set(hostNames), pxfHosts)
-
def fqdn_mock_result(value=None):
return 'c6401.ambari.apache.org' if value is None else value
- def __getHosts(self, componentsList, componentName):
- return [component["StackServiceComponents"] for component in
componentsList if component["StackServiceComponents"]["component_name"] ==
componentName][0]
-
-
- def
test_getComponentLayoutValidations_hawqsegment_not_co_located_with_datanode(self):
- """ Test validation warning for HAWQ segment not colocated with DATANODE
"""
-
- services = self.load_json("services-normal-hawq-3-hosts.json")
- hosts = self.load_json("hosts-3-hosts.json")
- componentsListList = [service["components"] for service in
services["services"]]
- componentsList = [item for sublist in componentsListList for item in
sublist]
-
- hawqsegmentComponent = self.__getHosts(componentsList, "HAWQSEGMENT")
- hawqsegmentComponent["hostnames"] = ['c6401.ambari.apache.org']
- datanodeComponent = self.__getHosts(componentsList, "DATANODE")
- datanodeComponent["hostnames"] = ['c6402.ambari.apache.org']
-
- self.insertHAWQServiceAdvisorInfo(services)
- validations = self.stackAdvisor.getComponentLayoutValidations(services,
hosts)
- expected = {
- 'type': 'host-component',
- 'level': 'WARN',
- 'component-name': 'HAWQSEGMENT',
- 'message': 'HAWQ Segment must be installed on all DataNodes. The
following 2 host(s) do not satisfy the colocation recommendation:
c6401.ambari.apache.org, c6402.ambari.apache.org',
- }
- self.assertEquals(validations[0], expected)
-
- datanodeComponent["hostnames"] = ['c6401.ambari.apache.org']
- validations = self.stackAdvisor.getComponentLayoutValidations(services,
hosts)
- self.assertEquals(len(validations), 0)
-
-
- def test_getComponentLayoutValidations_pxf_not_co_located_with_nn(self):
- """ Test warning is generated when PXF is not co-located with NAMENODE """
-
- services = self.load_json("services-hawq-pxf-hdfs.json")
- componentsListList = [service["components"] for service in
services["services"]]
- componentsList = [item for sublist in componentsListList for item in
sublist]
- nameNodeComponent = self.__getHosts(componentsList, "NAMENODE")
- dataNodeComponent = self.__getHosts(componentsList, "DATANODE")
- pxfComponent = self.__getHosts(componentsList, "PXF")
- nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"]
- dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org",
"c6403.ambari.apache.org"]
- pxfComponent["hostnames"] = ["c6402.ambari.apache.org",
"c6403.ambari.apache.org"]
-
- hosts = self.load_json("hosts-3-hosts.json")
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- self.assertEquals(len(hostsList), 3)
-
- self.insertPXFServiceAdvisorInfo(services)
- validations = [validation for validation in
self.stackAdvisor.getComponentLayoutValidations(services, hosts) if
validation["component-name"] == "PXF"]
- self.assertEquals(len(validations), 1)
- expected = {
- "type": 'host-component',
- "level": 'WARN',
- "component-name": 'PXF',
- "message": 'PXF must be installed on the NameNode, Standby NameNode and
all DataNodes. The following 1 host(s) do not satisfy the colocation
recommendation: c6401.ambari.apache.org'
- }
- self.assertEquals(validations[0], expected)
-
-
- def test_getComponentLayoutValidations_pxf_not_co_located_with_dn(self):
- """ Test warning is generated when PXF is not co-located with NAMENODE or
DATANODE """
-
- services = self.load_json("services-hawq-pxf-hdfs.json")
- componentsListList = [service["components"] for service in
services["services"]]
- componentsList = [item for sublist in componentsListList for item in
sublist]
- nameNodeComponent = self.__getHosts(componentsList, "NAMENODE")
- dataNodeComponent = self.__getHosts(componentsList, "DATANODE")
- pxfComponent = self.__getHosts(componentsList, "PXF")
- nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"]
- dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org",
"c6403.ambari.apache.org"]
- pxfComponent["hostnames"] = ["c6401.ambari.apache.org"]
-
- hosts = self.load_json("hosts-3-hosts.json")
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- self.assertEquals(len(hostsList), 3)
-
- self.insertPXFServiceAdvisorInfo(services)
- validations = [validation for validation in
self.stackAdvisor.getComponentLayoutValidations(services, hosts) if
validation["component-name"] == "PXF"]
- self.assertEquals(len(validations), 1)
- expected = {
- "type": 'host-component',
- "level": 'WARN',
- "component-name": 'PXF',
- "message": 'PXF must be installed on the NameNode, Standby NameNode and
all DataNodes. The following 2 host(s) do not satisfy the colocation
recommendation: c6402.ambari.apache.org, c6403.ambari.apache.org'
- }
- self.assertEquals(validations[0], expected)
-
-
- def
test_getComponentLayoutValidations_pxf_not_co_located_with_nn_or_dn(self):
- """ Test warning is generated when PXF is not co-located with NAMENODE or
DATANODE """
-
- services = self.load_json("services-hawq-pxf-hdfs.json")
- componentsListList = [service["components"] for service in
services["services"]]
- componentsList = [item for sublist in componentsListList for item in
sublist]
- nameNodeComponent = self.__getHosts(componentsList, "NAMENODE")
- dataNodeComponent = self.__getHosts(componentsList, "DATANODE")
- pxfComponent = self.__getHosts(componentsList, "PXF")
- nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"]
- dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org"]
- pxfComponent["hostnames"] = ["c6401.ambari.apache.org",
"c6402.ambari.apache.org", "c6403.ambari.apache.org"]
-
- hosts = self.load_json("hosts-3-hosts.json")
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- self.assertEquals(len(hostsList), 3)
-
- self.insertPXFServiceAdvisorInfo(services)
- validations = [validation for validation in
self.stackAdvisor.getComponentLayoutValidations(services, hosts) if
validation["component-name"] == "PXF"]
- self.assertEquals(len(validations), 1)
- expected = {
- "type": 'host-component',
- "level": 'WARN',
- "component-name": 'PXF',
- "message": 'PXF must be installed on the NameNode, Standby NameNode and
all DataNodes. The following 1 host(s) do not satisfy the colocation
recommendation: c6403.ambari.apache.org'
- }
- self.assertEquals(validations[0], expected)
-
-
- def test_getComponentLayoutValidations_pxf_co_located_with_nn_and_dn(self):
- """ Test NO warning is generated when PXF is co-located with NAMENODE and
DATANODE """
-
- services = self.load_json("services-hawq-pxf-hdfs.json")
- componentsListList = [service["components"] for service in
services["services"]]
- componentsList = [item for sublist in componentsListList for item in
sublist]
- nameNodeComponent = self.__getHosts(componentsList, "NAMENODE")
- dataNodeComponent = self.__getHosts(componentsList, "DATANODE")
- pxfComponent = self.__getHosts(componentsList, "PXF")
- nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"]
- dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org",
"c6403.ambari.apache.org"]
- pxfComponent["hostnames"] = ["c6401.ambari.apache.org",
"c6402.ambari.apache.org", "c6403.ambari.apache.org"]
-
- hosts = self.load_json("hosts-3-hosts.json")
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- self.assertEquals(len(hostsList), 3)
-
- self.insertPXFServiceAdvisorInfo(services)
- validations = [validation for validation in
self.stackAdvisor.getComponentLayoutValidations(services, hosts) if
validation["component-name"] == "PXF"]
- self.assertEquals(len(validations), 0)
-
-
- @patch('socket.getfqdn', side_effect=fqdn_mock_result)
- def test_getComponentLayoutValidations_hawq_3_Hosts(self, socket_mock):
- """ Test layout validations for HAWQ components on a 3-node cluster """
-
- # case-1: normal placement, no warnings
- services = self.load_json("services-normal-hawq-3-hosts.json")
- componentsListList = [service["components"] for service in
services["services"]]
- componentsList = [item for sublist in componentsListList for item in
sublist]
- hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for
component in componentsList if
component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
- hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for
component in componentsList if
component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
- self.assertEquals(len(hawqMasterHosts[0]), 1)
- self.assertEquals(len(hawqStandbyHosts[0]), 1)
- self.assertNotEquals(hawqMasterHosts[0][0], hawqStandbyHosts[0][0])
-
- hosts = self.load_json("hosts-3-hosts.json")
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- self.assertEquals(len(hostsList), 3)
-
- self.insertHAWQServiceAdvisorInfo(services)
- validations = self.stackAdvisor.getComponentLayoutValidations(services,
hosts)
- self.assertEquals(len(validations), 0)
-
- # case-2: HAWQ masters are collocated
- services = self.load_json("services-master_standby_colo-3-hosts.json")
- componentsListList = [service["components"] for service in
services["services"]]
- componentsList = [item for sublist in componentsListList for item in
sublist]
- hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for
component in componentsList if
component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
- hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for
component in componentsList if
component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
- self.assertEquals(len(hawqMasterHosts[0]), 1)
- self.assertEquals(len(hawqStandbyHosts[0]), 1)
- self.assertEquals(hawqMasterHosts[0][0], hawqStandbyHosts[0][0])
-
- self.insertHAWQServiceAdvisorInfo(services)
- validations = self.stackAdvisor.getComponentLayoutValidations(services,
hosts)
- self.assertEquals(len(validations), 1)
- expected = {
- 'component-name': 'HAWQSTANDBY',
- 'message': 'HAWQ Master and HAWQ Standby Master cannot be deployed on
the same host.',
- 'type': 'host-component',
- 'host': 'c6403.ambari.apache.org',
- 'level': 'ERROR'
- }
- self.assertEquals(validations[0], expected)
-
- # case-3: HAWQ Master and Ambari Server are collocated
- services = self.load_json("services-master_ambari_colo-3-hosts.json")
- componentsListList = [service["components"] for service in
services["services"]]
- componentsList = [item for sublist in componentsListList for item in
sublist]
- hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for
component in componentsList if
component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
- hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for
component in componentsList if
component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
- self.assertEquals(len(hawqMasterHosts[0]), 1)
- self.assertEquals(len(hawqStandbyHosts[0]), 1)
- self.assertNotEquals(hawqMasterHosts[0][0], hawqStandbyHosts[0][0])
- self.assertEquals(hawqMasterHosts[0][0], "c6401.ambari.apache.org")
-
- self.insertHAWQServiceAdvisorInfo(services)
- validations = self.stackAdvisor.getComponentLayoutValidations(services,
hosts)
- self.assertEquals(len(validations), 1)
- expected = {
- 'component-name': 'HAWQMASTER',
- 'message': 'The default Postgres port (5432) on the Ambari Server
conflicts with the default HAWQ Masters port. ' +
- 'If you are using port 5432 for Postgres, you must either
deploy the HAWQ Master on a different host ' +
- 'or configure a different port for the HAWQ Masters in the
HAWQ Configuration page.',
- 'type': 'host-component',
- 'host': 'c6401.ambari.apache.org',
- 'level': 'WARN'
- }
- self.assertEquals(validations[0], expected)
-
- # case-4: HAWQ Standby and Ambari Server are collocated
- services = self.load_json("services-standby_ambari_colo-3-hosts.json")
- componentsListList = [service["components"] for service in
services["services"]]
- componentsList = [item for sublist in componentsListList for item in
sublist]
- hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for
component in componentsList if
component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
- hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for
component in componentsList if
component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
- self.assertEquals(len(hawqMasterHosts[0]), 1)
- self.assertEquals(len(hawqStandbyHosts[0]), 1)
- self.assertNotEquals(hawqMasterHosts[0][0], hawqStandbyHosts[0][0])
- self.assertEquals(hawqStandbyHosts[0][0], "c6401.ambari.apache.org")
-
- self.insertHAWQServiceAdvisorInfo(services)
- validations = self.stackAdvisor.getComponentLayoutValidations(services,
hosts)
- self.assertEquals(len(validations), 1)
- expected = {
- 'component-name': 'HAWQSTANDBY',
- 'message': 'The default Postgres port (5432) on the Ambari Server
conflicts with the default HAWQ Masters port. ' +
- 'If you are using port 5432 for Postgres, you must either
deploy the HAWQ Standby Master on a different host ' +
- 'or configure a different port for the HAWQ Masters in the
HAWQ Configuration page.',
- 'type': 'host-component',
- 'host': 'c6401.ambari.apache.org',
- 'level': 'WARN'
- }
- self.assertEquals(validations[0], expected)
-
-
- @patch('socket.getfqdn', side_effect=fqdn_mock_result)
- def test_getComponentLayoutValidations_nohawq_3_Hosts(self, socket_mock):
- """ Test no failures when there are no HAWQ components on a 3-node cluster
"""
-
- # normal placement, no warnings
- services = self.load_json("services-normal-nohawq-3-hosts.json")
- componentsListList = [service["components"] for service in
services["services"]]
- componentsList = [item for sublist in componentsListList for item in
sublist]
- hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for
component in componentsList if
component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
- hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for
component in componentsList if
component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
- self.assertEquals(len(hawqMasterHosts), 0)
- self.assertEquals(len(hawqStandbyHosts), 0)
-
- hosts = self.load_json("hosts-3-hosts.json")
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- self.assertEquals(len(hostsList), 3)
-
- self.insertHAWQServiceAdvisorInfo(services)
- validations = self.stackAdvisor.getComponentLayoutValidations(services,
hosts)
- self.assertEquals(len(validations), 0)
-
-
@patch('socket.getfqdn', side_effect=fqdn_mock_result)
def test_getComponentLayoutValidations_sparkts_no_hive(self, socket_mock):
""" Test SparkTS is picked when Hive is not installed """
@@ -1989,112 +1305,6 @@ class TestHDP23StackAdvisor(TestCase):
self.stackAdvisor.recommendTezConfigurations(configurations, clusterData,
services, hosts)
self.assertEquals(configurations, expected)
- def createHAWQServiceAdvisor(self):
- path = os.path.join(self.testDirectory,
'../../../../../main/resources/common-services/HAWQ/2.0.0/service_advisor.py')
- service = {"service_name" : "HAWQ", "service_version" : "2.0",
"advisor_path" : path, "advisor_name" : "HAWQ200ServiceAdvisor"}
- service = {"StackServices" : service}
- return self.stackAdvisor.instantiateServiceAdvisor(service)
-
- def createPXFServiceAdvisor(self):
- path = os.path.join(self.testDirectory,
'../../../../../main/resources/common-services/PXF/3.0.0/service_advisor.py')
- service = {"service_name" : "PXF", "service_version" : "3.0",
"advisor_path" : path, "advisor_name" : "PXF300ServiceAdvisor"}
- service = {"StackServices" : service}
- return self.stackAdvisor.instantiateServiceAdvisor(service)
-
- def test_recommendHAWQConfigurations(self):
-
- hosts = {
- "items": [
- {
- "Hosts": {
- "host_name": "c6401.ambari.apache.org",
- "cpu_count" : 2,
- "total_mem": 33554432
- }
- },
- {
- "Hosts": {
- "host_name": "c6402.ambari.apache.org",
- "cpu_count" : 4,
- "total_mem": 33554433
- }
- },
- {
- "Hosts": {
- "host_name": "c6403.ambari.apache.org",
- "cpu_count" : 1,
- "total_mem": 33554434
- }
- },
- {
- "Hosts": {
- "host_name": "c6404.ambari.apache.org",
- "cpu_count" : 2,
- "total_mem": 33554435
- }
- }
- ]
- }
-
- # original cluster data with 3 segments
- services = self.load_json("services-normal-hawq-3-hosts.json")
- componentsListList = [service["components"] for service in
services["services"]]
- componentsList = [item for sublist in componentsListList for item in
sublist]
- hawqSegmentComponent = [component["StackServiceComponents"] for component
in componentsList if component["StackServiceComponents"]["component_name"] ==
"HAWQSEGMENT"][0]
-
- # setup default configuration values
- services["configurations"]["hawq-site"] = {"properties":
{"default_hash_table_bucket_number": "24",
-
"hawq_rm_nvseg_perquery_limit": "512",
-
"hawq_rm_yarn_address": "localhost:8032",
-
"hawq_rm_yarn_scheduler_address": "localhost:8030",
-
"hawq_global_rm_type": "none"}}
-
- services["configurations"]["hdfs-client"] = {"properties":
{"output.replace-datanode-on-failure": "true"}}
- services["configurations"]["hawq-sysctl-env"] = {"properties": {}}
-
- services["configurations"]["yarn-site"] = {"properties":
{"yarn.resourcemanager.address": "host1:8050",
-
"yarn.resourcemanager.scheduler.address": "host1:8030"}}
- services["services"].append({"StackServices" : {"service_name" : "YARN"},
"components":[]})
- configurations = {}
- clusterData = {}
-
- # Test 1 - with 3 segments
- self.assertEquals(len(hawqSegmentComponent["hostnames"]), 3)
- serviceAdvisor = self.createHAWQServiceAdvisor()
- serviceAdvisor.getServiceConfigurationRecommendations(configurations,
clusterData, services, hosts)
-
self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"],
str(3 * 6))
-
self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"],
"false")
-
- # check derived properties
-
self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_address"],
"host1:8050")
-
self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_scheduler_address"],
"host1:8030")
-
- # Test 2 - with 100 segments
- hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(100)]
- serviceAdvisor.getServiceConfigurationRecommendations(configurations,
clusterData, services, hosts)
-
self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"],
str(100 * 5))
-
self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"],
"true")
-
- # Test 3 - with 512 segments
- hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(512)]
- serviceAdvisor.getServiceConfigurationRecommendations(configurations,
clusterData, services, hosts)
-
self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"],
"512")
-
self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"],
"true")
-
- # Test 4 - with 513 segments
- hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(513)]
- serviceAdvisor.getServiceConfigurationRecommendations(configurations,
clusterData, services, hosts)
-
self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"],
"512")
-
self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"],
"true")
-
- # Test 5 - with no segments
- configurations = {}
- services["configurations"]["hawq-site"] =
{"properties":{"hawq_global_rm_type": "none"}}
- hawqSegmentComponent["hostnames"] = []
- serviceAdvisor.getServiceConfigurationRecommendations(configurations,
clusterData, services, hosts)
-
self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"],
"false")
- self.assertTrue("default_hash_table_bucket_number" not in
configurations["hawq-site"])
-
def test_validateHiveConfigurations(self):
properties = {"hive_security_authorization": "None",
"hive.exec.orc.default.stripe.size": "8388608",
@@ -2307,7 +1517,37 @@ class TestHDP23StackAdvisor(TestCase):
configurations = {
"core-site": {"properties": {}},
}
- services = self.load_json("services-hawq-3-hosts.json")
+ services = {
+ "services": [
+ {
+ "StackServices": {
+ "service_name": "HDFS"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "NAMENODE",
+ "hostnames": ["c6401.ambari.apache.org"]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices": {
+ "service_name": "HAWQ"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "HAWQMASTER",
+ "hostnames": ["c6401.ambari.apache.org"]
+ }
+ }
+ ]
+ }
+ ],
+ "configurations": configurations
+ }
services["configurations"]["hdfs-site"] = {}
services["configurations"]["hdfs-site"]["properties"] = {}
hosts = {}
@@ -2362,264 +1602,6 @@ class TestHDP23StackAdvisor(TestCase):
self.assertEqual(len(problems), 0)
- def test_validateHAWQSiteConfigurations(self):
- services = self.load_json("services-hawq-3-hosts.json")
- # setup default configuration values
- # Test hawq_rm_yarn_address and hawq_rm_scheduler_address are set correctly
- configurations = services["configurations"]
- configurations["hawq-site"] = {"properties": {"hawq_rm_yarn_address":
"localhost:8032",
-
"hawq_rm_yarn_scheduler_address": "localhost:8030"}}
- configurations["yarn-site"] = {"properties":
{"yarn.resourcemanager.address": "host1:8050",
-
"yarn.resourcemanager.scheduler.address": "host1:8030"}}
- services["services"].append({"StackServices" : {"service_name" : "YARN"},
"components":[]})
- properties = configurations["hawq-site"]["properties"]
- defaults = {}
- hosts = {}
-
- expected_warnings = {
- 'hawq_rm_yarn_address': {
- 'config-type': 'hawq-site',
- 'message': 'Expected value: host1:8050 (this property should have the
same value as the property yarn.resourcemanager.address in yarn-site)',
- 'type': 'configuration',
- 'config-name': 'hawq_rm_yarn_address',
- 'level': 'WARN'
- },
- 'hawq_rm_yarn_scheduler_address': {
- 'config-type': 'hawq-site',
- 'message': 'Expected value: host1:8030 (this property should have the
same value as the property yarn.resourcemanager.scheduler.address in
yarn-site)',
- 'type': 'configuration',
- 'config-name': 'hawq_rm_yarn_scheduler_address',
- 'level': 'WARN'
- }
- }
-
- serviceAdvisor = self.createHAWQServiceAdvisor()
- problems = serviceAdvisor.validateHAWQSiteConfigurations(properties,
defaults, configurations, services, hosts)
- problems_dict = {}
- for problem in problems:
- problems_dict[problem['config-name']] = problem
- self.assertEqual(len(problems), 2)
- self.assertEqual(problems_dict, expected_warnings)
-
- # Test hawq_master_directory multiple directories validation
- configurations["hawq-site"] = {"properties": {"hawq_master_directory":
"/data/hawq/master",
- "hawq_segment_directory":
"/data/hawq/segment"}}
- properties = configurations["hawq-site"]["properties"]
- problems = serviceAdvisor.validateHAWQSiteConfigurations(properties,
defaults, configurations, services, hosts)
- problems_dict = {}
- self.assertEqual(len(problems), 0)
- expected_warnings = {}
- self.assertEqual(problems_dict, expected_warnings)
-
- configurations["hawq-site"] = {"properties": {"hawq_master_directory":
"/data/hawq/master1,/data/hawq/master2",
- "hawq_segment_directory":
"/data/hawq/segment1 /data/hawq/segment2"}}
- properties = configurations["hawq-site"]["properties"]
- problems = serviceAdvisor.validateHAWQSiteConfigurations(properties,
defaults, configurations, services, hosts)
- problems_dict = {}
- for problem in problems:
- problems_dict[problem['config-name']] = problem
- self.assertEqual(len(problems), 2)
- expected_warnings = {
- 'hawq_master_directory': {
- 'config-type': 'hawq-site',
- 'message': 'Multiple directories for HAWQ Master directory are not
allowed.',
- 'type': 'configuration',
- 'config-name': 'hawq_master_directory',
- 'level': 'ERROR'
- },
- 'hawq_segment_directory': {
- 'config-type': 'hawq-site',
- 'message': 'Multiple directories for HAWQ Segment directory are not
allowed.',
- 'type': 'configuration',
- 'config-name': 'hawq_segment_directory',
- 'level': 'ERROR'
- }
- }
- self.assertEqual(problems_dict, expected_warnings)
-
- # Test hawq_global_rm_type validation
- services = {
- "services" : [
- {
- "StackServices" : {
- "service_name" : "HAWQ"
- },
- "components": []
- } ],
- "configurations":
- {
- "hawq-site": {
- "properties": {
- "hawq_global_rm_type": "yarn"
- }
- }
- }
- }
- properties = services["configurations"]["hawq-site"]["properties"]
-
- # case 1: hawq_global_rm_type is set as yarn, but YARN service is not
installed. Validation error expected.
- """
- Validation error expected is as below:
- [ {
- "config-type": "hawq-site",
- "message": "hawq_global_rm_type must be set to none
if YARN service is not installed",
- "type": "configuration",
- "config-name": "hawq_global_rm_type",
- "level": "ERROR"
- } ]
- """
- problems = serviceAdvisor.validateHAWQSiteConfigurations(properties,
defaults, services["configurations"], services, hosts)
- self.assertEqual(len(problems), 1)
- expected = {
- "config-type": "hawq-site",
- "message": "hawq_global_rm_type must be set to none if YARN service is
not installed",
- "type": "configuration",
- "config-name": "hawq_global_rm_type",
- "level": "ERROR"
- }
- self.assertEqual(problems[0], expected)
-
- # case 2: hawq_global_rm_type is set as yarn, and YARN service is
installed. No validation errors expected.
- services["services"].append({"StackServices" : {"service_name" : "YARN"},
"components":[]})
-
- problems = serviceAdvisor.validateHAWQSiteConfigurations(properties,
defaults, services["configurations"], services, hosts)
- self.assertEqual(len(problems), 0)
-
- # Test HAWQ Master port conflict with Ambari Server Postgres port
-
- # case 1: HAWQ Master is placed on Ambari Server and HAWQ Master port is
same as Ambari Server Postgres Port
- serviceAdvisor.isHawqMasterComponentOnAmbariServer =
MagicMock(return_value=True)
- configurations = {
- "hawq-site": {
- "properties":
- {"hawq_master_address_port": "5432"}
- }
- }
- problems = serviceAdvisor.validateHAWQSiteConfigurations(properties,
defaults, configurations, services, hosts)
- self.assertEqual(len(problems), 1)
- expected = {
- "config-name": "hawq_master_address_port",
- "config-type": "hawq-site",
- "level": "WARN",
- "message": "The default Postgres port (5432) on the Ambari Server
conflicts with the default HAWQ Masters port. "
- "If you are using port 5432 for Postgres, you must either
deploy the HAWQ Masters on a different host "
- "or configure a different port for the HAWQ Masters in the
HAWQ Configuration page.",
- "type": "configuration"}
- self.assertEqual(problems[0], expected)
-
- # case 2: HAWQ Master is placed on Ambari Server and HAWQ Master port is
different from Ambari Server Postgres Port
- serviceAdvisor.isHawqMasterComponentOnAmbariServer =
MagicMock(return_value=True)
- configurations["hawq-site"]["properties"]["hawq_master_address_port"] =
"10432"
- problems = serviceAdvisor.validateHAWQSiteConfigurations(properties,
defaults, configurations, services, hosts)
- self.assertEqual(len(problems), 0)
-
- # case 3: HAWQ Master is not placed on Ambari Server and HAWQ Master port
is same as Ambari Server Postgres Port
- serviceAdvisor.isHawqMasterComponentOnAmbariServer =
MagicMock(return_value=False)
- configurations["hawq-site"]["properties"]["hawq_master_address_port"] =
"5432"
- problems = serviceAdvisor.validateHAWQSiteConfigurations(properties,
defaults, configurations, services, hosts)
- self.assertEqual(len(problems), 0)
-
- # case 4: HAWQ Master is not placed on Ambari Server and HAWQ Master port
is different from Ambari Server Postgres Port
- serviceAdvisor.isHawqMasterComponentOnAmbariServer =
MagicMock(return_value=False)
- configurations["hawq-site"]["properties"]["hawq_master_address_port"] =
"10432"
- problems = serviceAdvisor.validateHAWQSiteConfigurations(properties,
defaults, configurations, services, hosts)
- self.assertEqual(len(problems), 0)
-
- # -------- test query limits warning ----------
- services = {
- "services": [
- { "StackServices": {"service_name": "HAWQ"},
- "components": [{
- "StackServiceComponents": {
- "component_name": "HAWQSEGMENT",
- "hostnames": []
- }}]
- }],
- "configurations": {}
- }
- # setup default configuration values
- configurations = services["configurations"]
- configurations["hawq-site"] = {"properties":
{"default_hash_table_bucket_number": "600",
-
"hawq_rm_nvseg_perquery_limit": "500"}}
- properties = configurations["hawq-site"]["properties"]
- defaults = {}
- hosts = {}
-
- expected = {
- 'config-type': 'hawq-site',
- 'message': 'Default buckets for Hash Distributed tables parameter value
should not be greater than the value of Virtual Segments Limit per Query
(Total) parameter, currently set to 500.',
- 'type': 'configuration',
- 'config-name': 'default_hash_table_bucket_number',
- 'level': 'ERROR'
- }
- problems = serviceAdvisor.validateHAWQSiteConfigurations(properties,
defaults, configurations, services, hosts)
- self.assertEqual(len(problems), 1)
- self.assertEqual(problems[0], expected)
-
- configurations["hawq-site"] = {"properties":
{"default_hash_table_bucket_number": "500",
-
"hawq_rm_nvseg_perquery_limit": "500"}}
- properties = configurations["hawq-site"]["properties"]
- problems = serviceAdvisor.validateHAWQSiteConfigurations(properties,
defaults, configurations, services, hosts)
- self.assertEqual(len(problems), 0)
-
-
- def test_validateHAWQHdfsClientConfigurations(self):
- services = {
- "services": [
- { "StackServices": {"service_name": "HAWQ"},
- "components": [{
- "StackServiceComponents": {
- "component_name": "HAWQSEGMENT",
- "hostnames": []
- }}]
- }],
- "configurations": {}
- }
- # setup default configuration values
- configurations = services["configurations"]
- configurations["hdfs-client"] = {"properties":
{"output.replace-datanode-on-failure": "true"}}
- properties = configurations["hdfs-client"]["properties"]
- defaults = {}
- hosts = {}
-
- # 1. Try with no hosts
- expected = {
- 'config-type': 'hdfs-client',
- 'message': 'output.replace-datanode-on-failure should be set to false
(unchecked) for clusters with 3 or less HAWQ Segments',
- 'type': 'configuration',
- 'config-name': 'output.replace-datanode-on-failure',
- 'level': 'WARN'
- }
-
- serviceAdvisor = self.createHAWQServiceAdvisor()
- problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(properties,
defaults, configurations, services, hosts)
- self.assertEqual(len(problems), 1)
- self.assertEqual(problems[0], expected)
-
- # 2. Try with 3 hosts
-
services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"]
= ["host1", "host2", "host3"]
- problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(properties,
defaults, configurations, services, hosts)
- self.assertEqual(len(problems), 1)
- self.assertEqual(problems[0], expected)
-
- # 3. Try with 4 hosts - default value
-
services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"]
= ["host1", "host2", "host3", "host4"]
- problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(properties,
defaults, configurations, services, hosts)
- self.assertEqual(len(problems), 0)
-
- # 4. Try with 4 hosts
- properties = {"output.replace-datanode-on-failure": "false"}
- expected = {
- 'config-type': 'hdfs-client',
- 'message': 'output.replace-datanode-on-failure should be set to true
(checked) for clusters with more than 3 HAWQ Segments',
- 'type': 'configuration',
- 'config-name': 'output.replace-datanode-on-failure',
- 'level': 'WARN'
- }
- problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(properties,
defaults, configurations, services, hosts)
- self.assertEqual(len(problems), 1)
- self.assertEqual(problems[0], expected)
-
def test_recommendRangerKMSConfigurations(self):
clusterData = {}
services = {
