AMBARI-16647: Move service advisor tests for HAWQ and PXF (Lav Jain via mithmatt)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fc3e7830 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fc3e7830 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fc3e7830 Branch: refs/heads/branch-2.4 Commit: fc3e7830ad22f4e2f85cba65b1cc83aa82026b31 Parents: ada4cd1 Author: Matt <[email protected]> Authored: Tue Jul 5 23:39:45 2016 -0700 Committer: Matt <[email protected]> Committed: Tue Jul 5 23:39:45 2016 -0700 ---------------------------------------------------------------------- .../HAWQ/2.0.0/service_advisor.py | 6 +- .../src/main/resources/stacks/stack_advisor.py | 25 +- .../HAWQ/test_alert_component_status.py | 141 + .../test_alert_segment_registration_status.py | 170 + .../HAWQ/test_alert_sync_status.py | 194 + .../common-services/HAWQ/test_hawqmaster.py | 425 +++ .../common-services/HAWQ/test_hawqsegment.py | 185 + .../common-services/HAWQ/test_hawqstandby.py | 241 ++ .../HAWQ/test_service_advisor.py | 1051 ++++++ .../python/common-services/HAWQ/test_utils.py | 29 + .../PXF/test_alerts_api_status.py | 77 + .../test/python/common-services/PXF/test_pxf.py | 154 + .../common-services/PXF/test_service_advisor.py | 427 +++ .../common-services/configs/hawq_default.json | 108 + .../common-services/configs/hosts-1-host.json | 93 + .../common-services/configs/hosts-3-hosts.json | 269 ++ .../common-services/configs/pxf_default.json | 87 + .../configs/services-hawq-3-hosts.json | 510 +++ .../configs/services-hawq-pxf-hdfs.json | 312 ++ .../services-master_ambari_colo-3-hosts.json | 510 +++ .../services-master_standby_colo-3-hosts.json | 510 +++ .../configs/services-nohawq-3-hosts.json | 160 + .../configs/services-normal-hawq-3-hosts.json | 521 +++ .../configs/services-normal-nohawq-3-hosts.json | 160 + .../services-standby_ambari_colo-3-hosts.json | 510 +++ .../2.3/HAWQ/test_alert_component_status.py | 141 - .../test_alert_segment_registration_status.py | 170 - .../stacks/2.3/HAWQ/test_alert_sync_status.py | 194 - .../python/stacks/2.3/HAWQ/test_hawqmaster.py | 429 --- .../python/stacks/2.3/HAWQ/test_hawqsegment.py | 178 - .../python/stacks/2.3/HAWQ/test_hawqstandby.py | 235 -- .../stacks/2.3/HAWQ/test_service_advisor.py | 318 -- .../test/python/stacks/2.3/HAWQ/test_utils.py | 29 - .../stacks/2.3/PXF/test_alerts_api_status.py | 77 - .../src/test/python/stacks/2.3/PXF/test_pxf.py | 148 - .../stacks/2.3/PXF/test_service_advisor.py | 111 - .../python/stacks/2.3/common/hosts-1-host.json | 93 - .../python/stacks/2.3/common/hosts-3-hosts.json | 269 -- .../stacks/2.3/common/services-hawq-1-host.json | 2563 ------------- .../2.3/common/services-hawq-3-hosts.json | 2563 ------------- .../2.3/common/services-hawq-pxf-hdfs.json | 3490 ------------------ .../services-master_ambari_colo-3-hosts.json | 2563 ------------- .../services-master_standby_colo-3-hosts.json | 2563 ------------- .../2.3/common/services-nohawq-3-hosts.json | 2213 ----------- .../common/services-normal-hawq-3-hosts.json | 2574 ------------- .../common/services-normal-nohawq-3-hosts.json | 2213 ----------- .../services-standby_ambari_colo-3-hosts.json | 2563 ------------- .../stacks/2.3/common/test_stack_advisor.py | 1082 +----- .../python/stacks/2.3/configs/hawq_default.json | 1327 ------- .../python/stacks/2.3/configs/pxf_default.json | 87 - 50 files changed, 6891 insertions(+), 28177 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py index e254094..16e2952 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py @@ -36,6 +36,10 @@ except Exception as e: class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): + def __init__(self, *args, **kwargs): + self.as_super = super(HAWQ200ServiceAdvisor, self) + self.as_super.__init__(*args, **kwargs) + def getHostsForMasterComponent(self, services, hosts, component, hostsList): if component["StackServiceComponents"]["component_name"] == 'HAWQSTANDBY': # Do not recommend HAWQSTANDBY on single node cluster, or cluster with no active hosts @@ -56,7 +60,7 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): return availableHosts[:1] return [ambariServerHost] - return super(HAWQ200ServiceAdvisor, self).getHostsForMasterComponent(services, hosts, component, hostsList) + return self.as_super.getHostsForMasterComponent(services, hosts, component, hostsList) def getNotPreferableOnServerComponents(self): return ['HAWQMASTER', 'HAWQSTANDBY'] http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index d8685c3..23c0320 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -307,33 +307,28 @@ class DefaultStackAdvisor(StackAdvisor): have an advisor. Stack-versions can extend this class to provide their own implement """ - services = None - # Dictionary that maps serviceName or componentName to serviceAdvisor - serviceAdvisorsDict = {} - """ - Filters the list of specified hosts object and returns - a list of hosts which are not in maintenance mode. - """ + def __init__(self): + self.services = None + # Dictionary that maps serviceName or componentName to serviceAdvisor + self.serviceAdvisorsDict = {} + + def getActiveHosts(self, hosts): + """ Filters the list of specified hosts object and returns + a list of hosts which are not in maintenance mode. """ hostsList = [] - - if (hosts is not None): + if hosts is not None: hostsList = [host['host_name'] for host in hosts if host.get('maintenance_state') is None or host.get('maintenance_state') == "OFF"] - return hostsList def getServiceAdvisor(self, key): - if len(self.serviceAdvisorsDict) == 0: self.loadServiceAdvisors() return self.serviceAdvisorsDict[key] - def loadServiceAdvisors(self, services=None): - - if self.services is None: - self.services = services + def loadServiceAdvisors(self): for service in self.services["services"]: serviceName = service["StackServices"]["service_name"] self.serviceAdvisorsDict[serviceName] = self.instantiateServiceAdvisor(service) http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/test/python/common-services/HAWQ/test_alert_component_status.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_alert_component_status.py b/ambari-server/src/test/python/common-services/HAWQ/test_alert_component_status.py new file mode 100644 index 0000000..b2e1d4d --- /dev/null +++ b/ambari-server/src/test/python/common-services/HAWQ/test_alert_component_status.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +# System imports +import os +import sys + +from mock.mock import patch + +# Local imports +from stacks.utils.RMFTestCase import * + +COMMON_SERVICES_ALERTS_DIR = "HAWQ/2.0.0/package/alerts" + +file_path = os.path.dirname(os.path.abspath(__file__)) +file_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(file_path))))) +file_path = os.path.join(file_path, "main", "resources", "common-services", COMMON_SERVICES_ALERTS_DIR) + +WORKING_CONFIGS = { + "{{hawq-site/hawq_master_address_port}}": "5432", + "{{hawq-site/hawq_segment_address_port}}": "40000", + "{{hawq-site/hawq_standby_address_host}}": "c6402.ambari.apache.org" + } + +class TestAlertComponentStatus(RMFTestCase): + + def setUp(self): + """ + Import the class under test. + Because the class is present in a different folder, append its dir to the system path. + Also, shorten the import name and make it a global so the test functions can access it. + :return: + """ + sys.path.append(file_path) + global alert_component_status + import alert_component_status + + def test_missing_configs(self): + """ + Check that the status is UNKNOWN when configs are missing. + """ + configs = None + [status, messages] = alert_component_status.execute(configurations=configs) + self.assertEqual(status, alert_component_status.RESULT_STATE_UNKNOWN) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'There were no configurations supplied to the script.') + + @patch("alert_component_status.is_component_running") + def test_hawq_master_ok(self, is_component_running_mock): + """ + Test that the status is OK when HAWQ Master is up + """ + # Mock calls + is_component_running_mock.return_value = True + + [status, messages] = alert_component_status.execute(configurations=WORKING_CONFIGS, parameters={'component_name': 'master'}) + self.assertEqual(status, alert_component_status.RESULT_STATE_OK) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'HAWQ Master is running') + + @patch("alert_component_status.is_component_running") + def test_hawq_master_critical(self, is_component_running_mock): + """ + Test that the status is CRITICIAL when HAWQ Master is down + """ + # Mock calls + is_component_running_mock.return_value = False + + [status, messages] = alert_component_status.execute(configurations=WORKING_CONFIGS, parameters={'component_name': 'master'}) + self.assertEqual(status, alert_component_status.RESULT_STATE_CRITICAL) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'HAWQ Master is not running') + + @patch("alert_component_status.is_component_running") + def test_hawq_standby_ok(self, is_component_running_mock): + """ + Test that the status is OK when HAWQ Standby is up + """ + # Mock calls + is_component_running_mock.return_value = True + + [status, messages] = alert_component_status.execute(configurations=WORKING_CONFIGS, parameters={'component_name': 'standby'}) + self.assertEqual(status, alert_component_status.RESULT_STATE_OK) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'HAWQ Standby is running') + + @patch("alert_component_status.is_component_running") + def test_hawq_standby_critical(self, is_component_running_mock): + """ + Test that the status is CRITICIAL when HAWQ Standby is down + """ + # Mock calls + is_component_running_mock.return_value = False + + [status, messages] = alert_component_status.execute(configurations=WORKING_CONFIGS, parameters={'component_name': 'standby'}) + self.assertEqual(status, alert_component_status.RESULT_STATE_CRITICAL) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'HAWQ Standby is not running') + + @patch("alert_component_status.is_component_running") + def test_hawq_segment_ok(self, is_component_running_mock): + """ + Test that the status is OK when HAWQ Segment is up + """ + # Mock calls + is_component_running_mock.return_value = True + + [status, messages] = alert_component_status.execute(configurations=WORKING_CONFIGS, parameters={'component_name': 'segment'}) + self.assertEqual(status, alert_component_status.RESULT_STATE_OK) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'HAWQ Segment is running') + + @patch("alert_component_status.is_component_running") + def test_hawq_segment_critical(self, is_component_running_mock): + """ + Test that the status is CRITICIAL when HAWQ Segment is down + """ + # Mock calls + is_component_running_mock.return_value = False + + [status, messages] = alert_component_status.execute(configurations=WORKING_CONFIGS, parameters={'component_name': 'segment'}) + self.assertEqual(status, alert_component_status.RESULT_STATE_CRITICAL) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'HAWQ Segment is not running') http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/test/python/common-services/HAWQ/test_alert_segment_registration_status.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_alert_segment_registration_status.py b/ambari-server/src/test/python/common-services/HAWQ/test_alert_segment_registration_status.py new file mode 100644 index 0000000..4066e29 --- /dev/null +++ b/ambari-server/src/test/python/common-services/HAWQ/test_alert_segment_registration_status.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +# System imports +import os +import sys +from resource_management.core.shell import call +from mock.mock import patch + +# Local imports +from stacks.utils.RMFTestCase import * + +COMMON_SERVICES_ALERTS_DIR = "HAWQ/2.0.0/package/alerts" + +file_path = os.path.dirname(os.path.abspath(__file__)) +file_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(file_path))))) +file_path = os.path.join(file_path, "main", "resources", "common-services", COMMON_SERVICES_ALERTS_DIR) + +RESULT_STATE_OK = 'OK' +RESULT_STATE_WARNING = 'WARNING' +RESULT_STATE_UNKNOWN = 'UNKNOWN' +RESULT_STATE_SKIPPED = 'SKIPPED' + +class TestAlertRegistrationStatus(RMFTestCase): + + HOST_LIST_A = ['HOST1','HOST2','HOST3','HOST4'] + HOST_LIST_B = ['HOST1','HOST3','HOST5','HOST4'] + HOST_LIST_C = ['HOST1','HOST2','HOST3'] + + def setUp(self): + """ + Import the class under test. + Because the class is present in a different folder, append its dir to the system path. + Also, shorten the import name and make it a global so the test functions can access it. + :return: + """ + sys.path.append(file_path) + global alert_segment_registration_status + import alert_segment_registration_status + + def test_missing_configs(self): + """ + Check if the status is UNKNOWN when configs are missing. + """ + configs = None + [status, messages] = alert_segment_registration_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_UNKNOWN) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'There were no configurations supplied to the script.') + + @patch("os.path.isfile", return_value=False) + def test_missing_slave_file(self, os_path_file_mock): + """ + Check if the status is SKIPPED when slaves file is missing. + """ + configs={ + "{{hawq-site/hawq_master_address_port}}": "5432" + } + [status, messages] = alert_segment_registration_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_SKIPPED) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'Slaves file is not present in /usr/local/hawq/etc') + + @patch("alert_segment_registration_status.get_segment_list_db") + @patch("alert_segment_registration_status.get_segment_list_ambari") + @patch("os.path.isfile", return_value=True) + def test_successful_registration_status(self, os_path_isfile_mock, get_segment_list_ambari_mock, get_segment_list_db_mock): + """ + Check if the status is OK if no difference in registration segment number and slaves count. + """ + get_segment_list_ambari_mock.return_value=self.HOST_LIST_A + get_segment_list_db_mock.return_value=self.HOST_LIST_A + configs={ + "{{hawq-site/hawq_master_address_port}}": "5432" + } + + [status, messages] = alert_segment_registration_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_OK) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'All HAWQ Segments are registered.') + + @patch("alert_segment_registration_status.get_segment_list_db") + @patch("alert_segment_registration_status.get_segment_list_ambari") + @patch("os.path.isfile", return_value=True) + def test_unsuccessful_registration_status_plural(self, os_path_isfile_mock, get_segment_list_ambari_mock, get_segment_list_db_mock): + """ + Check if the status is WARNING if a difference is present in registration segment number and slaves count. + """ + get_segment_list_ambari_mock.return_value=self.HOST_LIST_A + get_segment_list_db_mock.return_value=self.HOST_LIST_B + configs={ + "{{hawq-site/hawq_master_address_port}}": "5432" + } + + [status, messages] = alert_segment_registration_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_WARNING) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], '2 HAWQ Segments are not registered with HAWQ Master. Try restarting HAWQ service if a segment has been added/removed. Check the log file in /var/log/ambari-agent/ambari-alerts.log for more details on unregistered hosts.') + + @patch("alert_segment_registration_status.get_segment_list_db") + @patch("alert_segment_registration_status.get_segment_list_ambari") + @patch("os.path.isfile", return_value=True) + def test_unsuccessful_registration_status(self, os_path_isfile_mock, get_segment_list_ambari_mock, get_segment_list_db_mock): + """ + Check if the status is WARNING if a difference is present in registration segment number and slaves count. + """ + get_segment_list_ambari_mock.return_value=self.HOST_LIST_A + get_segment_list_db_mock.return_value=self.HOST_LIST_C + configs={ + "{{hawq-site/hawq_master_address_port}}": "5432" + } + + [status, messages] = alert_segment_registration_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_WARNING) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], '1 HAWQ Segment is not registered with HAWQ Master. Try restarting HAWQ service if a segment has been added/removed. Check the log file in /var/log/ambari-agent/ambari-alerts.log for more details on unregistered hosts.') + + @patch("alert_segment_registration_status.get_segment_list_db") + @patch("alert_segment_registration_status.get_segment_list_ambari") + @patch("os.path.isfile", return_value=True) + def test_exception_registration_status(self, os_path_isfile_mock, get_segment_list_ambari_mock, get_segment_list_db_mock): + """ + Check if the status is UNKNOWN if an exception is thrown when finding registration segment number and slaves count. + """ + get_segment_list_ambari_mock.return_value=self.HOST_LIST_A + get_segment_list_db_mock.side_effect=Exception("Exception raised to fail") + configs={ + "{{hawq-site/hawq_master_address_port}}": "5432" + } + + [status, messages] = alert_segment_registration_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_UNKNOWN) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'HAWQ Segments Registration Status cannot be determined.') + + @patch("alert_segment_registration_status.get_segment_list_db") + @patch("alert_segment_registration_status.get_segment_list_ambari") + @patch("os.path.isfile", return_value=True) + def test_unsuccessful_empty_db_registration_status(self, os_path_isfile_mock, get_segment_list_ambari_mock, get_segment_list_db_mock): + """ + Check if the status is WARNING if a difference is present in registration segment number and slaves count. + """ + get_segment_list_ambari_mock.return_value=[] + get_segment_list_db_mock.return_value=self.HOST_LIST_C + configs={ + "{{hawq-site/hawq_master_address_port}}": "5432" + } + + [status, messages] = alert_segment_registration_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_WARNING) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], '3 HAWQ Segments are not registered with HAWQ Master. Try restarting HAWQ service if a segment has been added/removed. Check the log file in /var/log/ambari-agent/ambari-alerts.log for more details on unregistered hosts.') + http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/test/python/common-services/HAWQ/test_alert_sync_status.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_alert_sync_status.py b/ambari-server/src/test/python/common-services/HAWQ/test_alert_sync_status.py new file mode 100644 index 0000000..a44cad1 --- /dev/null +++ b/ambari-server/src/test/python/common-services/HAWQ/test_alert_sync_status.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +# System imports +import os +import sys + +from mock.mock import patch + +# Local imports +from stacks.utils.RMFTestCase import * + +COMMON_SERVICES_ALERTS_DIR = "HAWQ/2.0.0/package/alerts" + +file_path = os.path.dirname(os.path.abspath(__file__)) +file_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(file_path))))) +file_path = os.path.join(file_path, "main", "resources", "common-services", COMMON_SERVICES_ALERTS_DIR) + +RESULT_STATE_OK = 'OK' +RESULT_STATE_WARNING = 'WARNING' +RESULT_STATE_UNKNOWN = 'UNKNOWN' +RESULT_STATE_SKIPPED = 'SKIPPED' + +class TestAlertSyncStatus(RMFTestCase): + + def setUp(self): + """ + Import the class under test. + Because the class is present in a different folder, append its dir to the system path. + Also, shorten the import name and make it a global so the test functions can access it. + :return: + """ + sys.path.append(file_path) + global alert_sync_status + import alert_sync_status + + def test_missing_configs(self): + """ + Check that the status is UNKNOWN when configs are missing. + """ + configs = None + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_UNKNOWN) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'There were no configurations supplied to the script.') + + + @patch("alert_sync_status.get_sync_status") + def test_no_standby_state(self, get_sync_status_mock): + """ + Test that the status is SKIPPED when HAWQSTANDBY is not in configurations + """ + configs = { + "{{hawq-site/hawq_master_address_port}}": "5432" + } + + # Mock calls + get_sync_status_mock.return_value = 'Not Configured' + + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_SKIPPED) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'HAWQSTANDBY is not installed.') + + + @patch("alert_sync_status.get_sync_status") + def test_synchronized_state(self, get_sync_status_mock): + """ + Test that the status is OK when HAWQSTANDBY is 'Synchronized' with HAWQMASTER + """ + configs = { + "{{hawq-site/hawq_master_address_port}}": "5432", + "{{hawq-site/hawq_standby_address_host}}": "c6402.ambari.apache.org" + } + + # Mock calls + get_sync_status_mock.return_value = ('Synchronized', "") + + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_OK) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'HAWQSTANDBY is in sync with HAWQMASTER.') + + + @patch("alert_sync_status.get_sync_status") + def test_synchronizing_state(self, get_sync_status_mock): + """ + Test that the status is OK when HAWQSTANDBY is 'Synchronizing' with HAWQMASTER + """ + configs = { + "{{hawq-site/hawq_master_address_port}}": "5432", + "{{hawq-site/hawq_standby_address_host}}": "c6402.ambari.apache.org" + } + + # Mock calls + get_sync_status_mock.return_value = ('Synchronizing', "") + + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_OK) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'HAWQSTANDBY is in sync with HAWQMASTER.') + + + @patch("alert_sync_status.get_sync_status") + def test_not_synchronized_state(self, get_sync_status_mock): + """ + Test that the status is WARNING when HAWQSTANDBY is 'Noe Synchronized' with HAWQMASTER + """ + configs = { + "{{hawq-site/hawq_master_address_port}}": "5432", + "{{hawq-site/hawq_standby_address_host}}": "c6402.ambari.apache.org" + } + + # Mock calls + get_sync_status_mock.return_value = ('Not Synchronized', "ERROR_MESSAGE") + + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_WARNING) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'HAWQSTANDBY is not in sync with HAWQMASTER. ERROR: ERROR_MESSAGE') + + + @patch("alert_sync_status.get_sync_status") + def test_none_state(self, get_sync_status_mock): + """ + Test that the status is UNKNOWN when HAWQMASTER returns summary_state as 'None' + """ + configs = { + "{{hawq-site/hawq_master_address_port}}": "5432", + "{{hawq-site/hawq_standby_address_host}}": "c6402.ambari.apache.org" + } + + # Mock calls + get_sync_status_mock.return_value = 'None' + + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_UNKNOWN) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'Sync status cannot be determined.') + + + @patch("alert_sync_status.get_sync_status") + def test_not_configured_state(self, get_sync_status_mock): + """ + Test that the status is UNKNOWN when HAWQMASTER returns summary_state as 'Not Configured' + """ + configs = { + "{{hawq-site/hawq_master_address_port}}": "5432", + "{{hawq-site/hawq_standby_address_host}}": "c6402.ambari.apache.org" + } + + # Mock calls + get_sync_status_mock.return_value = 'Not Configured' + + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_UNKNOWN) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'Sync status cannot be determined.') + + + @patch("alert_sync_status.get_sync_status") + def test_unknown_state(self, get_sync_status_mock): + """ + Test that the status is UNKNOWN when HAWQMASTER returns summary_state as 'Unknown' + """ + configs = { + "{{hawq-site/hawq_master_address_port}}": "5432", + "{{hawq-site/hawq_standby_address_host}}": "c6402.ambari.apache.org" + } + + # Mock calls + get_sync_status_mock.return_value = 'Unknown' + + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_UNKNOWN) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'Sync status cannot be determined.') http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/test/python/common-services/HAWQ/test_hawqmaster.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_hawqmaster.py b/ambari-server/src/test/python/common-services/HAWQ/test_hawqmaster.py new file mode 100644 index 0000000..62d54cf --- /dev/null +++ b/ambari-server/src/test/python/common-services/HAWQ/test_hawqmaster.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import os, json, copy, crypt +from mock.mock import patch +from stacks.utils.RMFTestCase import RMFTestCase, InlineTemplate, UnknownConfigurationMock + + +class TestHawqMaster(RMFTestCase): + + COMMON_SERVICES_PACKAGE_DIR = 'HAWQ/2.0.0/package' + STACK_VERSION = '2.3' + GPADMIN = 'gpadmin' + POSTGRES = 'postgres' + DEFAULT_IMMUTABLE_PATHS = ['/apps/hive/warehouse', '/apps/falcon', '/mr-history/done', '/app-logs', '/tmp'] + CONFIG_FILE = os.path.join(os.path.dirname(__file__), '../configs/hawq_default.json') + HAWQ_CHECK_COMMAND = 'source /usr/local/hawq/greenplum_path.sh && export PGHOST="c6403.ambari.apache.org" && hawq check -f /usr/local/hawq/etc/hawq_hosts --hadoop /usr/phd/current/hadoop-client --config /usr/local/hawq/etc/hawq_check.cnf ' + with open(CONFIG_FILE, "r") as f: + hawq_config = json.load(f) + + def setUp(self): + self.config_dict = copy.deepcopy(self.hawq_config) + + def __asserts_for_configure(self): + + self.assertResourceCalled('Group', self.GPADMIN, + ignore_failures = True + ) + + self.assertResourceCalled('User', self.GPADMIN, + gid = self.GPADMIN, + groups = [self.GPADMIN, u'hadoop'], + ignore_failures = True, + password = crypt.crypt(self.config_dict['configurations']['hawq-env']['hawq_password'], "$1$salt$") + ) + + self.assertResourceCalled('Group', self.POSTGRES, + ignore_failures = True + ) + + self.assertResourceCalled('User', self.POSTGRES, + gid = self.POSTGRES, + groups = [self.POSTGRES, u'hadoop'], + ignore_failures = True + ) + + self.assertResourceCalled('Execute', 'chown -R gpadmin:gpadmin /usr/local/hawq/', + timeout = 600 + ) + + self.assertResourceCalled('XmlConfig', 'hdfs-client.xml', + conf_dir = '/usr/local/hawq/etc/', + configurations = self.getConfig()['configurations']['hdfs-client'], + configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-client'], + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('XmlConfig', 'yarn-client.xml', + conf_dir = '/usr/local/hawq/etc/', + configurations = self.getConfig()['configurations']['yarn-client'], + configuration_attributes = self.getConfig()['configuration_attributes']['yarn-client'], + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('XmlConfig', 'hawq-site.xml', + conf_dir = '/usr/local/hawq/etc/', + configurations = self.getConfig()['configurations']['hawq-site'], + configuration_attributes = self.getConfig()['configuration_attributes']['hawq-site'], + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('File', '/usr/local/hawq/etc/hawq_check.cnf', + content = self.getConfig()['configurations']['hawq-check-env']['content'], + owner = self.GPADMIN, + group = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('File', '/usr/local/hawq/etc/slaves', + content = InlineTemplate('c6401.ambari.apache.org\nc6402.ambari.apache.org\nc6403.ambari.apache.org\n\n'), + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('Directory', '/data/hawq/master', + group = self.GPADMIN, + owner = self.GPADMIN, + create_parents = True + ) + + self.assertResourceCalled('Execute', 'chmod 700 /data/hawq/master', + user = 'root', + timeout = 600 + ) + + self.assertResourceCalled('Directory', '/data/hawq/tmp/master', + group = self.GPADMIN, + owner = self.GPADMIN, + create_parents = True + ) + + + @patch ('hawqmaster.common.__set_osparams') + def test_configure_default(self, set_osparams_mock): + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py', + classname = 'HawqMaster', + command = 'configure', + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.__asserts_for_configure() + self.assertNoMoreResources() + + + @patch ('hawqmaster.common.__set_osparams') + def test_install_default(self, set_osparams_mock): + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py', + classname = 'HawqMaster', + command = 'install', + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.__asserts_for_configure() + self.assertNoMoreResources() + + + @patch ('hawqmaster.common.__set_osparams') + def test_start_default(self, set_osparams_mock): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py', + classname = 'HawqMaster', + command = 'start', + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.__asserts_for_configure() + + self.assertResourceCalled('HdfsResource', '/hawq_data', + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + default_fs = u'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + type = 'directory', + action = ['create_on_execute'], + owner = self.GPADMIN, + group = self.GPADMIN, + user = u'hdfs', + mode = 493, + security_enabled = False, + kinit_path_local = '/usr/bin/kinit', + recursive_chown = True, + keytab = UnknownConfigurationMock(), + principal_name = UnknownConfigurationMock() + ) + + self.assertResourceCalled('HdfsResource', None, + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + default_fs = u'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + action = ['execute'], + user = u'hdfs', + security_enabled = False, + kinit_path_local = '/usr/bin/kinit', + keytab = UnknownConfigurationMock(), + principal_name = UnknownConfigurationMock() + ) + + self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && hawq init master -a -v --ignore-bad-hosts', + logoutput = True, + not_if = None, + only_if = None, + user = self.GPADMIN, + timeout = 900 + ) + + self.assertNoMoreResources() + + + def __asserts_for_stop(self, componentCommand, expectedCommand): + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py', + classname = 'HawqMaster', + command = componentCommand, + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.assertResourceCalled('Execute', expectedCommand, + logoutput = True, + not_if = None, + only_if = "netstat -tupln | egrep ':5432\\s' | egrep postgres", + user = self.GPADMIN, + timeout = 900 + ) + + self.assertNoMoreResources() + + + @patch ('hawqmaster.common.__set_osparams') + @patch ('common.get_local_hawq_site_property_value') + def test_stop_default(self, get_local_hawq_site_property_value_mock, set_osparams_mock): + """ Run Stop HAWQMASTER """ + + get_local_hawq_site_property_value_mock.return_value = 5432 + self.__asserts_for_stop('stop', 'source /usr/local/hawq/greenplum_path.sh && hawq stop master -M fast -a -v') + + + @patch ('hawqmaster.common.__set_osparams') + @patch ('common.get_local_hawq_site_property_value') + def test_stop_cluster_immediate(self, get_local_hawq_site_property_value_mock, set_osparams_mock): + """ Run Stop HAWQ Cluster Immediate Mode """ + + get_local_hawq_site_property_value_mock.return_value = 5432 + self.__asserts_for_stop('immediate_stop_hawq_service','source /usr/local/hawq/greenplum_path.sh && hawq stop cluster -M immediate -a -v') + + + def __asserts_for_hawq_check(self, expectedCommand): + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py', + classname = 'HawqMaster', + command = 'run_hawq_check', + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.assertResourceCalled('File', "/usr/local/hawq/etc/hawq_hosts", + content = InlineTemplate("{% for host in hawq_all_hosts %}{{host}}\n{% endfor %}"), + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('Execute', expectedCommand, + logoutput = True, + not_if = None, + only_if = None, + user=self.GPADMIN, + timeout=900 + ) + + self.assertNoMoreResources() + + + def test_run_hawq_check_case1(self): + """ Running HAWQ Check Case 1: Non HDFS-HA, Standalone Resource Management, Not Kerberized """ + + expectedCommand = self.HAWQ_CHECK_COMMAND + self.__asserts_for_hawq_check(expectedCommand) + + + def test_run_hawq_check_case2(self): + """ Running HAWQ Check Case 2: Non HDFS-HA, Standalone Resource Management, Kerberized """ + + self.config_dict['configurations']['cluster-env']['security_enabled'] = "true" + expectedCommand = "{0}--kerberos".format(self.HAWQ_CHECK_COMMAND) + self.__asserts_for_hawq_check(expectedCommand) + + + def test_run_hawq_check_case3(self): + """ Running HAWQ Check Case 3: Non HDFS-HA, YARN Resource Management Non YARN_HA, Not Kerberized """ + + self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" + expectedCommand = "{0}--yarn".format(self.HAWQ_CHECK_COMMAND) + self.__asserts_for_hawq_check(expectedCommand) + + + def test_run_hawq_check_case4(self): + """ Running HAWQ Check Case 4: Non HDFS-HA, YARN Resource Management Non YARN_HA, Kerberized """ + + self.config_dict['configurations']['cluster-env']['security_enabled'] = "true" + self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" + expectedCommand = "{0}--yarn --kerberos".format(self.HAWQ_CHECK_COMMAND) + self.__asserts_for_hawq_check(expectedCommand) + + + def test_run_hawq_check_case5(self): + """ Running HAWQ Check Case 5: Non HDFS-HA, YARN Resource Management YARN_HA, Not Kerberized """ + + self.config_dict['configurations']['yarn-site']['yarn.resourcemanager.ha.enabled'] = "true" + self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" + expectedCommand = "{0}--yarn-ha".format(self.HAWQ_CHECK_COMMAND) + self.__asserts_for_hawq_check(expectedCommand) + + + def test_run_hawq_check_case6(self): + """ Running HAWQ Check Case 6: Non HDFS-HA, YARN Resource Management YARN_HA, Kerberized """ + + self.config_dict['configurations']['cluster-env']['security_enabled'] = "true" + self.config_dict['configurations']['yarn-site']['yarn.resourcemanager.ha.enabled'] = "true" + self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" + expectedCommand = "{0}--yarn-ha --kerberos".format(self.HAWQ_CHECK_COMMAND) + self.__asserts_for_hawq_check(expectedCommand) + + + def test_run_hawq_check_case7(self): + """ Running HAWQ Check Case 7: HDFS-HA, Standalone Resource Management, Not Kerberized """ + + self.config_dict['configurations']['hdfs-site']['dfs.nameservices'] = "haservice" + expectedCommand = "{0}--hdfs-ha".format(self.HAWQ_CHECK_COMMAND) + self.__asserts_for_hawq_check(expectedCommand) + + + def test_run_hawq_check_case8(self): + """ Running HAWQ Check Case 8: HDFS-HA, Standalone Resource Management, Kerberized """ + + self.config_dict['configurations']['cluster-env']['security_enabled'] = "true" + self.config_dict['configurations']['hdfs-site']['dfs.nameservices'] = "haservice" + expectedCommand = "{0}--hdfs-ha --kerberos".format(self.HAWQ_CHECK_COMMAND) + self.__asserts_for_hawq_check(expectedCommand) + + + def test_run_hawq_check_case9(self): + """ Running HAWQ Check Case 9: HDFS-HA, YARN Resource Management Non YARN_HA, Not Kerberized """ + + self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" + self.config_dict['configurations']['hdfs-site']['dfs.nameservices'] = "haservice" + expectedCommand = "{0}--hdfs-ha --yarn".format(self.HAWQ_CHECK_COMMAND) + self.__asserts_for_hawq_check(expectedCommand) + + + def test_run_hawq_check_case10(self): + """ Running HAWQ Check Case 10: HDFS-HA, YARN Resource Management Non YARN_HA, Kerberized """ + + self.config_dict['configurations']['cluster-env']['security_enabled'] = "true" + self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" + self.config_dict['configurations']['hdfs-site']['dfs.nameservices'] = "haservice" + expectedCommand = "{0}--hdfs-ha --yarn --kerberos".format(self.HAWQ_CHECK_COMMAND) + self.__asserts_for_hawq_check(expectedCommand) + + + def test_run_hawq_check_case11(self): + """ Running HAWQ Check Case 11: HDFS-HA, YARN Resource Management YARN_HA, Not Kerberized """ + + self.config_dict['configurations']['yarn-site']['yarn.resourcemanager.ha.enabled'] = "true" + self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" + self.config_dict['configurations']['hdfs-site']['dfs.nameservices'] = "haservice" + expectedCommand = "{0}--hdfs-ha --yarn-ha".format(self.HAWQ_CHECK_COMMAND) + self.__asserts_for_hawq_check(expectedCommand) + + + def test_run_hawq_check_case12(self): + """ Running HAWQ Check Case 12: HDFS-HA, YARN Resource Management YARN_HA, Kerberized """ + + self.config_dict['configurations']['cluster-env']['security_enabled'] = "true" + self.config_dict['configurations']['yarn-site']['yarn.resourcemanager.ha.enabled'] = "true" + self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" + self.config_dict['configurations']['hdfs-site']['dfs.nameservices'] = "haservice" + expectedCommand = "{0}--hdfs-ha --yarn-ha --kerberos".format(self.HAWQ_CHECK_COMMAND) + self.__asserts_for_hawq_check(expectedCommand) + + + def test_resync_hawq_standby(self): + """ Run custom command Resync HAWQ Standby """ + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py', + classname = 'HawqMaster', + command = 'resync_hawq_standby', + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && export PGHOST="c6403.ambari.apache.org" && hawq init standby -n -a -v -M fast', + user = self.GPADMIN, + timeout = 900, + not_if = None, + only_if = None, + logoutput = True + ) + + self.assertNoMoreResources() + + + def test_remove_hawq_standby(self): + """ Run custom command Remove HAWQ Standby """ + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py', + classname = 'HawqMaster', + command = 'remove_hawq_standby', + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && export PGHOST="c6403.ambari.apache.org" && hawq init standby -a -v -r --ignore-bad-hosts', + user = self.GPADMIN, + timeout = 900, + not_if = None, + only_if = None, + logoutput = True + ) + + self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/test/python/common-services/HAWQ/test_hawqsegment.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_hawqsegment.py b/ambari-server/src/test/python/common-services/HAWQ/test_hawqsegment.py new file mode 100644 index 0000000..48203ca --- /dev/null +++ b/ambari-server/src/test/python/common-services/HAWQ/test_hawqsegment.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import os, json, copy, crypt +from mock.mock import patch +from stacks.utils.RMFTestCase import RMFTestCase + + +class TestHawqSegment(RMFTestCase): + + COMMON_SERVICES_PACKAGE_DIR = 'HAWQ/2.0.0/package' + STACK_VERSION = '2.3' + GPADMIN = 'gpadmin' + POSTGRES = 'postgres' + CONFIG_FILE = os.path.join(os.path.dirname(__file__), '../configs/hawq_default.json') + with open(CONFIG_FILE, "r") as f: + hawq_config = json.load(f) + + def setUp(self): + self.config_dict = copy.deepcopy(self.hawq_config) + + def __asserts_for_configure(self): + + self.assertResourceCalled('Group', self.GPADMIN, + ignore_failures = True + ) + + self.assertResourceCalled('User', self.GPADMIN, + gid = self.GPADMIN, + groups = [self.GPADMIN, u'hadoop'], + ignore_failures = True, + password = crypt.crypt(self.getConfig()['configurations']['hawq-env']['hawq_password'], "$1$salt$") + ) + + self.assertResourceCalled('Group', self.POSTGRES, + ignore_failures = True + ) + + self.assertResourceCalled('User', self.POSTGRES, + gid = self.POSTGRES, + groups = [self.POSTGRES, u'hadoop'], + ignore_failures = True + ) + + self.assertResourceCalled('Execute', 'chown -R gpadmin:gpadmin /usr/local/hawq/', + timeout = 600 + ) + + self.assertResourceCalled('XmlConfig', 'hdfs-client.xml', + conf_dir = '/usr/local/hawq/etc/', + configurations = self.getConfig()['configurations']['hdfs-client'], + configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-client'], + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('XmlConfig', 'yarn-client.xml', + conf_dir = '/usr/local/hawq/etc/', + configurations = self.getConfig()['configurations']['yarn-client'], + configuration_attributes = self.getConfig()['configuration_attributes']['yarn-client'], + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('XmlConfig', 'hawq-site.xml', + conf_dir = '/usr/local/hawq/etc/', + configurations = self.getConfig()['configurations']['hawq-site'], + configuration_attributes = self.getConfig()['configuration_attributes']['hawq-site'], + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('Directory', '/data/hawq/segment', + owner = self.GPADMIN, + group = self.GPADMIN, + create_parents = True + ) + + self.assertResourceCalled('Execute', 'chmod 700 /data/hawq/segment', + user = 'root', + timeout = 600 + ) + + self.assertResourceCalled('Directory', '/data/hawq/tmp/segment', + owner = self.GPADMIN, + group = self.GPADMIN, + create_parents = True + ) + + + @patch ('hawqsegment.common.__set_osparams') + def test_configure_default(self, set_osparams_mock): + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqsegment.py', + classname = 'HawqSegment', + command = 'configure', + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.__asserts_for_configure() + self.assertNoMoreResources() + + + @patch ('hawqsegment.common.__set_osparams') + def test_install_default(self, set_osparams_mock): + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqsegment.py', + classname = 'HawqSegment', + command = 'install', + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.__asserts_for_configure() + self.assertNoMoreResources() + + + @patch ('hawqsegment.common.__set_osparams') + def test_start_default(self, set_osparams_mock): + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqsegment.py', + classname = 'HawqSegment', + command = 'start', + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.__asserts_for_configure() + + self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && hawq init segment -a -v', + logoutput = True, + not_if = None, + only_if = None, + user = self.GPADMIN, + timeout = 900 + ) + + self.assertNoMoreResources() + + + @patch ('common.get_local_hawq_site_property_value') + def test_stop_default(self, get_local_hawq_site_property_value_mock): + get_local_hawq_site_property_value_mock.return_value = 40000 + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqsegment.py', + classname = 'HawqSegment', + command = 'stop', + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && hawq stop segment -M fast -a -v', + logoutput = True, + not_if = None, + only_if = "netstat -tupln | egrep ':40000\\s' | egrep postgres", + user = self.GPADMIN, + timeout = 900 + ) + + self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/test/python/common-services/HAWQ/test_hawqstandby.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_hawqstandby.py b/ambari-server/src/test/python/common-services/HAWQ/test_hawqstandby.py new file mode 100644 index 0000000..8674549 --- /dev/null +++ b/ambari-server/src/test/python/common-services/HAWQ/test_hawqstandby.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import os, json, copy, crypt +from mock.mock import patch +from stacks.utils.RMFTestCase import RMFTestCase, InlineTemplate + + +class TestHawqStandby(RMFTestCase): + + COMMON_SERVICES_PACKAGE_DIR = 'HAWQ/2.0.0/package' + STACK_VERSION = '2.3' + GPADMIN = 'gpadmin' + POSTGRES = 'postgres' + CONFIG_FILE = os.path.join(os.path.dirname(__file__), '../configs/hawq_default.json') + with open(CONFIG_FILE, "r") as f: + hawq_config = json.load(f) + + def setUp(self): + self.config_dict = copy.deepcopy(self.hawq_config) + + def __asserts_for_configure(self): + + self.assertResourceCalled('Group', self.GPADMIN, + ignore_failures = True + ) + + self.assertResourceCalled('User', self.GPADMIN, + gid = self.GPADMIN, + groups = [self.GPADMIN, u'hadoop'], + ignore_failures = True, + password = crypt.crypt(self.getConfig()['configurations']['hawq-env']['hawq_password'], "$1$salt$") + ) + + self.assertResourceCalled('Group', self.POSTGRES, + ignore_failures = True + ) + + self.assertResourceCalled('User', self.POSTGRES, + gid = self.POSTGRES, + groups = [self.POSTGRES, u'hadoop'], + ignore_failures = True + ) + + self.assertResourceCalled('Execute', 'chown -R gpadmin:gpadmin /usr/local/hawq/', + timeout = 600 + ) + + self.assertResourceCalled('XmlConfig', 'hdfs-client.xml', + conf_dir = '/usr/local/hawq/etc/', + configurations = self.getConfig()['configurations']['hdfs-client'], + configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-client'], + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('XmlConfig', 'yarn-client.xml', + conf_dir = '/usr/local/hawq/etc/', + configurations = self.getConfig()['configurations']['yarn-client'], + configuration_attributes = self.getConfig()['configuration_attributes']['yarn-client'], + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('XmlConfig', 'hawq-site.xml', + conf_dir = '/usr/local/hawq/etc/', + configurations = self.getConfig()['configurations']['hawq-site'], + configuration_attributes = self.getConfig()['configuration_attributes']['hawq-site'], + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('File', '/usr/local/hawq/etc/hawq_check.cnf', + content = self.getConfig()['configurations']['hawq-check-env']['content'], + owner = self.GPADMIN, + group = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('File', '/usr/local/hawq/etc/slaves', + content = InlineTemplate('c6401.ambari.apache.org\nc6402.ambari.apache.org\nc6403.ambari.apache.org\n\n'), + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('Directory', '/data/hawq/master', + group = self.GPADMIN, + owner = self.GPADMIN, + create_parents = True + ) + + self.assertResourceCalled('Execute', 'chmod 700 /data/hawq/master', + user = 'root', + timeout = 600 + ) + + self.assertResourceCalled('Directory', '/data/hawq/tmp/master', + group = self.GPADMIN, + owner = self.GPADMIN, + create_parents = True + ) + + + @patch ('hawqstandby.common.__set_osparams') + def test_configure_default(self, set_osparams_mock): + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqstandby.py', + classname = 'HawqStandby', + command = 'configure', + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.__asserts_for_configure() + self.assertNoMoreResources() + + + @patch ('hawqstandby.common.__set_osparams') + def test_install_default(self, set_osparams_mock): + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqstandby.py', + classname = 'HawqStandby', + command = 'install', + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.__asserts_for_configure() + self.assertNoMoreResources() + + + @patch ('hawqstandby.common.__set_osparams') + def test_start_default(self, set_osparams_mock): + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqstandby.py', + classname = 'HawqStandby', + command = 'start', + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.__asserts_for_configure() + + self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && hawq init standby -a -v', + logoutput = True, + not_if = None, + only_if = None, + user = self.GPADMIN, + timeout = 900 + ) + self.assertNoMoreResources() + + + @patch ('hawqstandby.common.__set_osparams') + @patch ('common.get_local_hawq_site_property_value') + def test_stop_default(self, get_local_hawq_site_property_value_mock, set_osparams_mock): + get_local_hawq_site_property_value_mock.return_value = 5432 + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqstandby.py', + classname = 'HawqStandby', + command = 'stop', + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && hawq stop standby -M fast -a -v', + logoutput = True, + not_if = None, + only_if = "netstat -tupln | egrep ':5432\\s' | egrep gpsyncmaster", + user = self.GPADMIN, + timeout = 900 + ) + + self.assertNoMoreResources() + + + @patch ('common.get_local_hawq_site_property_value') + def test_activate_hawq_standby(self, get_local_hawq_site_property_value_mock): + """Test Activate HAWQ Standby Command""" + + get_local_hawq_site_property_value_mock.return_value = 5432 + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqstandby.py', + classname = 'HawqStandby', + command = 'activate_hawq_standby', + config_dict = self.config_dict, + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.assertResourceCalled('XmlConfig', 'hawq-site.xml', + conf_dir = '/usr/local/hawq/etc/', + configurations = self.getConfig()['configurations']['hawq-site'], + configuration_attributes = self.getConfig()['configuration_attributes']['hawq-site'], + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && export PGHOST=\"c6402.ambari.apache.org\" && hawq activate standby -a -M fast -v --ignore-bad-hosts', + logoutput = True, + not_if = None, + only_if = None, + user = self.GPADMIN, + timeout = 900 + ) + + self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && hawq stop master -M fast -a -v', + logoutput = True, + not_if = None, + only_if = "netstat -tupln | egrep ':5432\\s' | egrep postgres", + user = self.GPADMIN, + timeout = 900 + ) + + self.assertNoMoreResources() \ No newline at end of file
