Repository: ambari Updated Branches: refs/heads/branch-2.4 4abaf2a1b -> 85c97e3a7
AMBARI-18110: Ambari unit tests for HAWQ are not being called (lavjain) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/85c97e3a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/85c97e3a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/85c97e3a Branch: refs/heads/branch-2.4 Commit: 85c97e3a7f3db4d85fc01a82e7286c4c45fdb99a Parents: 4abaf2a Author: ljainpivotalio <[email protected]> Authored: Fri Aug 12 14:34:47 2016 -0700 Committer: ljainpivotalio <[email protected]> Committed: Fri Aug 12 14:34:47 2016 -0700 ---------------------------------------------------------------------- .../HAWQ/2.0.0/package/scripts/common.py | 18 +- .../common-services/HAWQ/hawq_base_test_case.py | 129 +++++++++++++ .../HAWQ/test_alert_component_status.py | 9 +- .../test_alert_segment_registration_status.py | 9 +- .../HAWQ/test_alert_sync_status.py | 9 +- .../common-services/HAWQ/test_hawqmaster.py | 193 +++++-------------- .../common-services/HAWQ/test_hawqsegment.py | 120 ++---------- .../common-services/HAWQ/test_hawqstandby.py | 146 +++----------- .../HAWQ/test_service_advisor.py | 10 +- .../python/common-services/HAWQ/test_utils.py | 3 +- .../PXF/test_alerts_api_status.py | 6 +- .../test/python/common-services/PXF/test_pxf.py | 17 +- .../common-services/PXF/test_service_advisor.py | 22 ++- ambari-server/src/test/python/unitTests.py | 26 ++- 14 files changed, 303 insertions(+), 414 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/85c97e3a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py index 529c492..3ac7260 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py @@ -283,14 +283,9 @@ def start_component(component_name, port, data_dir): __check_dfs_truncate_enforced() if component_name == hawq_constants.MASTER: # Check the owner for hawq_data directory - kinit_cmd = "{0} -kt {1} {2};".format(params.kinit_path_local, params.hdfs_user_keytab, params.hdfs_principal_name) if params.security_enabled else "" data_dir_owner = hawq_constants.hawq_user_secured if params.security_enabled else hawq_constants.hawq_user - cmd = kinit_cmd + "hdfs dfs -ls {0} | sed '1d;s/ */ /g' | cut -d\\ -f3".format(params.hawq_hdfs_data_dir) - returncode, stdout = call(cmd, user=params.hdfs_superuser, timeout=300) - if returncode: - raise # Change owner recursively (if needed) - if stdout.strip() != data_dir_owner: + if __get_hdfs_dir_owner() != data_dir_owner: params.HdfsResource(params.hawq_hdfs_data_dir, type="directory", action="create_on_execute", @@ -324,6 +319,17 @@ def stop_component(component_name, mode): "{0} -M {1} -a -v".format(component_name, mode), only_if=utils.generate_hawq_process_status_cmd(component_name, port_number)) +def __get_hdfs_dir_owner(): + import params + + # Check the owner for hawq_data directory + kinit_cmd = "{0} -kt {1} {2};".format(params.kinit_path_local, params.hdfs_user_keytab, params.hdfs_principal_name) if params.security_enabled else "" + cmd = kinit_cmd + "hdfs dfs -ls {0} | sed '1d;s/ */ /g' | cut -d\\ -f3".format(params.hawq_hdfs_data_dir) + returncode, stdout = call(cmd, user=params.hdfs_superuser, timeout=300) + if returncode: + raise Fail("Unable to determine the ownership for HDFS dir {0}".format(params.hawq_hdfs_data_dir)) + return stdout.strip() + def __check_dfs_truncate_enforced(): """ If enforce_hdfs_truncate is set to True: http://git-wip-us.apache.org/repos/asf/ambari/blob/85c97e3a/ambari-server/src/test/python/common-services/HAWQ/hawq_base_test_case.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/hawq_base_test_case.py b/ambari-server/src/test/python/common-services/HAWQ/hawq_base_test_case.py new file mode 100644 index 0000000..892dca3 --- /dev/null +++ b/ambari-server/src/test/python/common-services/HAWQ/hawq_base_test_case.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import copy +import crypt +import json +import os + +from stacks.utils.RMFTestCase import RMFTestCase, InlineTemplate + + +class HawqBaseTestCase(RMFTestCase): + + HAWQ_PACKAGE_DIR = 'HAWQ/2.0.0/package' + TARGET_COMMON_SERVICES = RMFTestCase.TARGET_COMMON_SERVICES + SOURCE_HAWQ_SCRIPT = 'source /usr/local/hawq/greenplum_path.sh && ' + STACK_VERSION = '2.3' + GPADMIN = 'gpadmin' + POSTGRES = 'postgres' + CONF_DIR = '/usr/local/hawq/etc/' + CONFIG_FILE = os.path.join(os.path.dirname(__file__), '../configs/hawq_default.json') + with open(CONFIG_FILE, "r") as f: + hawq_config = json.load(f) + + def setUp(self): + self.config_dict = copy.deepcopy(self.hawq_config) + + def asserts_for_configure(self): + + self.assertResourceCalled('Group', self.GPADMIN, + ignore_failures = True + ) + + self.assertResourceCalled('User', self.GPADMIN, + gid = self.GPADMIN, + groups = [self.GPADMIN, u'hadoop'], + ignore_failures = True, + password = crypt.crypt(self.config_dict['configurations']['hawq-env']['hawq_password'], "$1$salt$") + ) + + self.assertResourceCalled('Group', self.POSTGRES, + ignore_failures = True + ) + + self.assertResourceCalled('User', self.POSTGRES, + gid = self.POSTGRES, + groups = [self.POSTGRES, u'hadoop'], + ignore_failures = True + ) + + self.assertResourceCalled('Execute', 'chown -R gpadmin:gpadmin /usr/local/hawq/', + timeout = 600 + ) + + self.assertResourceCalled('XmlConfig', 'hdfs-client.xml', + conf_dir = self.CONF_DIR, + configurations = self.getConfig()['configurations']['hdfs-client'], + configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-client'], + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('XmlConfig', 'yarn-client.xml', + conf_dir = self.CONF_DIR, + configurations = self.getConfig()['configurations']['yarn-client'], + configuration_attributes = self.getConfig()['configuration_attributes']['yarn-client'], + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('XmlConfig', 'hawq-site.xml', + conf_dir = self.CONF_DIR, + configurations = self.getConfig()['configurations']['hawq-site'], + configuration_attributes = self.getConfig()['configuration_attributes']['hawq-site'], + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + if self.COMPONENT_TYPE == 'master': + self.assertResourceCalled('File', self.CONF_DIR + 'hawq_check.cnf', + content = self.getConfig()['configurations']['hawq-check-env']['content'], + owner = self.GPADMIN, + group = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('File', self.CONF_DIR + 'slaves', + content = InlineTemplate('c6401.ambari.apache.org\nc6402.ambari.apache.org\nc6403.ambari.apache.org\n\n'), + group = self.GPADMIN, + owner = self.GPADMIN, + mode = 0644 + ) + + self.assertResourceCalled('Directory', '/data/hawq/' + self.COMPONENT_TYPE, + group = self.GPADMIN, + owner = self.GPADMIN, + create_parents = True + ) + + self.assertResourceCalled('Execute', 'chmod 700 /data/hawq/' + self.COMPONENT_TYPE, + user = 'root', + timeout = 600 + ) + + self.assertResourceCalled('Directory', '/data/hawq/tmp/' + self.COMPONENT_TYPE, + group = self.GPADMIN, + owner = self.GPADMIN, + create_parents = True + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/85c97e3a/ambari-server/src/test/python/common-services/HAWQ/test_alert_component_status.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_alert_component_status.py b/ambari-server/src/test/python/common-services/HAWQ/test_alert_component_status.py index b2e1d4d..971c618 100644 --- a/ambari-server/src/test/python/common-services/HAWQ/test_alert_component_status.py +++ b/ambari-server/src/test/python/common-services/HAWQ/test_alert_component_status.py @@ -21,17 +21,16 @@ limitations under the License. # System imports import os import sys - from mock.mock import patch # Local imports -from stacks.utils.RMFTestCase import * +from stacks.utils.RMFTestCase import RMFTestCase -COMMON_SERVICES_ALERTS_DIR = "HAWQ/2.0.0/package/alerts" +HAWQ_ALERTS_DIR = "HAWQ/2.0.0/package/alerts" file_path = os.path.dirname(os.path.abspath(__file__)) -file_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(file_path))))) -file_path = os.path.join(file_path, "main", "resources", "common-services", COMMON_SERVICES_ALERTS_DIR) +file_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(file_path)))) +file_path = os.path.join(file_path, "main", "resources", "common-services", HAWQ_ALERTS_DIR) WORKING_CONFIGS = { "{{hawq-site/hawq_master_address_port}}": "5432", http://git-wip-us.apache.org/repos/asf/ambari/blob/85c97e3a/ambari-server/src/test/python/common-services/HAWQ/test_alert_segment_registration_status.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_alert_segment_registration_status.py b/ambari-server/src/test/python/common-services/HAWQ/test_alert_segment_registration_status.py index 4066e29..f1fde066 100644 --- a/ambari-server/src/test/python/common-services/HAWQ/test_alert_segment_registration_status.py +++ b/ambari-server/src/test/python/common-services/HAWQ/test_alert_segment_registration_status.py @@ -21,17 +21,16 @@ limitations under the License. # System imports import os import sys -from resource_management.core.shell import call from mock.mock import patch # Local imports -from stacks.utils.RMFTestCase import * +from stacks.utils.RMFTestCase import RMFTestCase -COMMON_SERVICES_ALERTS_DIR = "HAWQ/2.0.0/package/alerts" +HAWQ_ALERTS_DIR = "HAWQ/2.0.0/package/alerts" file_path = os.path.dirname(os.path.abspath(__file__)) -file_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(file_path))))) -file_path = os.path.join(file_path, "main", "resources", "common-services", COMMON_SERVICES_ALERTS_DIR) +file_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(file_path)))) +file_path = os.path.join(file_path, "main", "resources", "common-services", HAWQ_ALERTS_DIR) RESULT_STATE_OK = 'OK' RESULT_STATE_WARNING = 'WARNING' http://git-wip-us.apache.org/repos/asf/ambari/blob/85c97e3a/ambari-server/src/test/python/common-services/HAWQ/test_alert_sync_status.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_alert_sync_status.py b/ambari-server/src/test/python/common-services/HAWQ/test_alert_sync_status.py index a44cad1..fa72b35 100644 --- a/ambari-server/src/test/python/common-services/HAWQ/test_alert_sync_status.py +++ b/ambari-server/src/test/python/common-services/HAWQ/test_alert_sync_status.py @@ -21,17 +21,16 @@ limitations under the License. # System imports import os import sys - from mock.mock import patch # Local imports -from stacks.utils.RMFTestCase import * +from stacks.utils.RMFTestCase import RMFTestCase -COMMON_SERVICES_ALERTS_DIR = "HAWQ/2.0.0/package/alerts" +HAWQ_ALERTS_DIR = "HAWQ/2.0.0/package/alerts" file_path = os.path.dirname(os.path.abspath(__file__)) -file_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(file_path))))) -file_path = os.path.join(file_path, "main", "resources", "common-services", COMMON_SERVICES_ALERTS_DIR) +file_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(file_path)))) +file_path = os.path.join(file_path, "main", "resources", "common-services", HAWQ_ALERTS_DIR) RESULT_STATE_OK = 'OK' RESULT_STATE_WARNING = 'WARNING' http://git-wip-us.apache.org/repos/asf/ambari/blob/85c97e3a/ambari-server/src/test/python/common-services/HAWQ/test_hawqmaster.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_hawqmaster.py b/ambari-server/src/test/python/common-services/HAWQ/test_hawqmaster.py index 62d54cf..a483656 100644 --- a/ambari-server/src/test/python/common-services/HAWQ/test_hawqmaster.py +++ b/ambari-server/src/test/python/common-services/HAWQ/test_hawqmaster.py @@ -18,153 +18,62 @@ See the License for the specific language governing permissions and limitations under the License. ''' -import os, json, copy, crypt from mock.mock import patch -from stacks.utils.RMFTestCase import RMFTestCase, InlineTemplate, UnknownConfigurationMock +from stacks.utils.RMFTestCase import InlineTemplate, UnknownConfigurationMock +from hawq_base_test_case import HawqBaseTestCase -class TestHawqMaster(RMFTestCase): +class TestHawqMaster(HawqBaseTestCase): - COMMON_SERVICES_PACKAGE_DIR = 'HAWQ/2.0.0/package' - STACK_VERSION = '2.3' - GPADMIN = 'gpadmin' - POSTGRES = 'postgres' + COMPONENT_TYPE = 'master' DEFAULT_IMMUTABLE_PATHS = ['/apps/hive/warehouse', '/apps/falcon', '/mr-history/done', '/app-logs', '/tmp'] - CONFIG_FILE = os.path.join(os.path.dirname(__file__), '../configs/hawq_default.json') - HAWQ_CHECK_COMMAND = 'source /usr/local/hawq/greenplum_path.sh && export PGHOST="c6403.ambari.apache.org" && hawq check -f /usr/local/hawq/etc/hawq_hosts --hadoop /usr/phd/current/hadoop-client --config /usr/local/hawq/etc/hawq_check.cnf ' - with open(CONFIG_FILE, "r") as f: - hawq_config = json.load(f) + HAWQ_CHECK_COMMAND = 'export PGHOST="c6403.ambari.apache.org" && hawq check -f /usr/local/hawq/etc/hawq_hosts --hadoop /usr/phd/current/hadoop-client --config /usr/local/hawq/etc/hawq_check.cnf ' - def setUp(self): - self.config_dict = copy.deepcopy(self.hawq_config) - - def __asserts_for_configure(self): - - self.assertResourceCalled('Group', self.GPADMIN, - ignore_failures = True - ) - - self.assertResourceCalled('User', self.GPADMIN, - gid = self.GPADMIN, - groups = [self.GPADMIN, u'hadoop'], - ignore_failures = True, - password = crypt.crypt(self.config_dict['configurations']['hawq-env']['hawq_password'], "$1$salt$") - ) - - self.assertResourceCalled('Group', self.POSTGRES, - ignore_failures = True - ) - - self.assertResourceCalled('User', self.POSTGRES, - gid = self.POSTGRES, - groups = [self.POSTGRES, u'hadoop'], - ignore_failures = True - ) - - self.assertResourceCalled('Execute', 'chown -R gpadmin:gpadmin /usr/local/hawq/', - timeout = 600 - ) - - self.assertResourceCalled('XmlConfig', 'hdfs-client.xml', - conf_dir = '/usr/local/hawq/etc/', - configurations = self.getConfig()['configurations']['hdfs-client'], - configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-client'], - group = self.GPADMIN, - owner = self.GPADMIN, - mode = 0644 - ) - - self.assertResourceCalled('XmlConfig', 'yarn-client.xml', - conf_dir = '/usr/local/hawq/etc/', - configurations = self.getConfig()['configurations']['yarn-client'], - configuration_attributes = self.getConfig()['configuration_attributes']['yarn-client'], - group = self.GPADMIN, - owner = self.GPADMIN, - mode = 0644 - ) - - self.assertResourceCalled('XmlConfig', 'hawq-site.xml', - conf_dir = '/usr/local/hawq/etc/', - configurations = self.getConfig()['configurations']['hawq-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['hawq-site'], - group = self.GPADMIN, - owner = self.GPADMIN, - mode = 0644 - ) - - self.assertResourceCalled('File', '/usr/local/hawq/etc/hawq_check.cnf', - content = self.getConfig()['configurations']['hawq-check-env']['content'], - owner = self.GPADMIN, - group = self.GPADMIN, - mode = 0644 - ) - - self.assertResourceCalled('File', '/usr/local/hawq/etc/slaves', - content = InlineTemplate('c6401.ambari.apache.org\nc6402.ambari.apache.org\nc6403.ambari.apache.org\n\n'), - group = self.GPADMIN, - owner = self.GPADMIN, - mode = 0644 - ) - - self.assertResourceCalled('Directory', '/data/hawq/master', - group = self.GPADMIN, - owner = self.GPADMIN, - create_parents = True - ) - - self.assertResourceCalled('Execute', 'chmod 700 /data/hawq/master', - user = 'root', - timeout = 600 - ) - - self.assertResourceCalled('Directory', '/data/hawq/tmp/master', - group = self.GPADMIN, - owner = self.GPADMIN, - create_parents = True - ) - - - @patch ('hawqmaster.common.__set_osparams') + @patch ('common.__set_osparams') def test_configure_default(self, set_osparams_mock): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py', + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqmaster.py', classname = 'HawqMaster', command = 'configure', config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) - self.__asserts_for_configure() + self.asserts_for_configure() self.assertNoMoreResources() - @patch ('hawqmaster.common.__set_osparams') + @patch ('common.__set_osparams') def test_install_default(self, set_osparams_mock): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py', + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqmaster.py', classname = 'HawqMaster', command = 'install', config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) - self.__asserts_for_configure() + self.asserts_for_configure() self.assertNoMoreResources() - @patch ('hawqmaster.common.__set_osparams') - def test_start_default(self, set_osparams_mock): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py', + @patch ('common.__set_osparams') + @patch ('utils.exec_psql_cmd') + @patch ('common.__get_hdfs_dir_owner') + def test_start_default(self, owner_mock, psql_mock, set_osparams_mock): + + owner_mock.return_value = 'postgres' + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqmaster.py', classname = 'HawqMaster', command = 'start', config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) - self.__asserts_for_configure() + self.asserts_for_configure() self.assertResourceCalled('HdfsResource', '/hawq_data', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, @@ -195,7 +104,7 @@ class TestHawqMaster(RMFTestCase): principal_name = UnknownConfigurationMock() ) - self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && hawq init master -a -v --ignore-bad-hosts', + self.assertResourceCalled('Execute', self.SOURCE_HAWQ_SCRIPT + 'hawq init master -a -v --ignore-bad-hosts', logoutput = True, not_if = None, only_if = None, @@ -206,14 +115,14 @@ class TestHawqMaster(RMFTestCase): self.assertNoMoreResources() - def __asserts_for_stop(self, componentCommand, expectedCommand): + def asserts_for_stop(self, componentCommand, expectedCommand): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py', + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqmaster.py', classname = 'HawqMaster', command = componentCommand, config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) self.assertResourceCalled('Execute', expectedCommand, @@ -227,35 +136,35 @@ class TestHawqMaster(RMFTestCase): self.assertNoMoreResources() - @patch ('hawqmaster.common.__set_osparams') + @patch ('common.__set_osparams') @patch ('common.get_local_hawq_site_property_value') def test_stop_default(self, get_local_hawq_site_property_value_mock, set_osparams_mock): """ Run Stop HAWQMASTER """ get_local_hawq_site_property_value_mock.return_value = 5432 - self.__asserts_for_stop('stop', 'source /usr/local/hawq/greenplum_path.sh && hawq stop master -M fast -a -v') + self.asserts_for_stop('stop', self.SOURCE_HAWQ_SCRIPT + 'hawq stop master -M fast -a -v') - @patch ('hawqmaster.common.__set_osparams') + @patch ('common.__set_osparams') @patch ('common.get_local_hawq_site_property_value') def test_stop_cluster_immediate(self, get_local_hawq_site_property_value_mock, set_osparams_mock): """ Run Stop HAWQ Cluster Immediate Mode """ get_local_hawq_site_property_value_mock.return_value = 5432 - self.__asserts_for_stop('immediate_stop_hawq_service','source /usr/local/hawq/greenplum_path.sh && hawq stop cluster -M immediate -a -v') + self.asserts_for_stop('immediate_stop_hawq_service', self.SOURCE_HAWQ_SCRIPT + 'hawq stop cluster -M immediate -a -v') def __asserts_for_hawq_check(self, expectedCommand): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py', + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqmaster.py', classname = 'HawqMaster', command = 'run_hawq_check', config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) - self.assertResourceCalled('File', "/usr/local/hawq/etc/hawq_hosts", + self.assertResourceCalled('File', self.CONF_DIR + 'hawq_hosts', content = InlineTemplate("{% for host in hawq_all_hosts %}{{host}}\n{% endfor %}"), group = self.GPADMIN, owner = self.GPADMIN, @@ -276,7 +185,7 @@ class TestHawqMaster(RMFTestCase): def test_run_hawq_check_case1(self): """ Running HAWQ Check Case 1: Non HDFS-HA, Standalone Resource Management, Not Kerberized """ - expectedCommand = self.HAWQ_CHECK_COMMAND + expectedCommand = self.SOURCE_HAWQ_SCRIPT + self.HAWQ_CHECK_COMMAND self.__asserts_for_hawq_check(expectedCommand) @@ -284,7 +193,7 @@ class TestHawqMaster(RMFTestCase): """ Running HAWQ Check Case 2: Non HDFS-HA, Standalone Resource Management, Kerberized """ self.config_dict['configurations']['cluster-env']['security_enabled'] = "true" - expectedCommand = "{0}--kerberos".format(self.HAWQ_CHECK_COMMAND) + expectedCommand = "{0}{1}--kerberos".format(self.SOURCE_HAWQ_SCRIPT, self.HAWQ_CHECK_COMMAND) self.__asserts_for_hawq_check(expectedCommand) @@ -292,7 +201,7 @@ class TestHawqMaster(RMFTestCase): """ Running HAWQ Check Case 3: Non HDFS-HA, YARN Resource Management Non YARN_HA, Not Kerberized """ self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" - expectedCommand = "{0}--yarn".format(self.HAWQ_CHECK_COMMAND) + expectedCommand = "{0}{1}--yarn".format(self.SOURCE_HAWQ_SCRIPT, self.HAWQ_CHECK_COMMAND) self.__asserts_for_hawq_check(expectedCommand) @@ -301,7 +210,7 @@ class TestHawqMaster(RMFTestCase): self.config_dict['configurations']['cluster-env']['security_enabled'] = "true" self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" - expectedCommand = "{0}--yarn --kerberos".format(self.HAWQ_CHECK_COMMAND) + expectedCommand = "{0}{1}--yarn --kerberos".format(self.SOURCE_HAWQ_SCRIPT, self.HAWQ_CHECK_COMMAND) self.__asserts_for_hawq_check(expectedCommand) @@ -310,7 +219,7 @@ class TestHawqMaster(RMFTestCase): self.config_dict['configurations']['yarn-site']['yarn.resourcemanager.ha.enabled'] = "true" self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" - expectedCommand = "{0}--yarn-ha".format(self.HAWQ_CHECK_COMMAND) + expectedCommand = "{0}{1}--yarn-ha".format(self.SOURCE_HAWQ_SCRIPT, self.HAWQ_CHECK_COMMAND) self.__asserts_for_hawq_check(expectedCommand) @@ -320,7 +229,7 @@ class TestHawqMaster(RMFTestCase): self.config_dict['configurations']['cluster-env']['security_enabled'] = "true" self.config_dict['configurations']['yarn-site']['yarn.resourcemanager.ha.enabled'] = "true" self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" - expectedCommand = "{0}--yarn-ha --kerberos".format(self.HAWQ_CHECK_COMMAND) + expectedCommand = "{0}{1}--yarn-ha --kerberos".format(self.SOURCE_HAWQ_SCRIPT, self.HAWQ_CHECK_COMMAND) self.__asserts_for_hawq_check(expectedCommand) @@ -328,7 +237,7 @@ class TestHawqMaster(RMFTestCase): """ Running HAWQ Check Case 7: HDFS-HA, Standalone Resource Management, Not Kerberized """ self.config_dict['configurations']['hdfs-site']['dfs.nameservices'] = "haservice" - expectedCommand = "{0}--hdfs-ha".format(self.HAWQ_CHECK_COMMAND) + expectedCommand = "{0}{1}--hdfs-ha".format(self.SOURCE_HAWQ_SCRIPT, self.HAWQ_CHECK_COMMAND) self.__asserts_for_hawq_check(expectedCommand) @@ -337,7 +246,7 @@ class TestHawqMaster(RMFTestCase): self.config_dict['configurations']['cluster-env']['security_enabled'] = "true" self.config_dict['configurations']['hdfs-site']['dfs.nameservices'] = "haservice" - expectedCommand = "{0}--hdfs-ha --kerberos".format(self.HAWQ_CHECK_COMMAND) + expectedCommand = "{0}{1}--hdfs-ha --kerberos".format(self.SOURCE_HAWQ_SCRIPT, self.HAWQ_CHECK_COMMAND) self.__asserts_for_hawq_check(expectedCommand) @@ -346,7 +255,7 @@ class TestHawqMaster(RMFTestCase): self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" self.config_dict['configurations']['hdfs-site']['dfs.nameservices'] = "haservice" - expectedCommand = "{0}--hdfs-ha --yarn".format(self.HAWQ_CHECK_COMMAND) + expectedCommand = "{0}{1}--hdfs-ha --yarn".format(self.SOURCE_HAWQ_SCRIPT, self.HAWQ_CHECK_COMMAND) self.__asserts_for_hawq_check(expectedCommand) @@ -356,7 +265,7 @@ class TestHawqMaster(RMFTestCase): self.config_dict['configurations']['cluster-env']['security_enabled'] = "true" self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" self.config_dict['configurations']['hdfs-site']['dfs.nameservices'] = "haservice" - expectedCommand = "{0}--hdfs-ha --yarn --kerberos".format(self.HAWQ_CHECK_COMMAND) + expectedCommand = "{0}{1}--hdfs-ha --yarn --kerberos".format(self.SOURCE_HAWQ_SCRIPT, self.HAWQ_CHECK_COMMAND) self.__asserts_for_hawq_check(expectedCommand) @@ -366,7 +275,7 @@ class TestHawqMaster(RMFTestCase): self.config_dict['configurations']['yarn-site']['yarn.resourcemanager.ha.enabled'] = "true" self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" self.config_dict['configurations']['hdfs-site']['dfs.nameservices'] = "haservice" - expectedCommand = "{0}--hdfs-ha --yarn-ha".format(self.HAWQ_CHECK_COMMAND) + expectedCommand = "{0}{1}--hdfs-ha --yarn-ha".format(self.SOURCE_HAWQ_SCRIPT, self.HAWQ_CHECK_COMMAND) self.__asserts_for_hawq_check(expectedCommand) @@ -377,22 +286,22 @@ class TestHawqMaster(RMFTestCase): self.config_dict['configurations']['yarn-site']['yarn.resourcemanager.ha.enabled'] = "true" self.config_dict['configurations']['hawq-site']['hawq_global_rm_type'] = "yarn" self.config_dict['configurations']['hdfs-site']['dfs.nameservices'] = "haservice" - expectedCommand = "{0}--hdfs-ha --yarn-ha --kerberos".format(self.HAWQ_CHECK_COMMAND) + expectedCommand = "{0}{1}--hdfs-ha --yarn-ha --kerberos".format(self.SOURCE_HAWQ_SCRIPT, self.HAWQ_CHECK_COMMAND) self.__asserts_for_hawq_check(expectedCommand) def test_resync_hawq_standby(self): """ Run custom command Resync HAWQ Standby """ - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py', + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqmaster.py', classname = 'HawqMaster', command = 'resync_hawq_standby', config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) - self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && export PGHOST="c6403.ambari.apache.org" && hawq init standby -n -a -v -M fast', + self.assertResourceCalled('Execute', self.SOURCE_HAWQ_SCRIPT + 'export PGHOST="c6403.ambari.apache.org" && hawq init standby -n -a -v -M fast', user = self.GPADMIN, timeout = 900, not_if = None, @@ -406,15 +315,15 @@ class TestHawqMaster(RMFTestCase): def test_remove_hawq_standby(self): """ Run custom command Remove HAWQ Standby """ - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py', + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqmaster.py', classname = 'HawqMaster', command = 'remove_hawq_standby', config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) - self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && export PGHOST="c6403.ambari.apache.org" && hawq init standby -a -v -r --ignore-bad-hosts', + self.assertResourceCalled('Execute', self.SOURCE_HAWQ_SCRIPT + 'export PGHOST="c6403.ambari.apache.org" && hawq init standby -a -v -r --ignore-bad-hosts', user = self.GPADMIN, timeout = 900, not_if = None, http://git-wip-us.apache.org/repos/asf/ambari/blob/85c97e3a/ambari-server/src/test/python/common-services/HAWQ/test_hawqsegment.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_hawqsegment.py b/ambari-server/src/test/python/common-services/HAWQ/test_hawqsegment.py index 48203ca..4237300 100644 --- a/ambari-server/src/test/python/common-services/HAWQ/test_hawqsegment.py +++ b/ambari-server/src/test/python/common-services/HAWQ/test_hawqsegment.py @@ -18,140 +18,58 @@ See the License for the specific language governing permissions and limitations under the License. ''' -import os, json, copy, crypt from mock.mock import patch -from stacks.utils.RMFTestCase import RMFTestCase +from hawq_base_test_case import HawqBaseTestCase -class TestHawqSegment(RMFTestCase): +class TestHawqSegment(HawqBaseTestCase): - COMMON_SERVICES_PACKAGE_DIR = 'HAWQ/2.0.0/package' - STACK_VERSION = '2.3' - GPADMIN = 'gpadmin' - POSTGRES = 'postgres' - CONFIG_FILE = os.path.join(os.path.dirname(__file__), '../configs/hawq_default.json') - with open(CONFIG_FILE, "r") as f: - hawq_config = json.load(f) + COMPONENT_TYPE = 'segment' - def setUp(self): - self.config_dict = copy.deepcopy(self.hawq_config) - - def __asserts_for_configure(self): - - self.assertResourceCalled('Group', self.GPADMIN, - ignore_failures = True - ) - - self.assertResourceCalled('User', self.GPADMIN, - gid = self.GPADMIN, - groups = [self.GPADMIN, u'hadoop'], - ignore_failures = True, - password = crypt.crypt(self.getConfig()['configurations']['hawq-env']['hawq_password'], "$1$salt$") - ) - - self.assertResourceCalled('Group', self.POSTGRES, - ignore_failures = True - ) - - self.assertResourceCalled('User', self.POSTGRES, - gid = self.POSTGRES, - groups = [self.POSTGRES, u'hadoop'], - ignore_failures = True - ) - - self.assertResourceCalled('Execute', 'chown -R gpadmin:gpadmin /usr/local/hawq/', - timeout = 600 - ) - - self.assertResourceCalled('XmlConfig', 'hdfs-client.xml', - conf_dir = '/usr/local/hawq/etc/', - configurations = self.getConfig()['configurations']['hdfs-client'], - configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-client'], - group = self.GPADMIN, - owner = self.GPADMIN, - mode = 0644 - ) - - self.assertResourceCalled('XmlConfig', 'yarn-client.xml', - conf_dir = '/usr/local/hawq/etc/', - configurations = self.getConfig()['configurations']['yarn-client'], - configuration_attributes = self.getConfig()['configuration_attributes']['yarn-client'], - group = self.GPADMIN, - owner = self.GPADMIN, - mode = 0644 - ) - - self.assertResourceCalled('XmlConfig', 'hawq-site.xml', - conf_dir = '/usr/local/hawq/etc/', - configurations = self.getConfig()['configurations']['hawq-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['hawq-site'], - group = self.GPADMIN, - owner = self.GPADMIN, - mode = 0644 - ) - - self.assertResourceCalled('Directory', '/data/hawq/segment', - owner = self.GPADMIN, - group = self.GPADMIN, - create_parents = True - ) - - self.assertResourceCalled('Execute', 'chmod 700 /data/hawq/segment', - user = 'root', - timeout = 600 - ) - - self.assertResourceCalled('Directory', '/data/hawq/tmp/segment', - owner = self.GPADMIN, - group = self.GPADMIN, - create_parents = True - ) - - - @patch ('hawqsegment.common.__set_osparams') + @patch ('common.__set_osparams') def test_configure_default(self, set_osparams_mock): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqsegment.py', + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqsegment.py', classname = 'HawqSegment', command = 'configure', config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) - self.__asserts_for_configure() + self.asserts_for_configure() self.assertNoMoreResources() - @patch ('hawqsegment.common.__set_osparams') + @patch ('common.__set_osparams') def test_install_default(self, set_osparams_mock): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqsegment.py', + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqsegment.py', classname = 'HawqSegment', command = 'install', config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) - self.__asserts_for_configure() + self.asserts_for_configure() self.assertNoMoreResources() - @patch ('hawqsegment.common.__set_osparams') + @patch ('common.__set_osparams') def test_start_default(self, set_osparams_mock): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqsegment.py', + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqsegment.py', classname = 'HawqSegment', command = 'start', config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) - self.__asserts_for_configure() + self.asserts_for_configure() - self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && hawq init segment -a -v', + self.assertResourceCalled('Execute', self.SOURCE_HAWQ_SCRIPT + 'hawq init segment -a -v', logoutput = True, not_if = None, only_if = None, @@ -166,15 +84,15 @@ class TestHawqSegment(RMFTestCase): def test_stop_default(self, get_local_hawq_site_property_value_mock): get_local_hawq_site_property_value_mock.return_value = 40000 - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqsegment.py', + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqsegment.py', classname = 'HawqSegment', command = 'stop', config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) - self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && hawq stop segment -M fast -a -v', + self.assertResourceCalled('Execute', self.SOURCE_HAWQ_SCRIPT + 'hawq stop segment -M fast -a -v', logoutput = True, not_if = None, only_if = "netstat -tupln | egrep ':40000\\s' | egrep postgres", http://git-wip-us.apache.org/repos/asf/ambari/blob/85c97e3a/ambari-server/src/test/python/common-services/HAWQ/test_hawqstandby.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_hawqstandby.py b/ambari-server/src/test/python/common-services/HAWQ/test_hawqstandby.py index 8674549..e553514 100644 --- a/ambari-server/src/test/python/common-services/HAWQ/test_hawqstandby.py +++ b/ambari-server/src/test/python/common-services/HAWQ/test_hawqstandby.py @@ -18,154 +18,58 @@ See the License for the specific language governing permissions and limitations under the License. ''' -import os, json, copy, crypt from mock.mock import patch -from stacks.utils.RMFTestCase import RMFTestCase, InlineTemplate +from hawq_base_test_case import HawqBaseTestCase -class TestHawqStandby(RMFTestCase): +class TestHawqStandby(HawqBaseTestCase): - COMMON_SERVICES_PACKAGE_DIR = 'HAWQ/2.0.0/package' - STACK_VERSION = '2.3' - GPADMIN = 'gpadmin' - POSTGRES = 'postgres' - CONFIG_FILE = os.path.join(os.path.dirname(__file__), '../configs/hawq_default.json') - with open(CONFIG_FILE, "r") as f: - hawq_config = json.load(f) + COMPONENT_TYPE = 'master' - def setUp(self): - self.config_dict = copy.deepcopy(self.hawq_config) - - def __asserts_for_configure(self): - - self.assertResourceCalled('Group', self.GPADMIN, - ignore_failures = True - ) - - self.assertResourceCalled('User', self.GPADMIN, - gid = self.GPADMIN, - groups = [self.GPADMIN, u'hadoop'], - ignore_failures = True, - password = crypt.crypt(self.getConfig()['configurations']['hawq-env']['hawq_password'], "$1$salt$") - ) - - self.assertResourceCalled('Group', self.POSTGRES, - ignore_failures = True - ) - - self.assertResourceCalled('User', self.POSTGRES, - gid = self.POSTGRES, - groups = [self.POSTGRES, u'hadoop'], - ignore_failures = True - ) - - self.assertResourceCalled('Execute', 'chown -R gpadmin:gpadmin /usr/local/hawq/', - timeout = 600 - ) - - self.assertResourceCalled('XmlConfig', 'hdfs-client.xml', - conf_dir = '/usr/local/hawq/etc/', - configurations = self.getConfig()['configurations']['hdfs-client'], - configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-client'], - group = self.GPADMIN, - owner = self.GPADMIN, - mode = 0644 - ) - - self.assertResourceCalled('XmlConfig', 'yarn-client.xml', - conf_dir = '/usr/local/hawq/etc/', - configurations = self.getConfig()['configurations']['yarn-client'], - configuration_attributes = self.getConfig()['configuration_attributes']['yarn-client'], - group = self.GPADMIN, - owner = self.GPADMIN, - mode = 0644 - ) - - self.assertResourceCalled('XmlConfig', 'hawq-site.xml', - conf_dir = '/usr/local/hawq/etc/', - configurations = self.getConfig()['configurations']['hawq-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['hawq-site'], - group = self.GPADMIN, - owner = self.GPADMIN, - mode = 0644 - ) - - self.assertResourceCalled('File', '/usr/local/hawq/etc/hawq_check.cnf', - content = self.getConfig()['configurations']['hawq-check-env']['content'], - owner = self.GPADMIN, - group = self.GPADMIN, - mode = 0644 - ) - - self.assertResourceCalled('File', '/usr/local/hawq/etc/slaves', - content = InlineTemplate('c6401.ambari.apache.org\nc6402.ambari.apache.org\nc6403.ambari.apache.org\n\n'), - group = self.GPADMIN, - owner = self.GPADMIN, - mode = 0644 - ) - - self.assertResourceCalled('Directory', '/data/hawq/master', - group = self.GPADMIN, - owner = self.GPADMIN, - create_parents = True - ) - - self.assertResourceCalled('Execute', 'chmod 700 /data/hawq/master', - user = 'root', - timeout = 600 - ) - - self.assertResourceCalled('Directory', '/data/hawq/tmp/master', - group = self.GPADMIN, - owner = self.GPADMIN, - create_parents = True - ) - - - @patch ('hawqstandby.common.__set_osparams') + @patch ('common.__set_osparams') def test_configure_default(self, set_osparams_mock): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqstandby.py', + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqstandby.py', classname = 'HawqStandby', command = 'configure', config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) - self.__asserts_for_configure() + self.asserts_for_configure() self.assertNoMoreResources() - @patch ('hawqstandby.common.__set_osparams') + @patch ('common.__set_osparams') def test_install_default(self, set_osparams_mock): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqstandby.py', + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqstandby.py', classname = 'HawqStandby', command = 'install', config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) - self.__asserts_for_configure() + self.asserts_for_configure() self.assertNoMoreResources() - @patch ('hawqstandby.common.__set_osparams') + @patch ('common.__set_osparams') def test_start_default(self, set_osparams_mock): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqstandby.py', + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqstandby.py', classname = 'HawqStandby', command = 'start', config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) - self.__asserts_for_configure() + self.asserts_for_configure() - self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && hawq init standby -a -v', + self.assertResourceCalled('Execute', self.SOURCE_HAWQ_SCRIPT + 'hawq init standby -a -v', logoutput = True, not_if = None, only_if = None, @@ -175,20 +79,20 @@ class TestHawqStandby(RMFTestCase): self.assertNoMoreResources() - @patch ('hawqstandby.common.__set_osparams') + @patch ('common.__set_osparams') @patch ('common.get_local_hawq_site_property_value') def test_stop_default(self, get_local_hawq_site_property_value_mock, set_osparams_mock): get_local_hawq_site_property_value_mock.return_value = 5432 - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqstandby.py', + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqstandby.py', classname = 'HawqStandby', command = 'stop', config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) - self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && hawq stop standby -M fast -a -v', + self.assertResourceCalled('Execute', self.SOURCE_HAWQ_SCRIPT + 'hawq stop standby -M fast -a -v', logoutput = True, not_if = None, only_if = "netstat -tupln | egrep ':5432\\s' | egrep gpsyncmaster", @@ -205,16 +109,16 @@ class TestHawqStandby(RMFTestCase): get_local_hawq_site_property_value_mock.return_value = 5432 - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqstandby.py', + self.executeScript(self.HAWQ_PACKAGE_DIR + '/scripts/hawqstandby.py', classname = 'HawqStandby', command = 'activate_hawq_standby', config_dict = self.config_dict, stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES + target = self.TARGET_COMMON_SERVICES ) self.assertResourceCalled('XmlConfig', 'hawq-site.xml', - conf_dir = '/usr/local/hawq/etc/', + conf_dir = self.CONF_DIR, configurations = self.getConfig()['configurations']['hawq-site'], configuration_attributes = self.getConfig()['configuration_attributes']['hawq-site'], group = self.GPADMIN, @@ -222,7 +126,7 @@ class TestHawqStandby(RMFTestCase): mode = 0644 ) - self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && export PGHOST=\"c6402.ambari.apache.org\" && hawq activate standby -a -M fast -v --ignore-bad-hosts', + self.assertResourceCalled('Execute', self.SOURCE_HAWQ_SCRIPT + 'export PGHOST=\"c6402.ambari.apache.org\" && hawq activate standby -a -M fast -v --ignore-bad-hosts', logoutput = True, not_if = None, only_if = None, @@ -230,7 +134,7 @@ class TestHawqStandby(RMFTestCase): timeout = 900 ) - self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh && hawq stop master -M fast -a -v', + self.assertResourceCalled('Execute', self.SOURCE_HAWQ_SCRIPT + 'hawq stop master -M fast -a -v', logoutput = True, not_if = None, only_if = "netstat -tupln | egrep ':5432\\s' | egrep postgres", http://git-wip-us.apache.org/repos/asf/ambari/blob/85c97e3a/ambari-server/src/test/python/common-services/HAWQ/test_service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_service_advisor.py b/ambari-server/src/test/python/common-services/HAWQ/test_service_advisor.py index e0f1c43..71c109f 100644 --- a/ambari-server/src/test/python/common-services/HAWQ/test_service_advisor.py +++ b/ambari-server/src/test/python/common-services/HAWQ/test_service_advisor.py @@ -16,13 +16,21 @@ See the License for the specific language governing permissions and limitations under the License. """ -import os, imp, json +import imp +import json +import os from unittest import TestCase + from mock.mock import patch, MagicMock class TestHAWQ200ServiceAdvisor(TestCase): + testDirectory = os.path.dirname(os.path.abspath(__file__)) + stack_advisor_path = os.path.join(testDirectory, '../../../../main/resources/stacks/stack_advisor.py') + with open(stack_advisor_path, 'rb') as fp: + imp.load_module('stack_advisor', fp, stack_advisor_path, ('.py', 'rb', imp.PY_SOURCE)) + serviceAdvisorPath = '../../../../main/resources/common-services/HAWQ/2.0.0/service_advisor.py' hawq200ServiceAdvisorPath = os.path.join(testDirectory, serviceAdvisorPath) with open(hawq200ServiceAdvisorPath, 'rb') as fp: http://git-wip-us.apache.org/repos/asf/ambari/blob/85c97e3a/ambari-server/src/test/python/common-services/HAWQ/test_utils.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_utils.py b/ambari-server/src/test/python/common-services/HAWQ/test_utils.py index 07737eb..fc88079 100644 --- a/ambari-server/src/test/python/common-services/HAWQ/test_utils.py +++ b/ambari-server/src/test/python/common-services/HAWQ/test_utils.py @@ -18,9 +18,8 @@ See the License for the specific language governing permissions and limitations under the License. ''' -from stacks.utils.RMFTestCase import * - import utils +from stacks.utils.RMFTestCase import RMFTestCase class TestUtils(RMFTestCase): http://git-wip-us.apache.org/repos/asf/ambari/blob/85c97e3a/ambari-server/src/test/python/common-services/PXF/test_alerts_api_status.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/PXF/test_alerts_api_status.py b/ambari-server/src/test/python/common-services/PXF/test_alerts_api_status.py index ee187e2..229265a 100644 --- a/ambari-server/src/test/python/common-services/PXF/test_alerts_api_status.py +++ b/ambari-server/src/test/python/common-services/PXF/test_alerts_api_status.py @@ -25,10 +25,10 @@ from mock.mock import patch # Local imports from stacks.utils.RMFTestCase import RMFTestCase -COMMON_SERVICES_ALERTS_DIR_PATH = "../../../../../main/resources/common-services/PXF/3.0.0/package/alerts" +PXF_ALERTS_DIR_PATH = "../../../../main/resources/common-services/PXF/3.0.0/package/alerts" current_dir = os.path.dirname(os.path.abspath(__file__)) -alerts_dir = os.path.abspath(os.path.join(current_dir, COMMON_SERVICES_ALERTS_DIR_PATH)) +pxf_alerts_dir = os.path.abspath(os.path.join(current_dir, PXF_ALERTS_DIR_PATH)) class TestAlertsApiStatus(RMFTestCase): @@ -38,7 +38,7 @@ class TestAlertsApiStatus(RMFTestCase): Because the class is present in a different folder, append its dir to the system path. Also, shorten the import name and make it a global so the test functions can access it. """ - sys.path.append(alerts_dir) + sys.path.append(pxf_alerts_dir) global api_status import api_status http://git-wip-us.apache.org/repos/asf/ambari/blob/85c97e3a/ambari-server/src/test/python/common-services/PXF/test_pxf.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/PXF/test_pxf.py b/ambari-server/src/test/python/common-services/PXF/test_pxf.py index c291261..b426d03 100644 --- a/ambari-server/src/test/python/common-services/PXF/test_pxf.py +++ b/ambari-server/src/test/python/common-services/PXF/test_pxf.py @@ -18,13 +18,16 @@ See the License for the specific language governing permissions and limitations under the License. ''' -import os, copy, json +import copy +import json +import os + from mock.mock import patch from stacks.utils.RMFTestCase import Template, RMFTestCase class TestPxf(RMFTestCase): - COMMON_SERVICES_PACKAGE_DIR = "PXF/3.0.0/package" + PXF_PACKAGE_DIR = "PXF/3.0.0/package" STACK_VERSION = "2.3" PXF_USER = 'pxf' PXF_GROUP = 'pxf' @@ -65,7 +68,7 @@ class TestPxf(RMFTestCase): @patch('shutil.copy2') def test_install_default(self, shutil_copy2_mock): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pxf.py", + self.executeScript(self.PXF_PACKAGE_DIR + "/scripts/pxf.py", classname="Pxf", command="install", config_dict=self.config_dict, @@ -96,7 +99,7 @@ class TestPxf(RMFTestCase): @patch('shutil.copy2') def test_configure_default(self, shutil_copy2_mock): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pxf.py", + self.executeScript(self.PXF_PACKAGE_DIR + "/scripts/pxf.py", classname="Pxf", command="configure", config_dict=self.config_dict, @@ -108,7 +111,7 @@ class TestPxf(RMFTestCase): @patch('shutil.copy2') def test_start_default(self, shutil_copy2_mock): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pxf.py", + self.executeScript(self.PXF_PACKAGE_DIR + "/scripts/pxf.py", classname="Pxf", command="start", config_dict=self.config_dict, @@ -128,7 +131,7 @@ class TestPxf(RMFTestCase): logoutput=True) def test_stop_default(self): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pxf.py", + self.executeScript(self.PXF_PACKAGE_DIR + "/scripts/pxf.py", classname="Pxf", command="stop", config_dict=self.config_dict, @@ -141,7 +144,7 @@ class TestPxf(RMFTestCase): logoutput=True) def test_status_default(self): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pxf.py", + self.executeScript(self.PXF_PACKAGE_DIR + "/scripts/pxf.py", classname="Pxf", command="status", config_dict=self.config_dict, http://git-wip-us.apache.org/repos/asf/ambari/blob/85c97e3a/ambari-server/src/test/python/common-services/PXF/test_service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/PXF/test_service_advisor.py b/ambari-server/src/test/python/common-services/PXF/test_service_advisor.py index 8b2a4ba..57ce594 100644 --- a/ambari-server/src/test/python/common-services/PXF/test_service_advisor.py +++ b/ambari-server/src/test/python/common-services/PXF/test_service_advisor.py @@ -16,21 +16,27 @@ See the License for the specific language governing permissions and limitations under the License. """ -import os, json +import imp +import json +import os from unittest import TestCase class TestPXF300ServiceAdvisor(TestCase): + testDirectory = os.path.dirname(os.path.abspath(__file__)) + stack_advisor_path = os.path.join(testDirectory, '../../../../main/resources/stacks/stack_advisor.py') + with open(stack_advisor_path, 'rb') as fp: + imp.load_module('stack_advisor', fp, stack_advisor_path, ('.py', 'rb', imp.PY_SOURCE)) + + serviceAdvisorPath = '../../../../main/resources/common-services/PXF/3.0.0/service_advisor.py' + pxf300ServiceAdvisorPath = os.path.join(testDirectory, serviceAdvisorPath) + with open(pxf300ServiceAdvisorPath, 'rb') as fp: + service_advisor_impl = imp.load_module('service_advisor_impl', fp, pxf300ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + def setUp(self): - import imp - self.testDirectory = os.path.dirname(os.path.abspath(__file__)) - self.pxf300ServiceAdvisorPath = os.path.join(self.testDirectory, '../../../../main/resources/common-services/PXF/3.0.0/service_advisor.py') - with open(self.pxf300ServiceAdvisorPath, 'rb') as fp: - service_advisor_impl = imp.load_module('service_advisor_impl', fp, self.pxf300ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) - serviceAdvisorClass = getattr(service_advisor_impl, 'PXF300ServiceAdvisor') + serviceAdvisorClass = getattr(self.service_advisor_impl, 'PXF300ServiceAdvisor') self.serviceAdvisor = serviceAdvisorClass() - self.PXF_PATH = "export HBASE_CLASSPATH=${HBASE_CLASSPATH}:/usr/lib/pxf/pxf-hbase.jar" def load_json(self, filename): http://git-wip-us.apache.org/repos/asf/ambari/blob/85c97e3a/ambari-server/src/test/python/unitTests.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/unitTests.py b/ambari-server/src/test/python/unitTests.py index b010804..037b6a5 100644 --- a/ambari-server/src/test/python/unitTests.py +++ b/ambari-server/src/test/python/unitTests.py @@ -98,15 +98,16 @@ def stack_test_executor(base_folder, service, stack, custom_tests, executor_resu test_mask = TEST_MASK server_src_dir = get_parent_path(base_folder, 'src') + script_folders = set() - base_stack_folder = os.path.join(server_src_dir, - "main", "resources", "stacks", get_stack_name(), stack) + if stack is not None: + base_stack_folder = os.path.join(server_src_dir, + "main", "resources", "stacks", get_stack_name(), stack) - script_folders = set() - for root, subFolders, files in os.walk(os.path.join(base_stack_folder, - "services", service)): - if os.path.split(root)[-1] in ["scripts", "files"] and service in root: - script_folders.add(root) + for root, subFolders, files in os.walk(os.path.join(base_stack_folder, + "services", service)): + if os.path.split(root)[-1] in ["scripts", "files"] and service in root: + script_folders.add(root) # Add the common-services scripts directories to the PATH base_commserv_folder = os.path.join(server_src_dir, "main", "resources", "common-services") @@ -163,7 +164,7 @@ def main(): sys.path.append(os.path.join(ambari_common_folder, "src/main/python")) sys.path.append(os.path.join(ambari_common_folder, "src/main/python/ambari_jinja2")) sys.path.append(os.path.join(ambari_common_folder, "src/test/python")) - sys.path.append(os.path.join(ambari_agent_folder, "src/main/python")) + sys.path.append(os.path.join(ambari_agent_folder, "src/main/python")) sys.path.append(os.path.join(ambari_server_folder, "src/test/python")) sys.path.append(os.path.join(ambari_server_folder, "src/main/python")) sys.path.append(os.path.join(ambari_server_folder, "src/main/resources/scripts")) @@ -189,6 +190,15 @@ def main(): 'service': service, 'stack': stack}) + #add tests for services under common-services + comm_serv_folder = os.path.join(pwd, 'common-services') + for service in os.listdir(comm_serv_folder): + current_service_dir = os.path.join(comm_serv_folder, service) + if os.path.isdir(current_service_dir) and service not in SERVICE_EXCLUDE: + test_variants.append({'directory': current_service_dir, + 'service': service, + 'stack': None}) + #run tests for every service in every stack in separate process has_failures = False test_runs = 0
