AMBARI-18943. If multiple HSI components are deployed then HSI start should not restart llap (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/795dd1f8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/795dd1f8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/795dd1f8 Branch: refs/heads/branch-2.5 Commit: 795dd1f814b8bf342a53c5a6c15ff60d568c2980 Parents: 351c20c Author: Sumit Mohanty <[email protected]> Authored: Sun Nov 20 15:26:38 2016 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Sun Nov 20 15:26:52 2016 -0800 ---------------------------------------------------------------------- .../package/scripts/hive_server_interactive.py | 22 +- .../0.12.0.2.0/package/scripts/params_linux.py | 1 + .../stacks/2.5/HIVE/test_hive_server_int.py | 112 ++ .../test/python/stacks/2.5/configs/hsi_ha.json | 1254 ++++++++++++++++++ 4 files changed, 1387 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/795dd1f8/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py index 996b4da..2c6d4b0 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py @@ -253,9 +253,24 @@ class HiveServerInteractiveDefault(HiveServerInteractive): def _llap_start(self, env, cleanup=False): import params env.set_params(params) + + LLAP_APP_NAME = 'llap0' + + if params.hive_server_interactive_ha: + """ + Check llap app state + """ + Logger.info("HSI HA is enabled. Checking if LLAP is already running ...") + status = self.check_llap_app_status(LLAP_APP_NAME, 2, params.hive_server_interactive_ha) + if status: + Logger.info("LLAP app '{0}' is already running.".format(LLAP_APP_NAME)) + return True + else: + Logger.info("LLAP app '{0}' is not running. llap will be started.".format(LLAP_APP_NAME)) + pass + Logger.info("Starting LLAP") LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir() - LLAP_APP_NAME = 'llap0' unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S') @@ -447,7 +462,7 @@ class HiveServerInteractiveDefault(HiveServerInteractive): Parameters: llap_app_name : deployed llap app name. num_retries : Number of retries to check the LLAP app status. """ - def check_llap_app_status(self, llap_app_name, num_retries): + def check_llap_app_status(self, llap_app_name, num_retries, return_immediately_if_stopped=False): # counters based on various states. curr_time = time.time() @@ -466,6 +481,9 @@ class HiveServerInteractiveDefault(HiveServerInteractive): Logger.error("Malformed JSON data received for LLAP app. Exiting ....") return False + if return_immediately_if_stopped and (llap_app_info['state'].upper() in ('APP_NOT_FOUND', 'COMPLETE')): + return False + if llap_app_info['state'].upper() == 'RUNNING_ALL': Logger.info( "LLAP app '{0}' in '{1}' state.".format(llap_app_name, llap_app_info['state'])) http://git-wip-us.apache.org/repos/asf/ambari/blob/795dd1f8/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index e214547..bba0f3b 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -312,6 +312,7 @@ hive_server_host = hive_server_hosts[0] if len(hive_server_hosts) > 0 else None hive_server_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts', []) hive_server_interactive_host = hive_server_interactive_hosts[0] if len(hive_server_interactive_hosts) > 0 else None +hive_server_interactive_ha = True if len(hive_server_interactive_hosts) > 1 else False # End, Common Hosts and Ports hive_transport_mode = config['configurations']['hive-site']['hive.server2.transport.mode'] http://git-wip-us.apache.org/repos/asf/ambari/blob/795dd1f8/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py index 7477457..3221d5d 100644 --- a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py +++ b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py @@ -131,6 +131,118 @@ class TestHiveServerInteractive(RMFTestCase): self.assertNoMoreResources() """ + Tests HSI start with llap package creation output having single line. + Sample output : "Prepared llap-slider-05Apr2016/run.sh for running LLAP" + HSI HA is enabled and llap is stopped, so it will be restartd + """ + #@patch("Script.get_tmp_dir()") + @patch("os.path.isfile") + @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") + @patch("socket.socket") + @patch("time.sleep") + def test_start_hsi_ha_with_llap_single_line_output(self, sleep_mock, socket_mock, copy_to_hfds_mock, is_file_mock): #, get_tmp_dir_mock): + self.maxDiff = None + copy_to_hfds_mock.return_value = False + s = socket_mock.return_value + is_file_mock.return_value = True + #get_tmp_dir_mock.return_value = "/var/lib/ambari-agent/tmp" + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server_interactive.py", + classname="HiveServerInteractive", + command="start", + config_file=self.get_src_folder() + "/test/python/stacks/2.5/configs/hsi_ha.json", + stack_version=self.STACK_VERSION, + target=RMFTestCase.TARGET_COMMON_SERVICES, + checked_call_mocks=[(0, """{ + \"state\" : \"COMPLETED\" + }""", ""), + (0, "Prepared llap-slider-05Apr2016/run.sh for running LLAP", ""), + (0, """{ + \"state\" : \"RUNNING_ALL\" + }""", ""), + (0, """{ + \"state\" : \"RUNNING_ALL\" + }""", ""), + (0, "OK.", "")], + ) + + self.assert_configure_default() + + self.assertResourceCalled('Execute', + '/home/hive/llap-slider-05Apr2016/run.sh', + logoutput= True, user='hive' + ) + self.assertResourceCalled('Execute', + 'hive --config /usr/hdp/current/hive-server2-hive2/conf/conf.server --service metatool -updateLocation hdfs://c6401.ambari.apache.org:8020 OK.', + environment={'PATH': '/usr/hdp/current/hadoop-client/bin'}, + user='hive' + ) + self.assertResourceCalled('Execute', + '/tmp/start_hiveserver2_interactive_script /var/run/hive/hive-server2-interactive.out /var/log/hive/hive-server2-interactive.err /var/run/hive/hive-interactive.pid /usr/hdp/current/hive-server2-hive2/conf/conf.server /var/log/hive', + environment={'HADOOP_HOME': '/usr/hdp/current/hadoop-client', + 'HIVE_BIN': 'hive2', + 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'}, + not_if="ls /var/run/hive/hive-interactive.pid >/dev/null 2>&1 && ps -p 123 >/dev/null 2>&1", + user='hive', + path=['/bin:/usr/hdp/current/hive-server2-hive2/bin:/usr/hdp/current/hadoop-client/bin'], + ) + self.assertResourceCalled('Execute', + '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/hdp/current/hive-server2-hive2/lib/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver', + path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], + tries=5, + try_sleep=10 + ) + self.assertNoMoreResources() + + """ + Tests HSI start with llap running and HA deployment. As llap is running it will not be started." + """ + #@patch("Script.get_tmp_dir()") + @patch("os.path.isfile") + @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") + @patch("socket.socket") + @patch("time.sleep") + def test_start_with_ha_and_llap_running(self, sleep_mock, socket_mock, copy_to_hfds_mock, is_file_mock): #, get_tmp_dir_mock): + self.maxDiff = None + copy_to_hfds_mock.return_value = False + s = socket_mock.return_value + is_file_mock.return_value = True + #get_tmp_dir_mock.return_value = "/var/lib/ambari-agent/tmp" + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server_interactive.py", + classname="HiveServerInteractive", + command="start", + config_file=self.get_src_folder() + "/test/python/stacks/2.5/configs/hsi_ha.json", + stack_version=self.STACK_VERSION, + target=RMFTestCase.TARGET_COMMON_SERVICES, + checked_call_mocks=[(0, """{ + \"state\" : \"RUNNING_ALL\" + }""", "")], + ) + + self.assert_configure_default() + + self.assertResourceCalled('Execute', + 'hive --config /usr/hdp/current/hive-server2-hive2/conf/conf.server --service metatool -updateLocation hdfs://c6401.ambari.apache.org:8020 OK.', + environment={'PATH': '/usr/hdp/current/hadoop-client/bin'}, + user='hive' + ) + self.assertResourceCalled('Execute', + '/tmp/start_hiveserver2_interactive_script /var/run/hive/hive-server2-interactive.out /var/log/hive/hive-server2-interactive.err /var/run/hive/hive-interactive.pid /usr/hdp/current/hive-server2-hive2/conf/conf.server /var/log/hive', + environment={'HADOOP_HOME': '/usr/hdp/current/hadoop-client', + 'HIVE_BIN': 'hive2', + 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'}, + not_if="ls /var/run/hive/hive-interactive.pid >/dev/null 2>&1 && ps -p 123 >/dev/null 2>&1", + user='hive', + path=['/bin:/usr/hdp/current/hive-server2-hive2/bin:/usr/hdp/current/hadoop-client/bin'], + ) + self.assertResourceCalled('Execute', + '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/hdp/current/hive-server2-hive2/lib/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver', + path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], + tries=5, + try_sleep=10 + ) + self.assertNoMoreResources() + + """ Tests HSI start with llap package creation output having multiple lines. Sample output : "UNWANTED STRING \n Prepared llap-slider-05Apr2016/run.sh for running LLAP \n UNWANTED STRING \n" """
