Repository: ambari Updated Branches: refs/heads/trunk 65ca08455 -> 9d3eeaaa9
AMBARI-22498. Remove trailing lines (if any) from llapstatus command before converting the o/p to JSON. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9d3eeaaa Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9d3eeaaa Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9d3eeaaa Branch: refs/heads/trunk Commit: 9d3eeaaa9eca4cda1b3c675da9341516d9d19706 Parents: 65ca084 Author: Swapan Shridhar <[email protected]> Authored: Wed Nov 22 10:24:38 2017 -0800 Committer: Swapan Shridhar <[email protected]> Committed: Wed Nov 22 10:24:38 2017 -0800 ---------------------------------------------------------------------- .../package/scripts/hive_server_interactive.py | 20 +++++++-- .../HIVE/running_withMOTDmsg_andTrailingMsg.txt | 46 ++++++++++++++++++++ .../stacks/2.5/HIVE/test_hive_server_int.py | 21 +++++++++ 3 files changed, 84 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9d3eeaaa/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py index c0b152e..caa3e9b 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py @@ -415,13 +415,14 @@ class HiveServerInteractiveDefault(HiveServerInteractive): """ - Remove extra lines from 'llapstatus' status output (eg: because of MOTD logging) so as to have a valid JSON data to be passed in - to JSON converter. + Remove extra lines (begginning/end) from 'llapstatus' status output (eg: because of MOTD logging) so as to have + a valid JSON data to be passed in to JSON converter. """ def _make_valid_json(self, output): ''' - Note: It is assumed right now that extra lines will be only at the start and not at the end. + Note: Extra lines (eg: because of MOTD) may be at the start or the end (some other logging getting appended) + of the passed-in data. Sample expected JSON to be passed for 'loads' is either of the form : @@ -457,6 +458,19 @@ class HiveServerInteractiveDefault(HiveServerInteractive): if (len_splits < 3): raise Fail ("Malformed JSON data received from 'llapstatus' command. Exiting ....") + # Firstly, remove extra lines from the END. + updated_splits = [] + for itr, line in enumerate(reversed(splits)): + if line == "}": # Our assumption of end of JSON data. + updated_splits = splits[:-itr] + break + + if len(updated_splits) > 0: + splits = updated_splits + len_splits = len(splits) + + + # Secondly, remove extra lines from the BEGGINNING. marker_idx = None # To detect where from to start reading for JSON data for idx, split in enumerate(splits): curr_elem = split.strip() http://git-wip-us.apache.org/repos/asf/ambari/blob/9d3eeaaa/ambari-server/src/test/python/stacks/2.5/HIVE/running_withMOTDmsg_andTrailingMsg.txt ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/running_withMOTDmsg_andTrailingMsg.txt b/ambari-server/src/test/python/stacks/2.5/HIVE/running_withMOTDmsg_andTrailingMsg.txt new file mode 100644 index 0000000..394faef --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.5/HIVE/running_withMOTDmsg_andTrailingMsg.txt @@ -0,0 +1,46 @@ +######## Hortonworks ############# +This is MOTD message, added for testing in qe infra +{ + "amInfo" : { + "appName" : "llap", + "appType" : "org-apache-slider", + "appId" : "application_1455662455106_10882", + "containerId" : "container_e14_1455662455106_10882_01_000001", + "hostname" : "HOST_REPLACED", + "amWebUrl" : "http://HOST_REPLACED:1025/"; + }, + "state" : "RUNNING_ALL", + "originalConfigurationPath" : "hdfs://HOST_REPLACED:8020/user/USER_REPLACED/.slider/cluster/llap/snapshot", + "generatedConfigurationPath" : "hdfs://HOST_REPLACED:8020/user/USER_REPLACED/.slider/cluster/llap/generated", + "desiredInstances" : 3, + "liveInstances" : 3, + "appStartTime" : 1459625802169, + "llapInstances" : [ { + "hostname" : "HOST_REPLACED", + "containerId" : "container_e14_1455662455106_10882_01_000003", + "statusUrl" : "http://HOST_REPLACED:15002/status";, + "webUrl" : "http://HOST_REPLACED:15002";, + "rpcPort" : 15001, + "mgmtPort" : 15004, + "shufflePort" : 15551 + }, { + "hostname" : "HOST_REPLACED", + "containerId" : "container_e14_1455662455106_10882_01_000002", + "statusUrl" : "http://HOST_REPLACED:15002/status";, + "webUrl" : "http://HOST_REPLACED:15002";, + "rpcPort" : 15001, + "mgmtPort" : 15004, + "shufflePort" : 15551 + }, { + "hostname" : "HOST_REPLACED", + "containerId" : "container_e14_1455662455106_10882_01_000004", + "statusUrl" : "http://HOST_REPLACED:15002/status";, + "webUrl" : "http://HOST_REPLACED:15002";, + "rpcPort" : 15001, + "mgmtPort" : 15004, + "shufflePort" : 15551 + } ] +} + +# THIS IS A DUMMY TRAILING MESSAGE 1 +# THIS IS A DUMMY TRAILING MESSAGE 2 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/9d3eeaaa/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py index cf79ec7..4eb16c2 100644 --- a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py +++ b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py @@ -915,6 +915,27 @@ class TestHiveServerInteractive(RMFTestCase): + # Tests for function '_make_valid_json()' : will be passed in with 'llapstatus' output which will be : + # (1). A string parseable as JSON, but has 2 and 3. + # (2). Has extra lines in beginning (eg: from MOTD logging embedded) + # AND/OR + # (3). Extra lines at the end. + + # Begginning and end lines need to be removed before parsed as JSON + def test_make_valid_json_11(self): + # Setting up input for fn. '_make_valid_json()' + input_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/running_withMOTDmsg_andTrailingMsg.txt","r") + llap_app_info = input_file_handle.read() + llap_app_info_as_json = self.hsi._make_valid_json(llap_app_info) + + # Set up expected output + expected_ouput_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/running.json","r") + expected_ouput_data = expected_ouput_file_handle.read() + expected_ouput_data_as_json = json.loads(expected_ouput_data) + + # Verification + self.assertEqual(llap_app_info_as_json, expected_ouput_data_as_json) + # Tests for fn : 'check_llap_app_status_in_hdp_tp()'
