Updated Branches: refs/heads/trunk 66d717a03 -> 59076a55f
AMBARI-3180. Helper script for HDP upgrade (update to script and unit test). Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/59076a55 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/59076a55 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/59076a55 Branch: refs/heads/trunk Commit: 59076a55fe2e149a4a7018e42d400b56265ae98f Parents: 66d717a Author: Sumit Mohanty <[email protected]> Authored: Wed Sep 11 11:32:29 2013 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Wed Sep 11 11:32:29 2013 -0700 ---------------------------------------------------------------------- .../src/main/python/UpgradeHelper_HDP2.py | 43 +++--- .../src/test/python/TestUpgradeScript_HDP2.py | 141 +++++++++---------- 2 files changed, 94 insertions(+), 90 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/59076a55/ambari-server/src/main/python/UpgradeHelper_HDP2.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/UpgradeHelper_HDP2.py b/ambari-server/src/main/python/UpgradeHelper_HDP2.py index 82974cc..20db21f 100644 --- a/ambari-server/src/main/python/UpgradeHelper_HDP2.py +++ b/ambari-server/src/main/python/UpgradeHelper_HDP2.py @@ -349,7 +349,7 @@ def get_mr1_mapping(options): hostmapping = {} for component in components: hostlist = [] - response = curl('-u', + response = curl(False, '-u', AUTH_FORMAT.format(options.user, options.password), GET_URL_FORMAT.format(options.hostname, options.clustername, component)) retcode, errdata = validate_response(response, True) @@ -405,7 +405,7 @@ def delete_mr(options): for key, value in hostmapping.items(): if (key in NON_CLIENTS) and (len(value) > 0): for host in value: - response = curl('-u', + response = curl(options.printonly, '-u', AUTH_FORMAT.format(options.user, options.password), '-X', 'PUT', '-d', PUT_IN_MAINTENANCE, @@ -418,7 +418,7 @@ def delete_mr(options): pass pass - response = curl('-u', + response = curl(options.printonly, '-u', AUTH_FORMAT.format(options.user, options.password), '-X', 'DELETE', SERVICE_URL_FORMAT.format(options.hostname, options.clustername)) @@ -444,7 +444,7 @@ def add_services(options): hostmapping = read_mapping() for service in service_comp.keys(): - response = curl('-u', + response = curl(options.printonly, '-u', AUTH_FORMAT.format(options.user, options.password), '-X', 'POST', SERVICE_URL_FORMAT.format(options.hostname, options.clustername, service)) @@ -452,7 +452,7 @@ def add_services(options): if not retcode == 0: raise FatalException(retcode, errdata) for component in service_comp[service]: - response = curl('-u', + response = curl(options.printonly, '-u', AUTH_FORMAT.format(options.user, options.password), '-X', 'POST', COMPONENT_URL_FORMAT.format(options.hostname, options.clustername, service, component)) @@ -460,7 +460,7 @@ def add_services(options): if not retcode == 0: raise FatalException(retcode, errdata) for host in hostmapping[new_old_host_map[component]]: - response = curl('-u', + response = curl(options.printonly, '-u', AUTH_FORMAT.format(options.user, options.password), '-X', 'POST', HOST_COMPONENT_URL_FORMAT.format(options.hostname, options.clustername, host, component)) @@ -476,7 +476,7 @@ def add_services(options): def update_config(options, properties, type): tag = "version" + str(int(time.time() * 1000)) properties_payload = {"Clusters": {"desired_config": {"type": type, "tag": tag, "properties": properties}}} - response = curl('-u', + response = curl(options.printonly, '-u', AUTH_FORMAT.format(options.user, options.password), '-X', 'PUT', '-d', json.dumps(properties_payload), @@ -503,7 +503,7 @@ def get_config(options, type): def get_config_resp(options, type, error_if_na=True): CONFIG_URL_FORMAT = URL_FORMAT + '/configurations?type={2}&tag={3}' - response = curl('-u', + response = curl(False, '-u', AUTH_FORMAT.format(options.user, options.password), URL_FORMAT.format(options.hostname, options.clustername)) retcode, errdata = validate_response(response, True) @@ -519,7 +519,7 @@ def get_config_resp(options, type, error_if_na=True): if tag != None: # Get the config with the tag and return properties - response = curl('-u', + response = curl(False, '-u', AUTH_FORMAT.format(options.user, options.password), CONFIG_URL_FORMAT.format(options.hostname, options.clustername, type, tag)) retcode, errdata = validate_response(response, True) @@ -638,14 +638,14 @@ def install_services(options): SERVICE_URL_FORMAT = URL_FORMAT + '/services?ServiceInfo/state=INIT' PUT_IN_INSTALLED = """{"ServiceInfo": {"state": "INSTALLED"}}""" - response = curl('-u', + response = curl(options.printonly, '-u', AUTH_FORMAT.format(options.user, options.password), '-X', 'PUT', '-d', PUT_IN_INSTALLED, SERVICE_URL_FORMAT.format(options.hostname, options.clustername)) - retcode, errdata = validate_response(response, True) + retcode, errdata = validate_response(response, not options.printonly) if not retcode == 0: - raise FatalException(retcode, errdata) + raise FatalException(retcode, errdata + "(Services may already be installed.)") pass @@ -662,17 +662,26 @@ def validate_response(response, expect_body): pass -def curl(*args): +def curl(print_only, *args): curl_path = '/usr/bin/curl' curl_list = [curl_path] for arg in args: curl_list.append(arg) + if print_only: + logger.info("Command to be executed: " + ' '.join(curl_list)) + return "" + pass logger.info(' '.join(curl_list)) - curl_result = subprocess.Popen( + osStat = subprocess.Popen( curl_list, stderr=subprocess.PIPE, - stdout=subprocess.PIPE).communicate()[0] - return curl_result + stdout=subprocess.PIPE) + out, err = osStat.communicate() + if 0 != osStat.returncode: + error = "curl call failed. out: " + out + " err: " + err + logger.error(error) + raise FatalException(osStat.returncode, error) + return out # # Main. @@ -683,7 +692,7 @@ def main(): parser.add_option("-n", "--printonly", action="store_true", dest="printonly", default=False, - help="Prints all the curl commands to be executed") + help="Prints all the curl commands to be executed (only for write/update actions)") parser.add_option("-o", "--log", dest="logfile", default=UPGRADE_LOG_FILE, help="Log file") http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/59076a55/ambari-server/src/test/python/TestUpgradeScript_HDP2.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestUpgradeScript_HDP2.py b/ambari-server/src/test/python/TestUpgradeScript_HDP2.py index 3010eec..e7b9446 100644 --- a/ambari-server/src/test/python/TestUpgradeScript_HDP2.py +++ b/ambari-server/src/test/python/TestUpgradeScript_HDP2.py @@ -93,10 +93,9 @@ class TestUpgradeHDP2Script(TestCase): opm = option_parser_mock.return_value path_exists_mock.return_value = True shutil_copy_mock = MagicMock() - options = MagicMock() + options = self.get_mock_options() args = ["backup-configs"] opm.parse_args.return_value = (options, args) - options.logfile = "logfile" def get_config_resp_side_effect(ops, type, error_if_na): if type == "global": @@ -138,23 +137,18 @@ class TestUpgradeHDP2Script(TestCase): backup_file_mock, file_handler_mock): file_handler_mock.return_value = logging.FileHandler('') # disable creating real file opm = option_parser_mock.return_value - options = MagicMock() + options = self.get_mock_options() args = ["save-mr-mapping"] opm.parse_args.return_value = (options, args) - options.logfile = "logfile" - options.user = "admin" - options.password = "admin" - options.hostname = "localhost" - options.clustername = "c1" curl_mock.side_effect = ['"href" : "', '"href" : "', '"href" : "'] json_loads_mock.return_value = {"host_components": [{"HostRoles": {"host_name": "host1"}}]} UpgradeHelper_HDP2.main() expected_curl_calls = [ - call("-u", "admin:admin", + call(False, "-u", "admin:admin", "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/MAPREDUCE_CLIENT";), - call("-u", "admin:admin", + call(False, "-u", "admin:admin", "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/TASKTRACKER";), - call("-u", "admin:admin", + call(False, "-u", "admin:admin", "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/JOBTRACKER";)] curl_mock.assert_has_calls(expected_curl_calls, any_order=True) self.assertTrue(write_mapping_mock.called) @@ -177,14 +171,9 @@ class TestUpgradeHDP2Script(TestCase): backup_file_mock, file_handler_mock, read_mapping_mock, get_yn_mock): file_handler_mock.return_value = logging.FileHandler('') # disable creating real file opm = option_parser_mock.return_value - options = MagicMock() + options = self.get_mock_options() args = ["delete-mr"] opm.parse_args.return_value = (options, args) - options.logfile = "logfile" - options.user = "admin" - options.password = "admin" - options.hostname = "localhost" - options.clustername = "c1" curl_mock.return_value = '' get_yn_mock.return_value = True read_mapping_mock.return_value = { @@ -193,13 +182,13 @@ class TestUpgradeHDP2Script(TestCase): "MAPREDUCE_CLIENT": ["c6401"]} UpgradeHelper_HDP2.main() expected_curl_calls = [ - call("-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""", + call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""", "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/TASKTRACKER";), - call("-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""", + call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""", "http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/TASKTRACKER";), - call("-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""", + call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""", "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/JOBTRACKER";), - call("-u", "admin:admin", "-X", "DELETE", + call(False, "-u", "admin:admin", "-X", "DELETE", "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE";)] curl_mock.assert_has_calls(expected_curl_calls, any_order=True) pass @@ -214,14 +203,9 @@ class TestUpgradeHDP2Script(TestCase): backup_file_mock, file_handler_mock, read_mapping_mock): file_handler_mock.return_value = logging.FileHandler('') # disable creating real file opm = option_parser_mock.return_value - options = MagicMock() + options = self.get_mock_options() args = ["add-yarn-mr2"] opm.parse_args.return_value = (options, args) - options.logfile = "logfile" - options.user = "admin" - options.password = "admin" - options.hostname = "localhost" - options.clustername = "c1" curl_mock.return_value = '' read_mapping_mock.return_value = { "TASKTRACKER": ["c6401", "c6402"], @@ -229,31 +213,31 @@ class TestUpgradeHDP2Script(TestCase): "MAPREDUCE_CLIENT": ["c6403"]} UpgradeHelper_HDP2.main() expected_curl_calls = [ - call("-u", "admin:admin", "-X", "POST", + call(False, "-u", "admin:admin", "-X", "POST", "http://localhost:8080/api/v1/clusters/c1/services/YARN";), - call("-u", "admin:admin", "-X", "POST", + call(False, "-u", "admin:admin", "-X", "POST", "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2";), - call("-u", "admin:admin", "-X", "POST", + call(False, "-u", "admin:admin", "-X", "POST", "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/HISTORYSERVER";), - call("-u", "admin:admin", "-X", "POST", + call(False, "-u", "admin:admin", "-X", "POST", "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT";), - call("-u", "admin:admin", "-X", "POST", + call(False, "-u", "admin:admin", "-X", "POST", "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/NODEMANAGER";), - call("-u", "admin:admin", "-X", "POST", + call(False, "-u", "admin:admin", "-X", "POST", "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/YARN_CLIENT";), - call("-u", "admin:admin", "-X", "POST", + call(False, "-u", "admin:admin", "-X", "POST", "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/RESOURCEMANAGER";), - call("-u", "admin:admin", "-X", "POST", + call(False, "-u", "admin:admin", "-X", "POST", "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/HISTORYSERVER";), - call("-u", "admin:admin", "-X", "POST", + call(False, "-u", "admin:admin", "-X", "POST", "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/NODEMANAGER";), - call("-u", "admin:admin", "-X", "POST", + call(False, "-u", "admin:admin", "-X", "POST", "http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/NODEMANAGER";), - call("-u", "admin:admin", "-X", "POST", + call(False, "-u", "admin:admin", "-X", "POST", "http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/YARN_CLIENT";), - call("-u", "admin:admin", "-X", "POST", + call(False, "-u", "admin:admin", "-X", "POST", "http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT";), - call("-u", "admin:admin", "-X", "POST", + call(False, "-u", "admin:admin", "-X", "POST", "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER";)] curl_mock.assert_has_calls(expected_curl_calls, any_order=True) pass @@ -267,18 +251,13 @@ class TestUpgradeHDP2Script(TestCase): backup_file_mock, file_handler_mock): file_handler_mock.return_value = logging.FileHandler('') # disable creating real file opm = option_parser_mock.return_value - options = MagicMock() + options = self.get_mock_options() args = ["install-yarn-mr2"] opm.parse_args.return_value = (options, args) - options.logfile = "logfile" - options.user = "admin" - options.password = "admin" - options.hostname = "localhost" - options.clustername = "c1" curl_mock.return_value = '"href" : "' UpgradeHelper_HDP2.main() expected_curl_calls = [ - call("-u", "admin:admin", "-X", "PUT", "-d", + call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"ServiceInfo": {"state": "INSTALLED"}}""", "http://localhost:8080/api/v1/clusters/c1/services?ServiceInfo/state=INIT";)] curl_mock.assert_has_calls(expected_curl_calls, any_order=True) @@ -353,22 +332,31 @@ class TestUpgradeHDP2Script(TestCase): self.validate_config_replacememt(curl_mock.call_args_list[4], "global") pass + @patch.object(UpgradeHelper_HDP2, "read_mapping") + @patch("subprocess.Popen") + @patch.object(UpgradeHelper_HDP2, "get_YN_input") @patch.object(logging, 'FileHandler') @patch.object(UpgradeHelper_HDP2, "backup_file") @patch.object(UpgradeHelper_HDP2, 'curl') @patch('optparse.OptionParser') - def test_get_and_parse_properties(self, option_parser_mock, curl_mock, + def test_print_only(self, option_parser_mock, curl_mock, + backup_file_mock, file_handler_mock, get_yn_mock, popen_mock, read_mapping_mock): + file_handler_mock.return_value = logging.FileHandler('') # disable creating real file + options = self.get_mock_options(True) + get_yn_mock.return_value = True + read_mapping_mock.return_value = { + "TASKTRACKER": ["c6401", "c6402"]} + UpgradeHelper_HDP2.delete_mr(options) + self.assertFalse(popen_mock.called) + pass + + @patch.object(logging, 'FileHandler') + @patch.object(UpgradeHelper_HDP2, "backup_file") + @patch.object(UpgradeHelper_HDP2, 'curl') + def test_get_and_parse_properties(self, curl_mock, backup_file_mock, file_handler_mock): file_handler_mock.return_value = logging.FileHandler('') # disable creating real file - opm = option_parser_mock.return_value - options = MagicMock() - args = ["update-configs"] - opm.parse_args.return_value = (options, args) - options.logfile = "logfile" - options.user = "admin" - options.password = "admin" - options.hostname = "localhost" - options.clustername = "c1" + options = self.get_mock_options() curl_mock.side_effect = [ """{ "href" : "http://localhost:8080/api/v1/clusters/c1";, "Clusters" : { @@ -420,32 +408,39 @@ class TestUpgradeHDP2Script(TestCase): def validate_update_config_call(self, call, type): args, kargs = call - self.assertTrue(args[3] == 'PUT') - self.assertTrue(type in args[5]) + self.assertTrue(args[4] == 'PUT') + self.assertTrue(type in args[6]) pass def validate_config_replacememt(self, call, type): args, kargs = call - self.assertFalse("REPLACE_WITH_" in args[5]) - self.assertFalse("REPLACE_JH_HOST" in args[5]) - self.assertFalse("REPLACE_RM_HOST" in args[5]) + self.assertFalse("REPLACE_WITH_" in args[6]) + self.assertFalse("REPLACE_JH_HOST" in args[6]) + self.assertFalse("REPLACE_RM_HOST" in args[6]) if type == "yarn-site": - self.assertTrue("c6401" in args[5]) - self.assertFalse("an_old_value" in args[5]) + self.assertTrue("c6401" in args[6]) + self.assertFalse("an_old_value" in args[6]) elif type == "mapred-site": - self.assertTrue("an_old_value" in args[5]) + self.assertTrue("an_old_value" in args[6]) elif type == "global": - self.assertTrue("global11" in args[5]) - self.assertTrue("an_old_value" in args[5]) - self.assertTrue("mapred.hosts.exclude" in args[5]) + self.assertTrue("global11" in args[6]) + self.assertTrue("an_old_value" in args[6]) + self.assertTrue("mapred.hosts.exclude" in args[6]) elif (type == "core-site") or (type == "hdfs-site"): - self.assertTrue("global11" in args[5]) - self.assertFalse("an_old_value" in args[5]) - self.assertFalse("mapred.hosts.exclude" in args[5]) + self.assertTrue("global11" in args[6]) + self.assertFalse("an_old_value" in args[6]) + self.assertFalse("mapred.hosts.exclude" in args[6]) pass - def test_read_config(self): - pass + def get_mock_options(self, printonly=False): + options = MagicMock() + options.logfile = "logfile" + options.user = "admin" + options.password = "admin" + options.hostname = "localhost" + options.clustername = "c1" + options.printonly = printonly + return options if __name__ == "__main__":
