Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-21674 c3a083826 -> 1374bde24
AMBARI-22394. Store and retrieve the ambari repo urls for os family during bootstrap (Sneha Kanekar via ncole) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1374bde2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1374bde2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1374bde2 Branch: refs/heads/branch-feature-AMBARI-21674 Commit: 1374bde242410c4610c857a4296cd62724baa327 Parents: c3a0838 Author: Nate Cole <[email protected]> Authored: Tue Jan 2 11:56:45 2018 -0500 Committer: Nate Cole <[email protected]> Committed: Tue Jan 2 11:56:45 2018 -0500 ---------------------------------------------------------------------- ambari-server/src/main/python/bootstrap.py | 45 +++++--- ambari-server/src/main/python/os_check_type.py | 4 - ambari-server/src/main/python/setupAgent.py | 35 +++++- ambari-server/src/test/python/TestBootstrap.py | 72 +++++++++++-- ambari-server/src/test/python/TestSetupAgent.py | 107 +++++++++++++++++++ 5 files changed, 233 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1374bde2/ambari-server/src/main/python/bootstrap.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/bootstrap.py b/ambari-server/src/main/python/bootstrap.py index f1c53ce..a1b44b7 100755 --- a/ambari-server/src/main/python/bootstrap.py +++ b/ambari-server/src/main/python/bootstrap.py @@ -36,6 +36,7 @@ import re from datetime import datetime from ambari_commons import OSCheck, OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_server.serverConfiguration import get_ambari_properties, AMBARI_REPO if OSCheck.is_windows_family(): from ambari_commons.os_utils import run_os_command, run_in_shell @@ -407,6 +408,7 @@ class BootstrapDefault(Bootstrap): TEMP_FOLDER = DEFAULT_AGENT_TEMP_FOLDER OS_CHECK_SCRIPT_FILENAME = "os_check_type.py" PASSWORD_FILENAME = "host_pass" + agent_os_type = "" def getRemoteName(self, filename): full_name = os.path.join(self.TEMP_FOLDER, filename) @@ -602,7 +604,7 @@ class BootstrapDefault(Bootstrap): else: return server_port - def getRunSetupWithPasswordCommand(self, expected_hostname): + def getRunSetupWithPasswordCommand(self, expected_hostname, ambariRepoUrl=""): setupFile = self.getRemoteName(self.SETUP_SCRIPT_FILENAME) passphrase = os.environ[AMBARI_PASSPHRASE_VAR_NAME] server = self.shared_state.ambari_server @@ -612,9 +614,9 @@ class BootstrapDefault(Bootstrap): passwordFile = self.getPasswordFile() return "{sudo} -S python ".format(sudo=AMBARI_SUDO) + str(setupFile) + " " + str(expected_hostname) + \ " " + str(passphrase) + " " + str(server)+ " " + quote_bash_args(str(user_run_as)) + " " + str(version) + \ - " " + str(port) + " < " + str(passwordFile) + " " + str(port) + " " + str(ambariRepoUrl) + " < " + str(passwordFile) - def getRunSetupWithoutPasswordCommand(self, expected_hostname): + def getRunSetupWithoutPasswordCommand(self, expected_hostname, ambariRepoUrl=""): setupFile=self.getRemoteName(self.SETUP_SCRIPT_FILENAME) passphrase=os.environ[AMBARI_PASSPHRASE_VAR_NAME] server=self.shared_state.ambari_server @@ -623,7 +625,7 @@ class BootstrapDefault(Bootstrap): port=self.getAmbariPort() return "{sudo} python ".format(sudo=AMBARI_SUDO) + str(setupFile) + " " + str(expected_hostname) + \ " " + str(passphrase) + " " + str(server)+ " " + quote_bash_args(str(user_run_as)) + " " + str(version) + \ - " " + str(port) + " " + str(port) + " " + str(ambariRepoUrl) def runCreatePythonWrapScript(self): params = self.shared_state @@ -643,13 +645,15 @@ class BootstrapDefault(Bootstrap): self.host_log.write("==========================\n") self.host_log.write("Running OS type check...") - command = "chmod a+x %s && %s %s %s" % \ + command = "chmod a+x %s && %s %s" % \ (self.getOsCheckScriptRemoteLocation(), - PYTHON_ENV, self.getOsCheckScriptRemoteLocation(), params.cluster_os_type) + PYTHON_ENV, self.getOsCheckScriptRemoteLocation()) ssh = SSH(params.user, params.sshPort, params.sshkey_file, self.host, command, params.bootdir, self.host_log) retcode = ssh.run() + # log contains agent os_type followed by Connection closed message + self.agent_os_type = retcode["log"].split()[0] self.host_log.write("\n") return retcode @@ -729,20 +733,35 @@ class BootstrapDefault(Bootstrap): self.host_log.write("\n") return retcode - def getRunSetupCommand(self, expected_hostname): + def getRunSetupCommand(self, expected_hostname, ambariRepoUrl=""): if self.hasPassword(): - return self.getRunSetupWithPasswordCommand(expected_hostname) + return self.getRunSetupWithPasswordCommand(expected_hostname, ambariRepoUrl) else: - return self.getRunSetupWithoutPasswordCommand(expected_hostname) + return self.getRunSetupWithoutPasswordCommand(expected_hostname, ambariRepoUrl) def runSetupAgent(self): params = self.shared_state self.host_log.write("==========================\n") self.host_log.write("Running setup agent script...") - command = self.getRunSetupCommand(self.host) - ssh = SSH(params.user, params.sshPort, params.sshkey_file, self.host, command, - params.bootdir, self.host_log) - retcode = ssh.run() + + retcode = None + if params.cluster_os_type == self.agent_os_type: + command = self.getRunSetupCommand(self.host) + ssh = SSH(params.user, params.sshPort, params.sshkey_file, self.host, command, + params.bootdir, self.host_log) + retcode = ssh.run() + else: + properties = get_ambari_properties() + ambariRepoUrl = properties[AMBARI_REPO + '.' + self.agent_os_type] + if ambariRepoUrl: + command = self.getRunSetupCommand(self.host, ambariRepoUrl) + ssh = SSH(params.user, params.sshPort, params.sshkey_file, self.host, command, + params.bootdir, self.host_log) + retcode = ssh.run() + else: + retcode = {"exitstatus": 1, "log": "Ambari repo not found for os_type '{0}'. Please set ambari repo baseurl using command: ambari-server setup --ambari-repo <ambari repo baseurl>.".format(self.agent_os_type), + "errormsg": "Ambari repo not found for os_type '{0}'".format(self.agent_os_type)} + self.host_log.write("\n") return retcode http://git-wip-us.apache.org/repos/asf/ambari/blob/1374bde2/ambari-server/src/main/python/os_check_type.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/os_check_type.py b/ambari-server/src/main/python/os_check_type.py index da37560..a4f0a14 100644 --- a/ambari-server/src/main/python/os_check_type.py +++ b/ambari-server/src/main/python/os_check_type.py @@ -23,10 +23,6 @@ from ambari_commons import OSCheck def main(argv=None): # Same logic that was in "os_type_check.sh" - if len(sys.argv) != 2: - print "Usage: <cluster_os>" - raise Exception("Error in number of arguments. Usage: <cluster_os>") - pass current_os = get_os_type() #log the current os type value to be used during bootstrap http://git-wip-us.apache.org/repos/asf/ambari/blob/1374bde2/ambari-server/src/main/python/setupAgent.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/setupAgent.py b/ambari-server/src/main/python/setupAgent.py index efc3d7c..b0aa7a3 100755 --- a/ambari-server/src/main/python/setupAgent.py +++ b/ambari-server/src/main/python/setupAgent.py @@ -24,6 +24,8 @@ import sys import logging import os import subprocess +import ConfigParser +import re from ambari_commons import OSCheck, OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl @@ -303,6 +305,7 @@ def checkServerReachability(host, port): # 3 User to run agent as # X 4 Project Version (Ambari) # X 5 Server port +# X 6 Ambari Repo Url def parseArguments(argv=None): if argv is None: # make sure that arguments was passed return {"exitstatus": 2, "log": "No arguments were passed"} @@ -316,6 +319,7 @@ def parseArguments(argv=None): user_run_as = args[3] projectVersion = "" server_port = 8080 + ambariRepoUrl = "" if len(args) > 4: projectVersion = args[4] @@ -326,7 +330,10 @@ def parseArguments(argv=None): except (Exception): server_port = 8080 - parsed_args = (expected_hostname, passPhrase, hostname, user_run_as, projectVersion, server_port) + if len(args) > 6: + ambariRepoUrl = args[6] + + parsed_args = (expected_hostname, passPhrase, hostname, user_run_as, projectVersion, server_port, ambariRepoUrl) return {"exitstatus": 0, "log": "", "parsed_args": parsed_args} def run_setup(argv=None): @@ -335,7 +342,7 @@ def run_setup(argv=None): if (retcode["exitstatus"] != 0): return retcode - (expected_hostname, passPhrase, hostname, user_run_as, projectVersion, server_port) = retcode["parsed_args"] + (expected_hostname, passPhrase, hostname, user_run_as, projectVersion, server_port, ambariRepoUrl) = retcode["parsed_args"] retcode = checkServerReachability(hostname, server_port) if (retcode["exitstatus"] != 0): @@ -352,9 +359,27 @@ def run_setup(argv=None): # Verify that the ambari repo file is available before trying to install ambari-agent ambari_repo_file = get_ambari_repo_file_full_name() if os.path.exists(ambari_repo_file): - retcode = installAgent(availableProjectVersion) - if (not retcode["exitstatus"] == 0): - return retcode + if ambariRepoUrl == "": + retcode = installAgent(availableProjectVersion) + if (not retcode["exitstatus"] == 0): + return retcode + else: + #update repo file to use given repo url + if OSCheck.is_suse_family() or OSCheck.is_redhat_family(): + config = ConfigParser.RawConfigParser() + config.read(ambari_repo_file) + baseurl = config.get(config.sections()[0], 'baseurl') + elif OSCheck.is_ubuntu_family(): + for line in open(ambari_repo_file,'r'): + match = re.search('^deb.*$', line) + if match is not None: + baseurl = match.group(0).split()[1] + break + subprocess.call(['sed', '-i', '-e', 's,' + baseurl + ',' + ambariRepoUrl + ',g', ambari_repo_file]) + retcode = installAgent(availableProjectVersion) + if (not retcode["exitstatus"] == 0): + #failed due to new ambari repo url + return retcode else: return {"exitstatus": 2, "log": "Ambari repo file not found: {0}".format(ambari_repo_file)} pass http://git-wip-us.apache.org/repos/asf/ambari/blob/1374bde2/ambari-server/src/test/python/TestBootstrap.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestBootstrap.py b/ambari-server/src/test/python/TestBootstrap.py index bea47f4..2549c9a 100644 --- a/ambari-server/src/test/python/TestBootstrap.py +++ b/ambari-server/src/test/python/TestBootstrap.py @@ -111,9 +111,9 @@ class TestBootstrap(TestCase): bootstrap_obj = Bootstrap("hostname", shared_state) utime = 1234 bootstrap_obj.getUtime = MagicMock(return_value=utime) - ret = bootstrap_obj.getRunSetupWithPasswordCommand("hostname") + ret = bootstrap_obj.getRunSetupWithPasswordCommand("hostname", "ambariRepoUrl") expected = "/var/lib/ambari-agent/tmp/ambari-sudo.sh -S python /var/lib/ambari-agent/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \ - "ambariServer root 8440 < /var/lib/ambari-agent/tmp/host_pass{0}".format(utime) + "ambariServer root 8440 {1} < /var/lib/ambari-agent/tmp/host_pass{0}".format(utime, "ambariRepoUrl") self.assertEquals(ret, expected) @@ -159,8 +159,8 @@ class TestBootstrap(TestCase): "setupAgentFile", "ambariServer", "centos6", version, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) - runSetupCommand = bootstrap_obj.getRunSetupCommand("hostname") - self.assertTrue(runSetupCommand.endswith(version + " 8440")) + runSetupCommand = bootstrap_obj.getRunSetupCommand("hostname", "ambariRepoUrl") + self.assertTrue(runSetupCommand.endswith(version + " 8440 ambariRepoUrl")) def test_agent_setup_command_without_project_version(self): @@ -170,8 +170,8 @@ class TestBootstrap(TestCase): "setupAgentFile", "ambariServer", "centos6", version, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) - runSetupCommand = bootstrap_obj.getRunSetupCommand("hostname") - self.assertTrue(runSetupCommand.endswith(" 8440")) + runSetupCommand = bootstrap_obj.getRunSetupCommand("hostname", "ambariRepoUrl") + self.assertTrue(runSetupCommand.endswith(" 8440 ambariRepoUrl")) # TODO: test_os_check_fail_fails_bootstrap_execution @@ -512,15 +512,16 @@ class TestBootstrap(TestCase): None, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) getOsCheckScriptRemoteLocation_mock.return_value = "OsCheckScriptRemoteLocation" - expected = 42 + expected = {'errormsg': '', 'exitstatus': 0, 'log': 'centos6 Connection to host1 closed.'} init_mock.return_value = None run_mock.return_value = expected res = bootstrap_obj.runOsCheckScript() self.assertEquals(res, expected) command = str(init_mock.call_args[0][4]) + self.assertEqual(res["log"].split()[0], "centos6") self.assertEqual(command, "chmod a+x OsCheckScriptRemoteLocation && " - "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/tmp OsCheckScriptRemoteLocation centos6") + "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/tmp OsCheckScriptRemoteLocation") @patch.object(SSH, "__init__") @@ -533,6 +534,7 @@ class TestBootstrap(TestCase): "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") bootstrap_obj = Bootstrap("hostname", shared_state) + bootstrap_obj.agent_os_type = "centos6" getRunSetupCommand_mock.return_value = "RunSetupCommand" expected = 42 init_mock.return_value = None @@ -542,6 +544,60 @@ class TestBootstrap(TestCase): command = str(init_mock.call_args[0][4]) self.assertEqual(command, "RunSetupCommand") + @patch.object(SSH, "__init__") + @patch.object(BootstrapDefault, "getRunSetupCommand") + @patch.object(SSH, "run") + @patch.object(HostLog, "write") + @patch("os.environ") + def test_runSetupAgentForAmbariRepoUrl(self, environ_mock, write_mock, run_mock, getRunSetupCommand_mock, init_mock,): + environ_mock.__getitem__.return_value = "/tmp/" + getRunSetupCommand_mock.return_value = "RunSetupCommand" + run_mock.return_value = {'log': 'log', 'exitstatus': 0} + init_mock.return_value = None + + if not os.path.exists('/tmp/ambari.properties'): + os.mknod('/tmp/ambari.properties') + + #If cluster_os and agent_os type are different and ambariRepoUrl is found in property file + shared_state = SharedState("root", "123", "sshkey_file", "scriptDir", "bootdir", + "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") + bootstrap_obj = Bootstrap("hostname", shared_state) + bootstrap_obj.agent_os_type = "redhat-ppc7" + f = open("/tmp/ambari.properties","a") + f.write("ambari.repo." + bootstrap_obj.agent_os_type + "=ambariRepoUrl") + f.close() + expected = {'log': 'log', 'exitstatus': 0} + run_mock.return_value = {'log': 'log', 'exitstatus': 0} + res = bootstrap_obj.runSetupAgent() + self.assertEqual(res,expected) + + f = open("/tmp/ambari.properties","r") + lines = f.readlines() + f.close() + f = open("/tmp/ambari.properties","w") + for line in lines: + if not "ambari.repo." + bootstrap_obj.agent_os_type in line: + f.write(line) + f.close() + + #If cluster_os and agent_os type are different and ambariRepoUrl is not found in property file + shared_state = SharedState("root", "123", "sshkey_file", "scriptDir", "bootdir", + "setupAgentFile", "ambariServer", "centos6", None, "8440", "root") + bootstrap_obj = Bootstrap("hostname", shared_state) + bootstrap_obj.agent_os_type = "redhat-ppc7" + expected = {"errormsg": "Ambari repo not found for os_type '{0}'".format(bootstrap_obj.agent_os_type), "exitstatus": 1, "log": "Ambari repo not found for os_type '{0}'. Please set ambari repo baseurl using command: ambari-server setup --ambari-repo <ambari repo baseurl>.".format(bootstrap_obj.agent_os_type)} + run_mock.return_value = {'log': 'log', 'exitstatus': 0} + res = bootstrap_obj.runSetupAgent() + self.assertEqual(res,expected) + + #If cluster_os and agent_os type are same + shared_state = SharedState("root", "123", "sshkey_file", "scriptDir", "bootdir", + "setupAgentFile", "ambariServer", "redhat-ppc7", None, "8440", "root") + bootstrap_obj = Bootstrap("hostname", shared_state) + bootstrap_obj.agent_os_type = "redhat-ppc7" + expected = {'log': 'log', 'exitstatus': 0} + res = bootstrap_obj.runSetupAgent() + self.assertEqual(res,expected) @patch.object(BootstrapDefault, "hasPassword") @patch.object(BootstrapDefault, "getRunSetupWithPasswordCommand") http://git-wip-us.apache.org/repos/asf/ambari/blob/1374bde2/ambari-server/src/test/python/TestSetupAgent.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestSetupAgent.py b/ambari-server/src/test/python/TestSetupAgent.py index 0b33e9f..64faf85 100644 --- a/ambari-server/src/test/python/TestSetupAgent.py +++ b/ambari-server/src/test/python/TestSetupAgent.py @@ -20,6 +20,9 @@ from mock.mock import MagicMock from unittest import TestCase from mock.mock import patch import sys +import ConfigParser +import os +import re from ambari_commons import OSCheck from only_for_platform import get_platform, not_for_platform, only_for_platform, os_distro_value, PLATFORM_WINDOWS @@ -59,6 +62,13 @@ class TestSetupAgent(TestCase): pass + def test_parse_argument(self,): + arguments=["", "test.hst", "passphrase", "test", "root", "", 8080, "ambariRepoUrl"] + ret=setup_agent.parseArguments(arguments) + parsed_args = ("test.hst", "passphrase", "test", "root", "", 8080, "ambariRepoUrl") + self.assertEqual(ret["exitstatus"], 0) + self.assertEqual(ret["parsed_args"], parsed_args) + @not_for_platform(PLATFORM_WINDOWS) @patch.object(setup_agent, 'execOsCommand') @patch("os.environ") @@ -524,6 +534,103 @@ class TestSetupAgent(TestCase): self.assertTrue("exitstatus" in ret) self.assertEqual(ret["exitstatus"], 1) self.assertTrue(get_ambari_repo_file_full_name_mock() in ret["log"]) # ambari repo file not found message + + # ambari repo base url passed for setting in repo file + def deleteDummyRepo(): + if os.path.isfile('/tmp/dummy.repo'): + os.remove('/tmp/dummy.repo') + + def createDummyRepo(): + if not os.path.isfile('/tmp/dummy.repo'): + os.mknod('/tmp/dummy.repo') + fn = open('/tmp/dummy.repo','w+') + fn.write("[AMBARI.2.4.2.4-2.x]\n") + fn.write("baseurl=repoUrl") + fn.close() + + # If agent installation is successful + createDummyRepo() + installAgent_mock.reset_mock() + getOptimalVersion_mock.reset_mock() + os_path_exists_mock.reset_mock() + installAgent_mock.return_value = {'log': 'log', 'exitstatus': 0} + getOptimalVersion_mock.return_value = {'log': '1.1.1', 'exitstatus': 0} + os_path_exists_mock.return_value = True + get_ambari_repo_file_full_name_mock.return_value = "/tmp/dummy.repo" + ret = setup_agent.main(("setupAgent.py","agents_host","password","server_hostname","root","1.1.1","8080","ambariRepoUrl")) + self.assertTrue(installAgent_mock.called) + self.assertTrue("exitstatus" in ret) + self.assertEqual(ret["exitstatus"], 0) + config = ConfigParser.RawConfigParser() + config.read("/tmp/dummy.repo") + baseurl = config.get(config.sections()[0],'baseurl') + self.assertEqual(baseurl, "ambariRepoUrl") + + # If agent installation fails + exit_mock.reset_mock() + installAgent_mock.reset_mock() + getOptimalVersion_mock.reset_mock() + os_path_exists_mock.reset_mock() + installAgent_mock.return_value = {'log': 'log', 'exitstatus': 1} + getOptimalVersion_mock.return_value = {'log': '1.1.1', 'exitstatus': 0} + os_path_exists_mock.return_value = True + get_ambari_repo_file_full_name_mock.return_value = "/tmp/dummy.repo" + ret = setup_agent.main(("setupAgent.py","agents_host","password","server_hostname","root","1.1.1","8080","ambariRepoUrl")) + self.assertTrue(installAgent_mock.called) + self.assertTrue("exitstatus" in ret) + self.assertEqual(ret["exitstatus"], 1) + deleteDummyRepo() + + #if os_type is ubuntu + is_suse_family_mock.return_value = False + is_ubuntu_family_mock.return_value = True + + def deleteDummyRepoUbuntu(): + if os.path.isfile('/tmp/dummy.list'): + os.remove('/tmp/dummy.list') + + def createDummyRepoUbuntu(): + if not os.path.isfile('/tmp/dummy.list'): + os.mknod('/tmp/dummy.list') + fn = open('/tmp/dummy.list','wt') + fn.write("#VERSION_NUMBER=2.4.2.4-2.x]\n") + fn.write("deb repoUrl Ambari main") + fn.close() + + # If agent installation is successful + createDummyRepoUbuntu() + installAgent_mock.reset_mock() + getOptimalVersion_mock.reset_mock() + os_path_exists_mock.reset_mock() + installAgent_mock.return_value = {'log': 'log', 'exitstatus': 0} + getOptimalVersion_mock.return_value = {'log': '1.1.1', 'exitstatus': 0} + os_path_exists_mock.return_value = True + get_ambari_repo_file_full_name_mock.return_value = "/tmp/dummy.list" + ret = setup_agent.main(("setupAgent.py","agents_host","password","server_hostname","root","1.1.1","8080","ambariRepoUrl")) + self.assertTrue(installAgent_mock.called) + self.assertTrue("exitstatus" in ret) + self.assertEqual(ret["exitstatus"], 0) + for line in open('/tmp/dummy.list','r'): + match = re.search('^deb.*$', line) + if match is not None: + baseurl = match.group(0).split()[1] + break + self.assertEqual(baseurl, "ambariRepoUrl") + + # If agent installation fails + exit_mock.reset_mock() + installAgent_mock.reset_mock() + getOptimalVersion_mock.reset_mock() + os_path_exists_mock.reset_mock() + installAgent_mock.return_value = {'log': 'log', 'exitstatus': 1} + getOptimalVersion_mock.return_value = {'log': '1.1.1', 'exitstatus': 0} + os_path_exists_mock.return_value = True + get_ambari_repo_file_full_name_mock.return_value = "/tmp/dummy.list" + ret = setup_agent.main(("setupAgent.py","agents_host","password","server_hostname","root","1.1.1","8080","ambariRepoUrl")) + self.assertTrue(installAgent_mock.called) + self.assertTrue("exitstatus" in ret) + self.assertEqual(ret["exitstatus"], 1) + deleteDummyRepoUbuntu() pass @patch.object(setup_agent, 'execOsCommand')
