Repository: ambari Updated Branches: refs/heads/trunk 0fc4e7f0e -> 5d07646ee
AMBARI-8476. Run sudo commands as sudo and sudo su ability in Resource Management (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5d07646e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5d07646e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5d07646e Branch: refs/heads/trunk Commit: 5d07646eedc1965def6c1b81d9bcea8b5ddc5673 Parents: 0fc4e7f Author: Andrew Onishuk <[email protected]> Authored: Sat Nov 29 00:29:08 2014 +0200 Committer: Andrew Onishuk <[email protected]> Committed: Sat Nov 29 00:29:08 2014 +0200 ---------------------------------------------------------------------- .../resource_management/TestCopyFromLocal.py | 8 +- .../resource_management/TestExecuteResource.py | 4 +- .../resource_management/TestGroupResource.py | 10 +-- .../resource_management/TestPackageResource.py | 24 +++--- .../resource_management/TestUserResource.py | 22 ++--- .../core/providers/package/apt.py | 22 ++--- .../core/providers/package/yumrpm.py | 13 +-- .../core/providers/package/zypper.py | 15 ++-- .../core/providers/system.py | 1 + .../core/resources/system.py | 15 +++- .../python/resource_management/core/shell.py | 86 ++++++++++++++++---- .../python/resource_management/core/sudo.py | 2 +- .../libraries/providers/copy_from_local.py | 5 +- 13 files changed, 149 insertions(+), 78 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py b/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py index 32a7963..3864a7c 100644 --- a/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py +++ b/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py @@ -17,9 +17,11 @@ limitations under the License. ''' from unittest import TestCase -from mock.mock import patch +from mock.mock import patch, MagicMock from resource_management import * +from resource_management.core import shell [email protected](shell, "call", new = MagicMock(return_value=(1, ""))) @patch.object(System, "os_family", new = 'redhat') class TestCopyFromLocal(TestCase): @@ -36,7 +38,7 @@ class TestCopyFromLocal(TestCase): call_arg_list = execute_hadoop_mock.call_args_list self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/', call_arg_list[0][0][0].command) - self.assertEquals({'not_if': "su - user1 -c ' export PATH=$PATH:/usr/bin ; hadoop fs -ls /apps/test//*.files' >/dev/null 2>&1", 'user': 'user1', 'bin_dir': '/usr/bin', 'conf_dir': '/etc/hadoop/conf'}, + self.assertEquals({'not_if': "/usr/bin/sudo -Hi su - user1 -s /bin/bash -c 'export PATH=/usr/bin ; {kinnit_if_needed} ; hadoop fs -ls {dest_path}'", 'user': 'user1', 'bin_dir': '/usr/bin', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[0][0][0].arguments) self.assertEquals('fs -chown user1 /apps/test//*.files', call_arg_list[1][0][0].command) self.assertEquals({'user': 'hdfs', 'bin_dir': '/usr/bin', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments) @@ -57,7 +59,7 @@ class TestCopyFromLocal(TestCase): call_arg_list = execute_hadoop_mock.call_args_list self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/', call_arg_list[0][0][0].command) - self.assertEquals({'not_if': "su - user1 -c ' export PATH=$PATH:/usr/bin ; hadoop fs -ls /apps/test//*.files' >/dev/null 2>&1", 'user': 'user1', 'bin_dir': '/usr/bin', 'conf_dir': '/etc/hadoop/conf'}, + self.assertEquals({'not_if': "/usr/bin/sudo -Hi su - user1 -s /bin/bash -c 'export PATH=/usr/bin ; {kinnit_if_needed} ; hadoop fs -ls {dest_path}'", 'user': 'user1', 'bin_dir': '/usr/bin', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[0][0][0].arguments) self.assertEquals('fs -chown user1:hdfs /apps/test//*.files', call_arg_list[1][0][0].command) self.assertEquals({'user': 'hdfs', 'bin_dir': '/usr/bin', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments) http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-agent/src/test/python/resource_management/TestExecuteResource.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestExecuteResource.py b/ambari-agent/src/test/python/resource_management/TestExecuteResource.py index 7820e47..b2af4db 100644 --- a/ambari-agent/src/test/python/resource_management/TestExecuteResource.py +++ b/ambari-agent/src/test/python/resource_management/TestExecuteResource.py @@ -176,8 +176,8 @@ class TestExecuteResource(TestCase): environment={'JAVA_HOME': '/test/java/home', 'PATH': "/bin"} ) - expected_command = '/usr/bin/sudo -Hsu test_user <<< \'export PATH=' + os.environ['PATH'] + ':/bin JAVA_HOME=/test/java/home; echo "1"\'' - self.assertEqual(popen_mock.call_args_list[0][0][0][3], expected_command) + expected_command = ['/usr/bin/sudo', '-Hi', 'su', '-', 'test_user', '-s', '/bin/bash', '-c', 'export PATH=' + os.environ['PATH'] + ':/bin JAVA_HOME=/test/java/home ; echo "1"'] + self.assertEqual(popen_mock.call_args_list[0][0][0], expected_command) @patch.object(subprocess, "Popen") http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-agent/src/test/python/resource_management/TestGroupResource.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestGroupResource.py b/ambari-agent/src/test/python/resource_management/TestGroupResource.py index 4245fdf..f80f78a 100644 --- a/ambari-agent/src/test/python/resource_management/TestGroupResource.py +++ b/ambari-agent/src/test/python/resource_management/TestGroupResource.py @@ -45,7 +45,7 @@ class TestGroupResource(TestCase): self.assertEqual(popen_mock.call_count, 1) - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'groupadd -p secure hadoop'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi groupadd -p secure hadoop"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) getgrnam_mock.assert_called_with('hadoop') @@ -66,7 +66,7 @@ class TestGroupResource(TestCase): self.assertEqual(popen_mock.call_count, 1) - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'groupmod -p secure -g 2 mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi groupmod -p secure -g 2 mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) getgrnam_mock.assert_called_with('mapred') @@ -90,7 +90,7 @@ class TestGroupResource(TestCase): except Fail: pass self.assertEqual(popen_mock.call_count, 1) - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'groupmod -p secure -g 2 mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi groupmod -p secure -g 2 mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) getgrnam_mock.assert_called_with('mapred') @@ -110,7 +110,7 @@ class TestGroupResource(TestCase): self.assertEqual(popen_mock.call_count, 1) - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'groupdel mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'groupdel mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) getgrnam_mock.assert_called_with('mapred') @@ -134,5 +134,5 @@ class TestGroupResource(TestCase): pass self.assertEqual(popen_mock.call_count, 1) - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'groupdel mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'groupdel mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) getgrnam_mock.assert_called_with('mapred') http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-agent/src/test/python/resource_management/TestPackageResource.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestPackageResource.py b/ambari-agent/src/test/python/resource_management/TestPackageResource.py index c762355..95e053f 100644 --- a/ambari-agent/src/test/python/resource_management/TestPackageResource.py +++ b/ambari-agent/src/test/python/resource_management/TestPackageResource.py @@ -36,13 +36,13 @@ class TestPackageResource(TestCase): Package("some_package", ) call_mock.assert_has_calls([call("dpkg --get-selections | grep ^some-package$ | grep -v deinstall"), - call("DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get -q -o Dpkg::Options::='--force-confdef'" - " --allow-unauthenticated --assume-yes install some-package", sudo=True), - call("apt-get update -qq", sudo=True) + call(['/usr/bin/apt-get', '-q', '-o', "Dpkg::Options::='--force-confdef'", '--allow-unauthenticated', '--assume-yes', 'install', 'some-package'], + sudo=True, env={'DEBIAN_FRONTEND': 'noninteractive'}), + call(['/usr/bin/apt-get', 'update', '-qq'], sudo=True) ]) - shell_mock.assert_has_calls([call("DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get -q -o Dpkg::Options::='--force-confdef' --allow-unauthenticated --assume-yes install some-package", sudo=True) - ]) + shell_mock.assert_has_calls([call(['/usr/bin/apt-get', '-q', '-o', "Dpkg::Options::='--force-confdef'", '--allow-unauthenticated', '--assume-yes', 'install', + 'some-package'], sudo=True)]) @patch.object(shell, "call") @patch.object(shell, "checked_call") @@ -53,8 +53,8 @@ class TestPackageResource(TestCase): Package("some_package", ) call_mock.assert_has_calls([call("dpkg --get-selections | grep ^some-package$ | grep -v deinstall"), - call("DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get -q -o Dpkg::Options::='--force-confdef'" - " --allow-unauthenticated --assume-yes install some-package", sudo=True) + call(['/usr/bin/apt-get', '-q', '-o', "Dpkg::Options::='--force-confdef'", '--allow-unauthenticated', '--assume-yes', 'install', 'some-package'], + sudo=True, env={'DEBIAN_FRONTEND': 'noninteractive'}) ]) self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called") @@ -70,7 +70,7 @@ class TestPackageResource(TestCase): Package("some_package", ) call_mock.assert_called_with('installed_pkgs=`rpm -qa some_package` ; [ ! -z "$installed_pkgs" ]') - shell_mock.assert_called_with("/usr/bin/yum -d 0 -e 0 -y install some_package", sudo=True) + shell_mock.assert_called_with(['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'install', 'some_package'], sudo=True) @patch.object(shell, "call") @patch.object(shell, "checked_call") @@ -81,7 +81,7 @@ class TestPackageResource(TestCase): Package("some_package", ) call_mock.assert_called_with('rpm -qa | grep ^some_package') - shell_mock.assert_called_with("/usr/bin/zypper --quiet install --auto-agree-with-licenses --no-confirm some_package", sudo=True) + shell_mock.assert_called_with(['/usr/bin/zypper', '--quiet', 'install', '--auto-agree-with-licenses', '--no-confirm', 'some_package'], sudo=True) @patch.object(shell, "call", new = MagicMock(return_value=(0, None))) @patch.object(shell, "checked_call") @@ -109,7 +109,7 @@ class TestPackageResource(TestCase): Package("some_package", action = "remove" ) - shell_mock.assert_called_with("/usr/bin/yum -d 0 -e 0 -y erase some_package", sudo=True) + shell_mock.assert_called_with(['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'erase', 'some_package'], sudo=True) @patch.object(shell, "call", new = MagicMock(return_value=(0, None))) @patch.object(shell, "checked_call") @@ -119,7 +119,7 @@ class TestPackageResource(TestCase): Package("some_package", action = "remove" ) - shell_mock.assert_called_with("/usr/bin/zypper --quiet remove --no-confirm some_package", sudo=True) + shell_mock.assert_called_with(['/usr/bin/zypper', '--quiet', 'remove', '--no-confirm', 'some_package'], sudo=True) @patch.object(shell, "call", new = MagicMock(return_value=(1, None))) @patch.object(shell, "checked_call") @@ -129,7 +129,7 @@ class TestPackageResource(TestCase): Package("some_package", version = "3.5.0" ) - shell_mock.assert_called_with("/usr/bin/yum -d 0 -e 0 -y install some_package-3.5.0", sudo=True) + shell_mock.assert_called_with(['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'install', 'some_package-3.5.0'], sudo=True) @replace_underscores def func_to_test(self, name): http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-agent/src/test/python/resource_management/TestUserResource.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestUserResource.py b/ambari-agent/src/test/python/resource_management/TestUserResource.py index f1d22e6..86b23c5 100644 --- a/ambari-agent/src/test/python/resource_management/TestUserResource.py +++ b/ambari-agent/src/test/python/resource_management/TestUserResource.py @@ -38,7 +38,7 @@ class TestUserResource(TestCase): with Environment('/') as env: user = User("mapred", action = "create", shell = "/bin/bash") - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'useradd -m -s /bin/bash mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi useradd -m -s /bin/bash mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) self.assertEqual(popen_mock.call_count, 1) @patch.object(subprocess, "Popen") @@ -52,7 +52,7 @@ class TestUserResource(TestCase): with Environment('/') as env: user = User("mapred", action = "create", shell = "/bin/bash") - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -s /bin/bash mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -s /bin/bash mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) self.assertEqual(popen_mock.call_count, 1) @patch.object(subprocess, "Popen") @@ -66,7 +66,7 @@ class TestUserResource(TestCase): with Environment('/') as env: user = User("mapred", action = "remove", shell = "/bin/bash") - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'userdel mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'userdel mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) self.assertEqual(popen_mock.call_count, 1) @patch.object(subprocess, "Popen") @@ -81,7 +81,7 @@ class TestUserResource(TestCase): user = User("mapred", action = "create", comment = "testComment", shell = "/bin/bash") - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -c testComment -s /bin/bash mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -c testComment -s /bin/bash mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) self.assertEqual(popen_mock.call_count, 1) @patch.object(subprocess, "Popen") @@ -96,7 +96,7 @@ class TestUserResource(TestCase): user = User("mapred", action = "create", home = "/test/home", shell = "/bin/bash") - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -s /bin/bash -d /test/home mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -s /bin/bash -d /test/home mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) self.assertEqual(popen_mock.call_count, 1) @patch.object(subprocess, "Popen") @@ -111,7 +111,7 @@ class TestUserResource(TestCase): user = User("mapred", action = "create", password = "secure", shell = "/bin/bash") - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -s /bin/bash -p secure mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -s /bin/bash -p secure mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) self.assertEqual(popen_mock.call_count, 1) @patch.object(subprocess, "Popen") @@ -125,7 +125,7 @@ class TestUserResource(TestCase): with Environment('/') as env: user = User("mapred", action = "create", shell = "/bin/sh") - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -s /bin/sh mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -s /bin/sh mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) self.assertEqual(popen_mock.call_count, 1) @patch.object(subprocess, "Popen") @@ -139,7 +139,7 @@ class TestUserResource(TestCase): with Environment('/') as env: user = User("mapred", action = "create", uid = "1", shell = "/bin/bash") - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -s /bin/bash -u 1 mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -s /bin/bash -u 1 mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) self.assertEqual(popen_mock.call_count, 1) @patch.object(subprocess, "Popen") @@ -153,7 +153,7 @@ class TestUserResource(TestCase): with Environment('/') as env: user = User("mapred", action = "create", gid = "1", shell = "/bin/bash") - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -s /bin/bash -g 1 mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -s /bin/bash -g 1 mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) self.assertEqual(popen_mock.call_count, 1) @patch.object(subprocess, "Popen") @@ -168,7 +168,7 @@ class TestUserResource(TestCase): user = User("mapred", action = "create", groups = ['1','2','3'], shell = "/bin/bash") - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -G 1,2,3 -s /bin/bash mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -G 1,2,3 -s /bin/bash mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) self.assertEqual(popen_mock.call_count, 1) @patch.object(subprocess, "Popen") @@ -181,5 +181,5 @@ class TestUserResource(TestCase): with Environment('/') as env: user = User("mapred", action = "create") - popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'useradd -m mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None) + popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi useradd -m mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None) self.assertEqual(popen_mock.call_count, 1) http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/core/providers/package/apt.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py index 0ad6ca5..c683be5 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py @@ -21,12 +21,14 @@ Ambari Agent from resource_management.core.providers.package import PackageProvider from resource_management.core import shell +from resource_management.core.shell import string_cmd_from_args_list from resource_management.core.logger import Logger -INSTALL_CMD = "DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get -q -o Dpkg::Options::='--force-confdef' --allow-unauthenticated --assume-yes install %s" -REPO_UPDATE_CMD = "apt-get update -qq" -REMOVE_CMD = "/usr/bin/apt-get -y -q remove %s" -CHECK_CMD = "dpkg --get-selections | grep ^%s$ | grep -v deinstall" +INSTALL_CMD_ENV = {'DEBIAN_FRONTEND':'noninteractive'} +INSTALL_CMD = ['/usr/bin/apt-get', '-q', '-o', "Dpkg::Options::=--force-confdef", '--allow-unauthenticated', '--assume-yes', 'install'] +REPO_UPDATE_CMD = ['/usr/bin/apt-get', 'update','-qq'] +REMOVE_CMD = ['/usr/bin/apt-get', '-y', '-q', 'remove'] +CHECK_CMD = "dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep ^%s$" def replace_underscores(function_to_decorate): def wrapper(*args): @@ -40,14 +42,14 @@ class AptProvider(PackageProvider): @replace_underscores def install_package(self, name): if not self._check_existence(name): - cmd = INSTALL_CMD % (name) - Logger.info("Installing package %s ('%s')" % (name, cmd)) - code, out = shell.call(cmd, sudo=True) + cmd = INSTALL_CMD + [name] + Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) + code, out = shell.call(cmd, sudo=True, env=INSTALL_CMD_ENV) # apt-get update wasn't done too long if code: Logger.info("Execution of '%s' returned %d. %s" % (cmd, code, out)) - Logger.info("Failed to install package %s. Executing `%s`" % (name, REPO_UPDATE_CMD)) + Logger.info("Failed to install package %s. Executing `%s`" % (name, string_cmd_from_args_list(REPO_UPDATE_CMD))) code, out = shell.call(REPO_UPDATE_CMD, sudo=True) if code: @@ -65,8 +67,8 @@ class AptProvider(PackageProvider): @replace_underscores def remove_package(self, name): if self._check_existence(name): - cmd = REMOVE_CMD % (name) - Logger.info("Removing package %s ('%s')" % (name, cmd)) + cmd = REMOVE_CMD + [name] + Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) shell.checked_call(cmd, sudo=True) else: Logger.info("Skipping removing non-existent package %s" % (name)) http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py index b026ab4..7d868b0 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py @@ -22,17 +22,18 @@ Ambari Agent from resource_management.core.providers.package import PackageProvider from resource_management.core import shell +from resource_management.core.shell import string_cmd_from_args_list from resource_management.core.logger import Logger -INSTALL_CMD = "/usr/bin/yum -d 0 -e 0 -y install %s" -REMOVE_CMD = "/usr/bin/yum -d 0 -e 0 -y erase %s" +INSTALL_CMD = ['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'install'] +REMOVE_CMD = ['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'erase'] CHECK_CMD = "installed_pkgs=`rpm -qa %s` ; [ ! -z \"$installed_pkgs\" ]" class YumProvider(PackageProvider): def install_package(self, name): if not self._check_existence(name): - cmd = INSTALL_CMD % (name) - Logger.info("Installing package %s ('%s')" % (name, cmd)) + cmd = INSTALL_CMD + [name] + Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) shell.checked_call(cmd, sudo=True) else: Logger.info("Skipping installing existent package %s" % (name)) @@ -42,8 +43,8 @@ class YumProvider(PackageProvider): def remove_package(self, name): if self._check_existence(name): - cmd = REMOVE_CMD % (name) - Logger.info("Removing package %s ('%s')" % (name, cmd)) + cmd = REMOVE_CMD + [name] + Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) shell.checked_call(cmd, sudo=True) else: Logger.info("Skipping removing non-existent package %s" % (name)) http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py index 293a454..764da73 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py @@ -22,17 +22,18 @@ Ambari Agent from resource_management.core.providers.package import PackageProvider from resource_management.core import shell +from resource_management.core.shell import string_cmd_from_args_list from resource_management.core.logger import Logger -INSTALL_CMD = "/usr/bin/zypper --quiet install --auto-agree-with-licenses --no-confirm %s" -REMOVE_CMD = "/usr/bin/zypper --quiet remove --no-confirm %s" -CHECK_CMD = "rpm -qa | grep ^%s" +INSTALL_CMD = ['/usr/bin/zypper', '--quiet', 'install', '--auto-agree-with-licenses', '--no-confirm'] +REMOVE_CMD = ['/usr/bin/zypper', '--quiet', 'remove', '--no-confirm'] +CHECK_CMD = "installed_pkgs=`rpm -qa %s` ; [ ! -z \"$installed_pkgs\" ]" class ZypperProvider(PackageProvider): def install_package(self, name): if not self._check_existence(name): - cmd = INSTALL_CMD % (name) - Logger.info("Installing package %s ('%s')" % (name, cmd)) + cmd = INSTALL_CMD + [name] + Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) shell.checked_call(cmd, sudo=True) else: Logger.info("Skipping installing existent package %s" % (name)) @@ -42,8 +43,8 @@ class ZypperProvider(PackageProvider): def remove_package(self, name): if self._check_existence(name): - cmd = REMOVE_CMD % (name) - Logger.info("Removing package %s ('%s')" % (name, cmd)) + cmd = REMOVE_CMD + [name] + Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) shell.checked_call(cmd, sudo=True) else: Logger.info("Skipping removing non-existent package %s" % (name)) http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/core/providers/system.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/system.py b/ambari-common/src/main/python/resource_management/core/providers/system.py index 6ec5bb5..db58898 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/system.py +++ b/ambari-common/src/main/python/resource_management/core/providers/system.py @@ -246,6 +246,7 @@ class ExecuteProvider(Provider): wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout, path=self.resource.path, + output_file=self.resource.output_file, sudo=self.resource.sudo) break except Fail as ex: http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/core/resources/system.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/resources/system.py b/ambari-common/src/main/python/resource_management/core/resources/system.py index 49e3a7b..a86fa54 100644 --- a/ambari-common/src/main/python/resource_management/core/resources/system.py +++ b/ambari-common/src/main/python/resource_management/core/resources/system.py @@ -74,6 +74,13 @@ class Execute(Resource): command = ResourceArgument(default=lambda obj: obj.name) creates = ResourceArgument() + """ + cwd won't work for: + - commands run as sudo + - commands run as user (which uses sudo as well) + + This is because non-interactive sudo commands doesn't support that. + """ cwd = ResourceArgument() # this runs command with a specific env variables, env={'JAVA_HOME': '/usr/jdk'} environment = ResourceArgument(default={}) @@ -102,9 +109,15 @@ class Execute(Resource): - try_sleep """ wait_for_finish = BooleanArgument(default=True) + output_file = ResourceArgument() + """ + For calling more advanced commands use as_sudo(command) option. + Example: + command1 = as_sudo(["cat,"/etc/passwd"]) + " | grep user" + command2 = as_sudo(["ls", "/root/example.txt") + " && " + as_sudo(["rm","-f","example.txt"]) + """ sudo = BooleanArgument(default=False) - class ExecuteScript(Resource): action = ForcedListArgument(default="run") code = ResourceArgument(required=True) http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/core/shell.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/shell.py b/ambari-common/src/main/python/resource_management/core/shell.py index 2a472f2..7dbdc79 100644 --- a/ambari-common/src/main/python/resource_management/core/shell.py +++ b/ambari-common/src/main/python/resource_management/core/shell.py @@ -21,7 +21,7 @@ Ambari Agent """ import os -__all__ = ["checked_call", "call", "quote_bash_args"] +__all__ = ["checked_call", "call", "quote_bash_args", "as_user", "as_sudo"] import string import subprocess @@ -31,16 +31,18 @@ from exceptions import Fail from exceptions import ExecuteTimeoutException from resource_management.core.logger import Logger +SUDO_ENVIRONMENT_PLACEHOLDER = "{ENV_PLACEHOLDER}" + def checked_call(command, logoutput=False, - cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, path=None, sudo=False): - return _call(command, logoutput, True, cwd, env, preexec_fn, user, wait_for_finish, timeout, path, sudo) + cwd=None, env={}, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, path=None, output_file=None, sudo=False): + return _call(command, logoutput, True, cwd, env, preexec_fn, user, wait_for_finish, timeout, path, output_file, sudo) def call(command, logoutput=False, - cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, path=None, sudo=False): - return _call(command, logoutput, False, cwd, env, preexec_fn, user, wait_for_finish, timeout, path, sudo) + cwd=None, env={}, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, path=None, output_file=None, sudo=False): + return _call(command, logoutput, False, cwd, env, preexec_fn, user, wait_for_finish, timeout, path, output_file, sudo) def _call(command, logoutput=False, throw_on_failure=True, - cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, path=None, sudo=False): + cwd=None, env={}, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, path=None, output_file=None, sudo=False): """ Execute shell command @@ -53,22 +55,32 @@ def _call(command, logoutput=False, throw_on_failure=True, """ # convert to string and escape if isinstance(command, (list, tuple)): - command = ' '.join(quote_bash_args(x) for x in command) + command = string_cmd_from_args_list(command) + elif sudo: + # Since ambari user sudoer privileges may be restricted, + # without having /bin/bash permission. + # Running interpreted shell commands in scope of 'sudo' is not possible. + # + # In that case while passing string, + # any bash symbols eventually added to command like && || ; < > | << >> would cause problems. + # + # In case of need to create more complicated commands with sudo use as_sudo(command) function. + err_msg = Logger.get_protected_text(("String command '%s' cannot be run as sudo. Please supply the command as a tuple of arguments") % (command)) + raise Fail(err_msg) # In case we will use sudo, we have to put all the environment inside the command, # since Popen environment gets reset within sudo. - export_command = reduce(lambda str,x: '{0} {1}={2}'.format(str,x,quote_bash_args(env[x])), env, 'export') + '; ' if env else '' - + environment_str = reduce(lambda str,x: '{0} {1}={2}'.format(str,x,quote_bash_args(env[x])), env,'') + command = command.replace(SUDO_ENVIRONMENT_PLACEHOLDER, environment_str, 1) # replace placeholder from as_sudo / as_user if present + + bash_run_command = command if not sudo else "/usr/bin/sudo {0} -Hi {1}".format(environment_str, command) + if user: - bash_run_command = "/usr/bin/sudo -Hsu {0} <<< {1}".format(quote_bash_args(user), quote_bash_args(export_command + command)) - # Go to home directory. In case we are in folder, which user cannot open, we might run into troubles with some utils - cwd = os.path.expanduser('~'+user) if not cwd and os.path.exists(os.path.expanduser('~'+user)) else cwd - elif sudo: - bash_run_command = "/usr/bin/sudo -s <<< {0}".format(quote_bash_args(export_command + command)) + # Outter environment gets reset within su. That's why we can't use environment passed to Popen. + su_export_command = "export {0} ; ".format(environment_str) if environment_str else "" + subprocess_command = ["/usr/bin/sudo","-Hi","su", "-", user, "-s", "/bin/bash", "-c", su_export_command + bash_run_command] else: - bash_run_command = command - - subprocess_command = ["/bin/bash","--login","-c", bash_run_command] + subprocess_command = ["/bin/bash","--login","-c", bash_run_command] proc = subprocess.Popen(subprocess_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd, env=env, shell=False, @@ -102,6 +114,44 @@ def _call(command, logoutput=False, throw_on_failure=True, return code, out +def as_sudo(command, env=SUDO_ENVIRONMENT_PLACEHOLDER): + """ + command - list or tuple of arguments. + env - when run as part of Execute resource, this SHOULD NOT be used. + It automatically gets replaced later by call, checked_call. This should be used in not_if, only_if + """ + if isinstance(command, (list, tuple)): + command = string_cmd_from_args_list(command) + else: + # Since ambari user sudoer privileges may be restricted, + # without having /bin/bash permission, and /bin/su permission. + # Running interpreted shell commands in scope of 'sudo' is not possible. + # + # In that case while passing string, + # any bash symbols eventually added to command like && || ; < > | << >> would cause problems. + err_msg = Logger.get_protected_text(("String command '%s' cannot be run as sudo. Please supply the command as a tuple/list of arguments") % (command)) + raise Fail(err_msg) + + if env != SUDO_ENVIRONMENT_PLACEHOLDER: + env = reduce(lambda str,x: '{0} {1}={2}'.format(str,x,quote_bash_args(env[x])), env, '') + + return "/usr/bin/sudo {0} -Hi {1}".format(env, command) + +def as_user(command, user , env=SUDO_ENVIRONMENT_PLACEHOLDER): + if isinstance(command, (list, tuple)): + command = string_cmd_from_args_list(command) + + if env != SUDO_ENVIRONMENT_PLACEHOLDER: + env = reduce(lambda str,x: '{0} {1}={2}'.format(str,x,quote_bash_args(env[x])), env, '') + + export_command = "export {0} ; ".format(env) + + result_command = "/usr/bin/sudo -Hi su - {0} -s /bin/bash -c {1}".format(user, quote_bash_args(export_command + command)) + return result_command + +def string_cmd_from_args_list(command): + return ' '.join(quote_bash_args(x) for x in command) + def on_timeout(proc, q): q.put(True) if proc.poll() == None: @@ -117,4 +167,4 @@ def quote_bash_args(command): for char in command: if char not in valid: return "'" + command.replace("'", "'\"'\"'") + "'" - return command + return command \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/core/sudo.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/sudo.py b/ambari-common/src/main/python/resource_management/core/sudo.py index 31cd468..fb81666 100644 --- a/ambari-common/src/main/python/resource_management/core/sudo.py +++ b/ambari-common/src/main/python/resource_management/core/sudo.py @@ -28,7 +28,7 @@ def chown(path, owner, group): if owner: shell.checked_call(["chown", owner, path], sudo=True) if group: - shell.checked_call(["chgrp", owner, path], sudo=True) + shell.checked_call(["chgrp", group, path], sudo=True) # os.chmod replacement def chmod(path, mode): http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py b/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py index 19f4669..d854c55 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py @@ -25,6 +25,7 @@ from resource_management import * class CopyFromLocalProvider(Provider): def action_run(self): + path = self.resource.path dest_dir = self.resource.dest_dir dest_file = self.resource.dest_file @@ -45,8 +46,8 @@ class CopyFromLocalProvider(Provider): copy_cmd = format("fs -copyFromLocal {path} {dest_dir}") dest_path = dest_dir + os.sep + dest_file_name # Need to run unless as resource user - su_cmd = 'su - {0} -c'.format(owner) - unless_cmd = format("{su_cmd} '{kinnit_if_needed} export PATH=$PATH:{bin_dir} ; hadoop fs -ls {dest_path}' >/dev/null 2>&1") + unless_cmd = as_user("{kinnit_if_needed} ; hadoop fs -ls {dest_path}", owner, env={'PATH':bin_dir}) + ExecuteHadoop(copy_cmd, not_if=unless_cmd,
