AMBARI-9943. Umask 027 + non-root result in failures during deploy (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/72dfdd49 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/72dfdd49 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/72dfdd49 Branch: refs/heads/branch-2.0.0 Commit: 72dfdd499f6102ce4efa1ed45ed7a48e13aabe0d Parents: 34d004c Author: Andrew Onishuk <[email protected]> Authored: Thu Mar 5 17:55:26 2015 +0200 Committer: Andrew Onishuk <[email protected]> Committed: Thu Mar 5 17:55:26 2015 +0200 ---------------------------------------------------------------------- ambari-agent/conf/unix/ambari-env.sh | 1 + .../TestDirectoryResource.py | 35 +++++++------- .../resource_management/TestExecuteResource.py | 3 +- .../resource_management/TestFileResource.py | 40 ++++++++-------- .../resource_management/TestLinkResource.py | 30 ++++++------ .../TestPropertiesFileResource.py | 20 ++++---- .../TestXmlConfigResource.py | 20 ++++---- .../core/providers/system.py | 50 ++++++++++---------- .../python/resource_management/core/sudo.py | 29 ++++++++++++ .../HIVE/0.12.0.2.0/package/scripts/hive.py | 11 +++-- .../scripts/shared_initialization.py | 4 ++ .../stacks/2.0.6/HIVE/test_hive_metastore.py | 6 +++ .../stacks/2.0.6/HIVE/test_hive_server.py | 6 +++ .../hooks/before-INSTALL/test_before_install.py | 5 ++ .../stacks/2.1/HIVE/test_hive_metastore.py | 16 +++++-- 15 files changed, 168 insertions(+), 108 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/72dfdd49/ambari-agent/conf/unix/ambari-env.sh ---------------------------------------------------------------------- diff --git a/ambari-agent/conf/unix/ambari-env.sh b/ambari-agent/conf/unix/ambari-env.sh index bf7ca55..a5f350f 100644 --- a/ambari-agent/conf/unix/ambari-env.sh +++ b/ambari-agent/conf/unix/ambari-env.sh @@ -17,3 +17,4 @@ # given through environment variable AMBARI_PASSPHRASE="DEV" export PATH=$PATH:/var/lib/ambari-agent +export PYTHONPATH=$PYTHONPATH:/usr/lib/python2.6/site-packages http://git-wip-us.apache.org/repos/asf/ambari/blob/72dfdd49/ambari-agent/src/test/python/resource_management/TestDirectoryResource.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestDirectoryResource.py b/ambari-agent/src/test/python/resource_management/TestDirectoryResource.py index fe64400..f42de93 100644 --- a/ambari-agent/src/test/python/resource_management/TestDirectoryResource.py +++ b/ambari-agent/src/test/python/resource_management/TestDirectoryResource.py @@ -19,7 +19,6 @@ limitations under the License. from unittest import TestCase from mock.mock import patch, MagicMock import os -import shutil from resource_management.core.system import System from resource_management.core import Environment, Fail, sudo from resource_management.core.resources import Directory @@ -27,10 +26,10 @@ from resource_management.core.resources import Directory @patch.object(System, "os_family", new = 'redhat') class TestDirectoryResource(TestCase): - @patch.object(os.path, "exists") + @patch.object(sudo, "path_exists") @patch.object(sudo, "makedirs") - @patch.object(os.path, "isdir") - @patch.object(os, "stat") + @patch.object(sudo, "path_isdir") + @patch.object(sudo, "stat") @patch.object(sudo,"chmod") @patch.object(sudo,"chown") @patch("resource_management.core.providers.system._coerce_uid") @@ -59,11 +58,11 @@ class TestDirectoryResource(TestCase): os_chown_mock.assert_any_call('/a/b/c/d', 'hdfs', None) os_chown_mock.assert_any_call('/a/b/c/d', None, 'hadoop') - @patch.object(os.path, "exists") + @patch.object(sudo, "path_exists") @patch.object(os.path, "dirname") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_isdir") @patch.object(sudo, "makedir") - @patch.object(os, "stat") + @patch.object(sudo, "stat") @patch.object(sudo,"chmod") @patch.object(sudo,"chown") @patch("resource_management.core.providers.system._coerce_uid") @@ -92,9 +91,9 @@ class TestDirectoryResource(TestCase): os_chown_mock.assert_any_call('/a/b/c/d', 'hdfs', None) os_chown_mock.assert_any_call('/a/b/c/d', None, 'hadoop') - @patch.object(os.path, "exists") + @patch.object(sudo, "path_exists") @patch.object(os.path, "dirname") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_isdir") def test_create_directory_failed_no_parent(self, isdir_mock, os_dirname_mock, os_path_exists_mock): os_path_exists_mock.return_value = False @@ -115,8 +114,8 @@ class TestDirectoryResource(TestCase): self.assertEqual('Applying u"Directory[\'/a/b/c/d\']" failed, parent directory /a/b/c doesn\'t exist', str(e)) - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") def test_create_directory_path_is_file_or_line(self, isdir_mock, os_path_exists_mock): os_path_exists_mock.return_value = True isdir_mock.return_value = False @@ -134,9 +133,9 @@ class TestDirectoryResource(TestCase): self.assertEqual('Applying u"Directory[\'/a/b/c/d\']" failed, file /a/b/c/d already exists', str(e)) - @patch.object(shutil, "rmtree") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "rmtree") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") def test_delete_directory(self, isdir_mock, os_path_exists_mock, rmtree_mock): os_path_exists_mock.return_value = True isdir_mock.return_value = True @@ -148,7 +147,7 @@ class TestDirectoryResource(TestCase): rmtree_mock.assert_called_with('/a/b/c/d') - @patch.object(os.path, "exists") + @patch.object(sudo, "path_exists") def test_delete_noexisting_directory(self, os_path_exists_mock): os_path_exists_mock.return_value = False @@ -157,9 +156,9 @@ class TestDirectoryResource(TestCase): action='delete' ) - @patch.object(shutil, "rmtree") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "rmtree") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") def test_delete_directory_with_path_to_file(self, isdir_mock, os_path_exists_mock, rmtree_mock): os_path_exists_mock.return_value = True isdir_mock.return_value = False http://git-wip-us.apache.org/repos/asf/ambari/blob/72dfdd49/ambari-agent/src/test/python/resource_management/TestExecuteResource.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestExecuteResource.py b/ambari-agent/src/test/python/resource_management/TestExecuteResource.py index 216fd1a..d92f905 100644 --- a/ambari-agent/src/test/python/resource_management/TestExecuteResource.py +++ b/ambari-agent/src/test/python/resource_management/TestExecuteResource.py @@ -22,6 +22,7 @@ from mock.mock import patch, MagicMock, call from resource_management.core.system import System from resource_management.core.resources.system import Execute from resource_management.core.environment import Environment +from resource_management.core import sudo import subprocess import logging @@ -63,7 +64,7 @@ class TestExecuteResource(TestCase): self.assertTrue(popen_mock.called, 'subprocess.Popen should have been called!') self.assertFalse(proc_communicate_mock.called, 'proc.communicate should not have been called!') - @patch.object(os.path, "exists") + @patch.object(sudo, "path_exists") @patch.object(subprocess, "Popen") def test_attribute_creates(self, popen_mock, exists_mock): exists_mock.return_value = True http://git-wip-us.apache.org/repos/asf/ambari/blob/72dfdd49/ambari-agent/src/test/python/resource_management/TestFileResource.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestFileResource.py b/ambari-agent/src/test/python/resource_management/TestFileResource.py index 28fa610..f60fc12 100644 --- a/ambari-agent/src/test/python/resource_management/TestFileResource.py +++ b/ambari-agent/src/test/python/resource_management/TestFileResource.py @@ -32,7 +32,7 @@ import resource_management @patch.object(System, "os_family", new = 'redhat') class TestFileResource(TestCase): @patch.object(os.path, "dirname") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_isdir") def test_action_create_dir_exist(self, isdir_mock, dirname_mock): """ Tests if 'create' action fails when path is existent directory @@ -53,7 +53,7 @@ class TestFileResource(TestCase): self.assertFalse(dirname_mock.called) @patch.object(os.path, "dirname") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_isdir") def test_action_create_parent_dir_non_exist(self, isdir_mock, dirname_mock): """ Tests if 'create' action fails when parent directory of path @@ -79,8 +79,8 @@ class TestFileResource(TestCase): @patch("resource_management.core.providers.system._ensure_metadata") @patch.object(sudo, "read_file") @patch.object(sudo, "create_file") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") def test_action_create_non_existent_file(self, isdir_mock, exists_mock, create_file_mock, read_file_mock, ensure_mock): """ Tests if 'create' action create new non existent file and write proper data @@ -103,8 +103,8 @@ class TestFileResource(TestCase): @patch("resource_management.core.providers.system._ensure_metadata") @patch.object(sudo, "read_file") @patch.object(sudo, "create_file") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") def test_action_create_replace(self, isdir_mock, exists_mock, create_file_mock, read_file_mock, ensure_mock): """ Tests if 'create' action rewrite existent file with new data @@ -125,8 +125,8 @@ class TestFileResource(TestCase): @patch.object(sudo, "unlink") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") def test_action_delete_is_directory(self, isdir_mock, exist_mock, unlink_mock): """ Tests if 'delete' action fails when path is directory @@ -151,8 +151,8 @@ class TestFileResource(TestCase): self.assertEqual(unlink_mock.call_count, 0) @patch.object(sudo, "unlink") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") def test_action_delete(self, isdir_mock, exist_mock, unlink_mock): """ Tests if 'delete' action removes file @@ -173,7 +173,7 @@ class TestFileResource(TestCase): self.assertEqual(unlink_mock.call_count, 1) - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_isdir") def test_attribute_path(self, isdir_mock): """ Tests 'path' attribute @@ -198,8 +198,8 @@ class TestFileResource(TestCase): @patch("resource_management.core.providers.system._ensure_metadata") @patch.object(sudo, "read_file") @patch.object(sudo, "create_file") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") def test_attribute_backup(self, isdir_mock, exists_mock, create_file_mock, read_file_mock, ensure_mock, backup_file_mock): """ Tests 'backup' attribute @@ -233,8 +233,8 @@ class TestFileResource(TestCase): @patch("resource_management.core.providers.system._ensure_metadata") @patch("__builtin__.open") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") def test_attribute_replace(self, isdir_mock, exists_mock, open_mock, ensure_mock): """ Tests 'replace' attribute @@ -265,10 +265,10 @@ class TestFileResource(TestCase): @patch("resource_management.core.providers.system._coerce_gid") @patch.object(sudo, "chown") @patch.object(sudo, "chmod") - @patch.object(os, "stat") + @patch.object(sudo, "stat") @patch.object(sudo, "create_file") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") def test_ensure_metadata(self, isdir_mock, exists_mock, create_file_mock, stat_mock, chmod_mock, chown_mock, gid_mock, uid_mock): """ @@ -327,8 +327,8 @@ class TestFileResource(TestCase): @patch("resource_management.core.providers.system.FileProvider._get_content") @patch.object(sudo, "read_file") @patch.object(sudo, "create_file") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") def test_action_create_encoding(self, isdir_mock, exists_mock, create_file_mock, read_file_mock, get_content_mock ,ensure_mock): isdir_mock.side_effect = [False, True] http://git-wip-us.apache.org/repos/asf/ambari/blob/72dfdd49/ambari-agent/src/test/python/resource_management/TestLinkResource.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestLinkResource.py b/ambari-agent/src/test/python/resource_management/TestLinkResource.py index cdb6061..2d5194c 100644 --- a/ambari-agent/src/test/python/resource_management/TestLinkResource.py +++ b/ambari-agent/src/test/python/resource_management/TestLinkResource.py @@ -30,8 +30,8 @@ import os class TestLinkResource(TestCase): @patch.object(os.path, "realpath") - @patch.object(os.path, "lexists") - @patch.object(os.path, "islink") + @patch.object(sudo, "path_lexists") + @patch.object(sudo, "path_lexists") @patch.object(sudo, "unlink") @patch.object(sudo, "symlink") def test_action_create_relink(self, symlink_mock, unlink_mock, @@ -49,13 +49,11 @@ class TestLinkResource(TestCase): symlink_mock.assert_called_with("/a/b/link_to_path", "/some_path") @patch.object(os.path, "realpath") - @patch.object(os.path, "lexists") - @patch.object(os.path, "islink") - def test_action_create_failed_due_to_file_exists(self, islink_mock, + @patch.object(sudo, "path_lexists") + def test_action_create_failed_due_to_file_exists(self, lexists_mock, realmock): - lexists_mock.return_value = True + lexists_mock.side_effect = [True, False] realmock.return_value = "/old_to_link_path" - islink_mock.return_value = False with Environment('/') as env: try: Link("/some_path", @@ -67,7 +65,7 @@ class TestLinkResource(TestCase): self.assertEqual("LinkProvider[Link['/some_path']] trying to create a symlink with the same name as an existing file or directory", str(e)) - @patch.object(os.path, "lexists") + @patch.object(sudo, "path_lexists") @patch.object(sudo, "symlink") def test_action_create_symlink_clean_create(self, symlink_mock, lexists_mock): lexists_mock.return_value = False @@ -80,8 +78,8 @@ class TestLinkResource(TestCase): symlink_mock.assert_called_with("/a/b/link_to_path", "/some_path") @patch.object(os.path, "isdir") - @patch.object(os.path, "exists") - @patch.object(os.path, "lexists") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_lexists") @patch.object(sudo, "link") def test_action_create_hardlink_clean_create(self, link_mock, lexists_mock, exists_mock, isdir_mock): @@ -97,8 +95,8 @@ class TestLinkResource(TestCase): link_mock.assert_called_with("/a/b/link_to_path", "/some_path") - @patch.object(os.path, "exists") - @patch.object(os.path, "lexists") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_lexists") def test_action_create_hardlink_target_doesnt_exist(self, lexists_mock, exists_mock): lexists_mock.return_value = False @@ -115,9 +113,9 @@ class TestLinkResource(TestCase): self.assertEqual('Failed to apply u"Link[\'/some_path\']", linking to nonexistent location /a/b/link_to_path', str(e)) - @patch.object(os.path, "isdir") - @patch.object(os.path, "exists") - @patch.object(os.path, "lexists") + @patch.object(sudo, "path_isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_lexists") def test_action_create_hardlink_target_is_dir(self, lexists_mock, exists_mock, isdir_mock): lexists_mock.return_value = False @@ -136,7 +134,7 @@ class TestLinkResource(TestCase): str(e)) @patch.object(sudo, "unlink") - @patch.object(os.path, "exists") + @patch.object(sudo, "path_exists") def test_action_delete(self, exists_mock, unlink_mock): exists_mock.return_value = True http://git-wip-us.apache.org/repos/asf/ambari/blob/72dfdd49/ambari-agent/src/test/python/resource_management/TestPropertiesFileResource.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestPropertiesFileResource.py b/ambari-agent/src/test/python/resource_management/TestPropertiesFileResource.py index bdb64de..1147928 100644 --- a/ambari-agent/src/test/python/resource_management/TestPropertiesFileResource.py +++ b/ambari-agent/src/test/python/resource_management/TestPropertiesFileResource.py @@ -38,8 +38,8 @@ class TestPropertiesFIleResource(TestCase): @patch("resource_management.core.providers.system._ensure_metadata") @patch.object(sudo, "create_file") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") @patch.object(time, "asctime") def test_action_create_empty_properties_without_dir(self, time_asctime_mock, @@ -71,8 +71,8 @@ class TestPropertiesFIleResource(TestCase): @patch("resource_management.core.providers.system._ensure_metadata") @patch.object(sudo, "create_file") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") @patch.object(time, "asctime") def test_action_create_empty_properties_with_dir(self, time_asctime_mock, @@ -104,8 +104,8 @@ class TestPropertiesFIleResource(TestCase): @patch("resource_management.core.providers.system._ensure_metadata") @patch.object(sudo, "create_file") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") @patch.object(time, "asctime") def test_action_create_properties_simple(self, time_asctime_mock, @@ -137,8 +137,8 @@ class TestPropertiesFIleResource(TestCase): @patch("resource_management.core.providers.system._ensure_metadata") @patch.object(sudo, "create_file") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") @patch.object(time, "asctime") def test_action_create_properties_with_metacharacters(self, time_asctime_mock, @@ -176,8 +176,8 @@ class TestPropertiesFIleResource(TestCase): @patch("resource_management.core.providers.system._ensure_metadata") @patch.object(sudo, "read_file") @patch.object(sudo, "create_file") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") @patch.object(time, "asctime") def test_action_create_properties_rewrite_content(self, time_asctime_mock, http://git-wip-us.apache.org/repos/asf/ambari/blob/72dfdd49/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py b/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py index 4affd31..041fc9a 100644 --- a/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py +++ b/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py @@ -38,8 +38,8 @@ class TestXmlConfigResource(TestCase): @patch("resource_management.core.providers.system._ensure_metadata") @patch.object(sudo, "create_file") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") @patch.object(time, "asctime") def test_action_create_empty_xml_config(self, time_asctime_mock, @@ -67,8 +67,8 @@ class TestXmlConfigResource(TestCase): @patch("resource_management.core.providers.system._ensure_metadata") @patch.object(sudo, "create_file") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") @patch.object(time, "asctime") def test_action_create_simple_xml_config(self, time_asctime_mock, @@ -96,8 +96,8 @@ class TestXmlConfigResource(TestCase): @patch("resource_management.core.providers.system._ensure_metadata") @patch.object(sudo, "create_file") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") @patch.object(time, "asctime") def test_action_create_xml_config_with_metacharacters(self, time_asctime_mock, @@ -148,8 +148,8 @@ class TestXmlConfigResource(TestCase): @patch("resource_management.core.providers.system._ensure_metadata") @patch.object(sudo, "create_file") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") @patch.object(time, "asctime") def test_action_create_xml_config_sorted_by_key(self, time_asctime_mock, @@ -180,8 +180,8 @@ class TestXmlConfigResource(TestCase): create_file_mock.assert_called_with('/dir/conf/file.xml', u'<!--Wed 2014-02-->\n <configuration>\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>first</name>\n <value>should be first</value>\n </property>\n \n <property>\n <name>second</name>\n <value>should be second</value>\n </property>\n \n <property>\n <name>third</name>\n <value>should be third</value>\n </property>\n \n <property>\n <name>z_last</name>\n <value>should be last</value>\n </property>\n \n </configuration>\n') @patch("resource_management.libraries.providers.xml_config.File") - @patch.object(os.path, "exists") - @patch.object(os.path, "isdir") + @patch.object(sudo, "path_exists") + @patch.object(sudo, "path_isdir") def test_action_create_arguments(self, os_path_isdir_mock ,os_path_exists_mock, file_mock): os_path_isdir_mock.side_effect = [False, True] http://git-wip-us.apache.org/repos/asf/ambari/blob/72dfdd49/ambari-common/src/main/python/resource_management/core/providers/system.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/system.py b/ambari-common/src/main/python/resource_management/core/providers/system.py index ac63b21..95ba80d 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/system.py +++ b/ambari-common/src/main/python/resource_management/core/providers/system.py @@ -27,7 +27,6 @@ import grp import os import pwd import time -import shutil from resource_management.core import shell from resource_management.core import sudo from resource_management.core.base import Fail @@ -59,8 +58,8 @@ def _coerce_gid(group): def _ensure_metadata(path, user, group, mode=None, cd_access=None): - stat = os.stat(path) - + stat = sudo.stat(path) + if user: uid = _coerce_uid(user) if stat.st_uid != uid: @@ -77,10 +76,9 @@ def _ensure_metadata(path, user, group, mode=None, cd_access=None): sudo.chown(path, None, group) if mode: - existing_mode = stat.st_mode & 07777 - if existing_mode != mode: + if stat.st_mode != mode: Logger.info("Changing permission for %s from %o to %o" % ( - path, existing_mode, mode)) + path, stat.st_mode, mode)) sudo.chmod(path, mode) if cd_access: @@ -89,7 +87,7 @@ def _ensure_metadata(path, user, group, mode=None, cd_access=None): dir_path = path while dir_path != os.sep: - if os.path.isdir(dir_path): + if sudo.path_isdir(dir_path): sudo.chmod_extended(dir_path, cd_access+"+x") dir_path = os.path.split(dir_path)[0] @@ -99,16 +97,16 @@ class FileProvider(Provider): def action_create(self): path = self.resource.path - if os.path.isdir(path): + if sudo.path_isdir(path): raise Fail("Applying %s failed, directory with name %s exists" % (self.resource, path)) dirname = os.path.dirname(path) - if not os.path.isdir(dirname): + if not sudo.path_isdir(dirname): raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname)) write = False content = self._get_content() - if not os.path.exists(path): + if not sudo.path_exists(path): write = True reason = "it doesn't exist" elif self.resource.replace: @@ -135,10 +133,10 @@ class FileProvider(Provider): def action_delete(self): path = self.resource.path - if os.path.isdir(path): + if sudo.path_isdir(path): raise Fail("Applying %s failed, %s is directory not file!" % (self.resource, path)) - if os.path.exists(path): + if sudo.path_exists(path): Logger.info("Deleting %s" % self.resource) sudo.unlink(path) @@ -157,7 +155,7 @@ class DirectoryProvider(Provider): def action_create(self): path = self.resource.path - if not os.path.exists(path): + if not sudo.path_exists(path): Logger.info("Creating directory %s" % self.resource) if self.resource.recursive: if self.resource.recursive_permission: @@ -167,12 +165,12 @@ class DirectoryProvider(Provider): sudo.makedirs(path, self.resource.mode or 0755) else: dirname = os.path.dirname(path) - if not os.path.isdir(dirname): + if not sudo.path_isdir(dirname): raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname)) sudo.makedir(path, self.resource.mode or 0755) - if not os.path.isdir(path): + if not sudo.path_isdir(path): raise Fail("Applying %s failed, file %s already exists" % (self.resource, path)) _ensure_metadata(path, self.resource.owner, self.resource.group, @@ -191,44 +189,44 @@ class DirectoryProvider(Provider): dir_prefix="" for folder in folders: dir_prefix=os.path.join(dir_prefix, folder) - if not os.path.exists(dir_prefix): + if not sudo.path_exists(dir_prefix): sudo.makedir(dir_prefix, mode or 0755) _ensure_metadata(dir_prefix, None, None, mode) def action_delete(self): path = self.resource.path - if os.path.exists(path): - if not os.path.isdir(path): + if sudo.path_exists(path): + if not sudo.path_isdir(path): raise Fail("Applying %s failed, %s is not a directory" % (self.resource, path)) Logger.info("Removing directory %s and all its content" % self.resource) - shutil.rmtree(path) + sudo.rmtree(path) class LinkProvider(Provider): def action_create(self): path = self.resource.path - if os.path.lexists(path): + if sudo.path_lexists(path): oldpath = os.path.realpath(path) if oldpath == self.resource.to: return - if not os.path.islink(path): + if not sudo.path_lexists(path): raise Fail( "%s trying to create a symlink with the same name as an existing file or directory" % self) Logger.info("%s replacing old symlink to %s" % (self.resource, oldpath)) sudo.unlink(path) if self.resource.hard: - if not os.path.exists(self.resource.to): + if not sudo.path_exists(self.resource.to): raise Fail("Failed to apply %s, linking to nonexistent location %s" % (self.resource, self.resource.to)) - if os.path.isdir(self.resource.to): + if sudo.path_isdir(self.resource.to): raise Fail("Failed to apply %s, cannot create hard link to a directory (%s)" % (self.resource, self.resource.to)) Logger.info("Creating hard %s" % self.resource) sudo.link(self.resource.to, path) else: - if not os.path.exists(self.resource.to): + if not sudo.path_exists(self.resource.to): Logger.info("Warning: linking to nonexistent location %s" % self.resource.to) Logger.info("Creating symbolic %s" % self.resource) @@ -236,7 +234,7 @@ class LinkProvider(Provider): def action_delete(self): path = self.resource.path - if os.path.exists(path): + if sudo.path_exists(path): Logger.info("Deleting %s" % self.resource) sudo.unlink(path) @@ -254,7 +252,7 @@ def _preexec_fn(resource): class ExecuteProvider(Provider): def action_run(self): if self.resource.creates: - if os.path.exists(self.resource.creates): + if sudo.path_exists(self.resource.creates): Logger.info("Skipping %s due to creates" % self.resource) return http://git-wip-us.apache.org/repos/asf/ambari/blob/72dfdd49/ambari-common/src/main/python/resource_management/core/sudo.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/sudo.py b/ambari-common/src/main/python/resource_management/core/sudo.py index ae21f84..938e95b 100644 --- a/ambari-common/src/main/python/resource_management/core/sudo.py +++ b/ambari-common/src/main/python/resource_management/core/sudo.py @@ -59,6 +59,10 @@ def link(source, link_name): def unlink(path): shell.checked_call(["rm","-f", path], sudo=True) +# shutil.rmtree +def rmtree(path): + shell.checked_call(["rm","-rf", path], sudo=True) + # fp.write replacement def create_file(filename, content): """ @@ -84,3 +88,28 @@ def read_file(filename): with tmpf: with open(tmpf.name, "rb") as fp: return fp.read() + +# os.path.exists +def path_exists(path): + return (shell.call(["test", "-e", path], sudo=True)[0] == 0) + +# os.path.isdir +def path_isdir(path): + return (shell.call(["test", "-d", path], sudo=True)[0] == 0) + +# os.path.lexists +def path_lexists(path): + return (shell.call(["test", "-L", path], sudo=True)[0] == 0) + +# os.stat +def stat(path): + class Stat: + def __init__(self, path): + # TODO: check this on Ubuntu + out = shell.checked_call(["stat", "-c", "%u %g %a", path], sudo=True)[1] + uid_str, gid_str, mode_str = out.split(' ') + self.st_uid = int(uid_str) + self.st_gid = int(gid_str) + self.st_mode = int(mode_str, 8) + + return Stat(path) http://git-wip-us.apache.org/repos/asf/ambari/blob/72dfdd49/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py index 64bcebc..c988bb2 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py @@ -91,14 +91,15 @@ def hive(name=None): "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p}") - check_schema_created_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " + check_schema_created_cmd = as_user(format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_bin}/schematool -info " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " - "-passWord {hive_metastore_user_passwd!p}") + "-passWord {hive_metastore_user_passwd!p}"), params.hive_user) Execute(create_schema_cmd, - not_if = check_schema_created_cmd + not_if = check_schema_created_cmd, + user = params.hive_user ) elif name == 'hiveserver2': File(params.start_hiveserver2_path, @@ -214,3 +215,7 @@ def jdbc_connector(): path=["/bin", "/usr/bin/"], sudo=True ) + + File(params.target, + mode = 0644, + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/72dfdd49/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py index 745402a..0d64644 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py @@ -20,6 +20,7 @@ limitations under the License. import os from resource_management import * +import getpass def setup_java(): """ @@ -64,6 +65,9 @@ def setup_java(): Execute(("chgrp","-R", params.user_group, params.java_home), sudo = True, ) + Execute(("chown","-R", getpass.getuser(), params.java_home), + sudo = True, + ) def install_packages(): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/72dfdd49/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py index 9153a84..9698bae 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py @@ -203,6 +203,9 @@ class TestHiveMetastore(RMFTestCase): path = ['/bin', '/usr/bin/'], sudo = True, ) + self.assertResourceCalled('File', '/usr/lib/hive/lib//mysql-connector-java.jar', + mode = 0644, + ) self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar', content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'), ) @@ -292,6 +295,9 @@ class TestHiveMetastore(RMFTestCase): path = ['/bin', '/usr/bin/'], sudo = True, ) + self.assertResourceCalled('File', '/usr/lib/hive/lib//mysql-connector-java.jar', + mode = 0644, + ) self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar', content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'), ) http://git-wip-us.apache.org/repos/asf/ambari/blob/72dfdd49/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py index 5230196..f0d6d7a 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py @@ -323,6 +323,9 @@ class TestHiveServer(RMFTestCase): path = ['/bin', '/usr/bin/'], sudo = True, ) + self.assertResourceCalled('File', '/usr/lib/hive/lib//mysql-connector-java.jar', + mode = 0644, + ) self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar', content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'), ) @@ -444,6 +447,9 @@ class TestHiveServer(RMFTestCase): path = ['/bin', '/usr/bin/'], sudo = True, ) + self.assertResourceCalled('File', '/usr/lib/hive/lib//mysql-connector-java.jar', + mode = 0644, + ) self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar', content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'), ) http://git-wip-us.apache.org/repos/asf/ambari/blob/72dfdd49/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py index e038ddf..1304eef 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py +++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py @@ -21,7 +21,9 @@ limitations under the License. from mock.mock import MagicMock, call, patch from resource_management import * from stacks.utils.RMFTestCase import * +import getpass [email protected](getpass, "getuser", new = MagicMock(return_value='some_user')) @patch.object(Hook, "run_custom_hook", new = MagicMock()) class TestHookBeforeInstall(RMFTestCase): def test_hook_default(self): @@ -57,4 +59,7 @@ class TestHookBeforeInstall(RMFTestCase): self.assertResourceCalled('Execute', ('chgrp', '-R', u'hadoop', u'/usr/jdk64/jdk1.7.0_45'), sudo = True, ) + self.assertResourceCalled('Execute', ('chown', '-R', 'some_user', u'/usr/jdk64/jdk1.7.0_45'), + sudo = True, + ) self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/72dfdd49/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py index 990eac8..ae0acec 100644 --- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py @@ -222,6 +222,9 @@ class TestHiveMetastore(RMFTestCase): path = ['/bin', '/usr/bin/'], sudo = True, ) + self.assertResourceCalled('File', '/usr/lib/hive/lib//mysql-connector-java.jar', + mode = 0644, + ) self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar', content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'), ) @@ -230,8 +233,9 @@ class TestHiveMetastore(RMFTestCase): mode = 0755, ) self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/lib/hive/bin/schematool -initSchema -dbType mysql -userName hive -passWord aaa', - not_if = 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/lib/hive/bin/schematool -info -dbType mysql -userName hive -passWord aaa', - ) + not_if = "ambari-sudo.sh su hive -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/lib/hive/bin/schematool -info -dbType mysql -userName hive -passWord aaa'", + user = 'hive', + ) self.assertResourceCalled('Directory', '/var/run/hive', owner = 'hive', group = 'hadoop', @@ -302,6 +306,9 @@ class TestHiveMetastore(RMFTestCase): path = ['/bin', '/usr/bin/'], sudo = True, ) + self.assertResourceCalled('File', '/usr/lib/hive/lib//mysql-connector-java.jar', + mode = 0644, + ) self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar', content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'), ) @@ -310,8 +317,9 @@ class TestHiveMetastore(RMFTestCase): mode = 0755, ) self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/lib/hive/bin/schematool -initSchema -dbType mysql -userName hive -passWord asd', - not_if = 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/lib/hive/bin/schematool -info -dbType mysql -userName hive -passWord asd', - ) + not_if = "ambari-sudo.sh su hive -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/lib/hive/bin/schematool -info -dbType mysql -userName hive -passWord asd'", + user = 'hive', + ) self.assertResourceCalled('Directory', '/var/run/hive', owner = 'hive', group = 'hadoop',
