AMBARI-12331. on non-root agent RU Oozie restart fails (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6ea0b83c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6ea0b83c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6ea0b83c Branch: refs/heads/branch-2.1 Commit: 6ea0b83c84bcd23a84577996d6e5b7e7a4af98ba Parents: 58fc42e Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Wed Jul 8 17:48:03 2015 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Wed Jul 8 17:48:03 2015 +0300 ---------------------------------------------------------------------- .../libraries/functions/tar_archive.py | 40 ++--- .../1.4.0.2.0/package/scripts/flume_upgrade.py | 5 +- .../0.5.0.2.2/package/scripts/knox_gateway.py | 6 +- .../package/scripts/oozie_server_upgrade.py | 23 +-- .../stacks/2.0.6/HDFS/test_hdfs_client.py | 3 + .../stacks/2.0.6/OOZIE/test_oozie_server.py | 124 +++++++++------ .../stacks/2.1/FALCON/test_falcon_server.py | 158 ++++++++++++++++++- .../python/stacks/2.2/KNOX/test_knox_gateway.py | 85 ++++++---- 8 files changed, 309 insertions(+), 135 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py index f1fb597..49caf88 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py @@ -18,16 +18,12 @@ See the License for the specific language governing permissions and limitations under the License. ''' -import os -import tarfile -from contextlib import closing +from resource_management.core.resources.system import Execute def archive_dir(output_filename, input_dir): - with closing(tarfile.open(output_filename, "w:gz")) as tar: - try: - tar.add(input_dir, arcname=os.path.basename(".")) - finally: - tar.close() + Execute(('tar', '-zcvf', output_filename, input_dir), + sudo = True, + ) def archive_directory_dereference(archive, directory): @@ -38,19 +34,15 @@ def archive_directory_dereference(archive, directory): :param directory: the directory to include :return: None """ - tarball = None - try: - # !!! dereference must be TRUE since the conf is a symlink and we want - # its contents instead of the actual symlink - tarball = tarfile.open(archive, mode="w", dereference=True) - - # tar the files, chopping off everything in front of directory - # /foo/bar/conf/a, /foo/bar/conf/b, /foo/bar/conf/dir/c - # becomes - # a - # b - # dir/c - tarball.add(directory, arcname=os.path.relpath(directory, start=directory)) - finally: - if tarball: - tarball.close() \ No newline at end of file + + Execute(('tar', '-zcvhf', archive, directory), + sudo = True, + ) + +def untar_archive(archive, directory): + """ + :param directory: can be a symlink and is followed + """ + Execute(('tar','-xvf',archive,'-C',directory+"/"), + sudo = True, + ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py index 32cc1b9..7743619 100644 --- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py +++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py @@ -70,10 +70,7 @@ def pre_start_restore(): directoryMappings[directory]) if os.path.isfile(archive): - Logger.info('Extracting {0} to {1}'.format(archive, directory)) - Execute(('tar','-xvf',archive,'-C',directory+"/"), - sudo = True, - ) + tar_archive.untar_archive(archive, directory) # cleanup if os.path.exists(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)): http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py index 38a55bc..8117312 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py @@ -21,7 +21,7 @@ import os import tarfile from resource_management.libraries.script.script import Script -from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import conf_select, tar_archive from resource_management.libraries.functions import hdp_select from resource_management.libraries.functions.check_process_status import check_process_status from resource_management.libraries.functions import format @@ -134,9 +134,7 @@ class KnoxGatewayDefault(KnoxGateway): sudo = True, ) - Execute(('tar','-xvf',conf_tar_source_path,'-C',extract_dir), - sudo = True, - ) + tar_archive.untar_archive(conf_tar_source_path, extract_dir) File(conf_tar_dest_path, action = "delete", http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py index fb19ddf..6225eae 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py @@ -25,7 +25,7 @@ import tempfile from resource_management.core import shell from resource_management.core.logger import Logger from resource_management.core.exceptions import Fail -from resource_management.core.resources.system import Execute +from resource_management.core.resources.system import Execute, Directory from resource_management.libraries.functions import Direction from resource_management.libraries.functions import format from resource_management.libraries.functions import compare_versions @@ -87,17 +87,12 @@ class OozieUpgrade(Script): Logger.info('Extracting {0} to {1}'.format(archive, directory)) - tarball = None - try: - tarball = tarfile.open(archive, "r") - tarball.extractall(directory) - finally: - if tarball: - tarball.close() + tar_archive.untar_archive(archive, directory) # cleanup - shutil.rmtree(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)) - + Directory(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR), + action="delete", + ) @staticmethod def prepare_libext_directory(): @@ -112,11 +107,9 @@ class OozieUpgrade(Script): target_version_needs_compression_libraries = compare_versions( format_hdp_stack_version(params.version), '2.2.1.0') >= 0 - if not os.path.isdir(params.oozie_libext_customer_dir): - os.makedirs(params.oozie_libext_customer_dir, 0o777) - - # ensure that it's rwx for all - os.chmod(params.oozie_libext_customer_dir, 0o777) + Directory(params.oozie_libext_customer_dir, + mode = 0777, + ) # get all hadooplzo* JAR files # hdp-select set hadoop-client has not run yet, therefore we cannot use http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py index 41fd771..06d94e9 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py @@ -64,6 +64,9 @@ class Test(RMFTestCase): self.assertResourceCalled('PropertiesFile', '/tmp/123/startup.properties', properties = UnknownConfigurationMock(), ) + self.assertResourceCalled('Execute', ('tar', '-zcvf', u'/tmp/HDFS_CLIENT-configs.tar.gz', '/tmp/123'), + sudo = True, + ) self.assertResourceCalled('Directory', '/tmp/123', action = ['delete'], ) http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py index 71744c3..39fd14b 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py @@ -26,6 +26,7 @@ from resource_management.core.exceptions import Fail from resource_management.libraries import functions from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil import hashlib +import tempfile md5_mock = MagicMock() md5_mock.hexdigest.return_value = "abc123hash" @@ -33,6 +34,7 @@ md5_mock.hexdigest.return_value = "abc123hash" @patch("platform.linux_distribution", new = MagicMock(return_value="Linux")) @patch.object(hashlib, "md5", new=MagicMock(return_value=md5_mock)) @patch.object(WebHDFSUtil, "run_command", new=MagicMock(return_value={})) +@patch.object(tempfile, "gettempdir", new=MagicMock(return_value="/tmp")) class TestOozieServer(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "OOZIE/4.0.0.2.0/package" STACK_VERSION = "2.0.6" @@ -913,17 +915,15 @@ class TestOozieServer(RMFTestCase): put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) - @patch("tarfile.open") @patch("os.path.isdir") @patch("os.path.exists") @patch("os.path.isfile") @patch("os.remove") - @patch("os.chmod") @patch("shutil.rmtree", new = MagicMock()) @patch("glob.iglob") @patch("shutil.copy2", new = MagicMock()) - def test_upgrade(self, glob_mock, chmod_mock, remove_mock, - isfile_mock, exists_mock, isdir_mock, tarfile_open_mock): + def test_upgrade(self, glob_mock, remove_mock, + isfile_mock, exists_mock, isdir_mock): def exists_mock_side_effect(path): if path == '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar': @@ -945,23 +945,7 @@ class TestOozieServer(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES, call_mocks = [(0, prepare_war_stdout)] ) - - # 2 calls to tarfile.open (1 directories, read + write) - self.assertTrue(tarfile_open_mock.called) - self.assertEqual(tarfile_open_mock.call_count,2) - - # check the call args for creation of the tarfile - call_args = tarfile_open_mock.call_args_list[0] - call_argument_tarfile = call_args[0][0] - call_kwargs = call_args[1] - - self.assertTrue("oozie-upgrade-backup/oozie-conf-backup.tar" in call_argument_tarfile) - self.assertEquals(True, call_kwargs["dereference"]) - - self.assertTrue(chmod_mock.called) - self.assertEqual(chmod_mock.call_count,1) - chmod_mock.assert_called_once_with('/usr/hdp/current/oozie-server/libext-customer', 511) - + self.assertTrue(isfile_mock.called) self.assertEqual(isfile_mock.call_count,3) isfile_mock.assert_called_with('/usr/share/HDP-oozie/ext-2.2.zip') @@ -970,20 +954,39 @@ class TestOozieServer(RMFTestCase): self.assertEqual(glob_mock.call_count,1) glob_mock.assert_called_with('/usr/hdp/2.2.1.0-2135/hadoop/lib/hadoop-lzo*.jar') - self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', '2.2.1.0-2135'), sudo=True,) + self.assertResourceCalled('Execute', ('tar', + '-zcvhf', + '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', + '/usr/hdp/current/oozie-server/conf/'), + sudo = True, + ) + self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', u'2.2.1.0-2135'), + sudo = True, + ) + self.assertResourceCalled('Execute', ('tar', + '-xvf', + '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', + '-C', + '/usr/hdp/current/oozie-server/conf//'), + sudo = True, + ) + self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup', + action = ['delete'], + ) + self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext-customer', + mode = 0777, + ) self.assertNoMoreResources() - @patch("tarfile.open") @patch("os.path.isdir") @patch("os.path.exists") @patch("os.path.isfile") @patch("os.remove") - @patch("os.chmod") @patch("shutil.rmtree", new = MagicMock()) @patch("glob.iglob") @patch("shutil.copy2", new = MagicMock()) - def test_upgrade_23(self, glob_mock, chmod_mock, remove_mock, - isfile_mock, exists_mock, isdir_mock, tarfile_open_mock): + def test_upgrade_23(self, glob_mock, remove_mock, + isfile_mock, exists_mock, isdir_mock): def exists_mock_side_effect(path): if path == '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar': @@ -1016,14 +1019,6 @@ class TestOozieServer(RMFTestCase): mocks_dict = mocks_dict ) - # 2 calls to tarfile.open (1 directories, read + write) - self.assertTrue(tarfile_open_mock.called) - self.assertEqual(tarfile_open_mock.call_count,2) - - self.assertTrue(chmod_mock.called) - self.assertEqual(chmod_mock.call_count,1) - chmod_mock.assert_called_once_with('/usr/hdp/current/oozie-server/libext-customer', 511) - self.assertTrue(isfile_mock.called) self.assertEqual(isfile_mock.call_count,3) isfile_mock.assert_called_with('/usr/share/HDP-oozie/ext-2.2.zip') @@ -1032,7 +1027,28 @@ class TestOozieServer(RMFTestCase): self.assertEqual(glob_mock.call_count,1) glob_mock.assert_called_with('/usr/hdp/2.3.0.0-1234/hadoop/lib/hadoop-lzo*.jar') - self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', '2.3.0.0-1234'), sudo=True) + self.assertResourceCalled('Execute', ('tar', + '-zcvhf', + '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', + '/usr/hdp/current/oozie-server/conf/'), + sudo = True, + ) + self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', '2.3.0.0-1234'), + sudo = True, + ) + self.assertResourceCalled('Execute', ('tar', + '-xvf', + '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', + '-C', + '/usr/hdp/current/oozie-server/conf//'), + sudo = True, + ) + self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup', + action = ['delete'], + ) + self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext-customer', + mode = 0777, + ) self.assertNoMoreResources() self.assertEquals(2, mocks_dict['call'].call_count) @@ -1045,16 +1061,14 @@ class TestOozieServer(RMFTestCase): mocks_dict['call'].call_args_list[0][0][0]) - @patch("tarfile.open") @patch("os.path.isdir") @patch("os.path.exists") @patch("os.path.isfile") @patch("os.remove") - @patch("os.chmod") @patch("shutil.rmtree", new = MagicMock()) @patch("shutil.copy2", new = MagicMock()) - def test_downgrade_no_compression_library_copy(self, chmod_mock, remove_mock, - isfile_mock, exists_mock, isdir_mock, tarfile_open_mock): + def test_downgrade_no_compression_library_copy(self, remove_mock, + isfile_mock, exists_mock, isdir_mock): isdir_mock.return_value = True exists_mock.return_value = False @@ -1069,19 +1083,31 @@ class TestOozieServer(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES, call_mocks = [(0, prepare_war_stdout)]) - # 2 calls to tarfile.open (1 directories, read + write) - self.assertTrue(tarfile_open_mock.called) - self.assertEqual(tarfile_open_mock.call_count,2) - - self.assertTrue(chmod_mock.called) - self.assertEqual(chmod_mock.call_count,1) - chmod_mock.assert_called_once_with('/usr/hdp/current/oozie-server/libext-customer', 511) - self.assertTrue(isfile_mock.called) self.assertEqual(isfile_mock.call_count,2) isfile_mock.assert_called_with('/usr/share/HDP-oozie/ext-2.2.zip') - - self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', '2.2.0.0-0000'), sudo=True) + self.assertResourceCalled('Execute', ('tar', + '-zcvhf', + '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', + '/usr/hdp/current/oozie-server/conf/'), + sudo = True, + ) + self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', u'2.2.0.0-0000'), + sudo = True, + ) + self.assertResourceCalled('Execute', ('tar', + '-xvf', + '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', + '-C', + '/usr/hdp/current/oozie-server/conf//'), + sudo = True, + ) + self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup', + action = ['delete'], + ) + self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext-customer', + mode = 0777, + ) self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py index 087b72c..ba2c026 100644 --- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py +++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py @@ -23,9 +23,10 @@ from mock.mock import MagicMock, patch import shutil from stacks.utils.RMFTestCase import * import tarfile - +import tempfile @patch("platform.linux_distribution", new = MagicMock(return_value="Linux")) +@patch.object(tempfile, "gettempdir", new=MagicMock(return_value="/tmp")) class TestFalconServer(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "FALCON/0.5.0.2.1/package" STACK_VERSION = "2.1" @@ -191,7 +192,7 @@ class TestFalconServer(RMFTestCase): tarfile_open_mock): isdir_mock.return_value = True - exists_mock.side_effect = [False,False,True] + exists_mock.side_effect = [False,False,True, True] isfile_mock.return_value = True self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", @@ -207,11 +208,159 @@ class TestFalconServer(RMFTestCase): self.assertResourceCalled('File', '/var/run/falcon/falcon.pid', action = ['delete']) - self.assertResourceCalled('Execute', ('hdp-select', 'set', 'falcon-server', '2.2.1.0-2135'), sudo=True) + self.assertResourceCalled('Execute', ('tar', + '-zcvhf', + '/tmp/falcon-upgrade-backup/falcon-conf-backup.tar', + '/usr/hdp/current/falcon-server/conf'), + sudo = True, + ) + self.assertResourceCalled('Execute', ('tar', + '-zcvhf', + '/tmp/falcon-upgrade-backup/falcon-local-backup.tar', + u'/hadoop/falcon'), + sudo = True, + ) + self.assertResourceCalled('Execute', ('hdp-select', 'set', 'falcon-server', u'2.2.1.0-2135'), + sudo = True, + ) + self.assertResourceCalled('Directory', '/var/run/falcon', + owner = 'falcon', + recursive = True, + ) + self.assertResourceCalled('Directory', '/var/log/falcon', + owner = 'falcon', + recursive = True, + ) + self.assertResourceCalled('Directory', '/usr/hdp/current/falcon-server/webapp', + owner = 'falcon', + recursive = True, + ) + self.assertResourceCalled('Directory', '/usr/hdp/current/falcon-server', + owner = 'falcon', + recursive = True, + ) + self.assertResourceCalled('Directory', '/etc/falcon', + recursive = True, + mode = 0755, + ) + self.assertResourceCalled('Directory', '/usr/hdp/current/falcon-server/conf', + owner = 'falcon', + recursive = True, + ) + self.assertResourceCalled('File', '/usr/hdp/current/falcon-server/conf/falcon-env.sh', + owner = 'falcon', + content = InlineTemplate(self.getConfig()['configurations']['falcon-env']['content']), + ) + self.assertResourceCalled('File', '/usr/hdp/current/falcon-server/conf/client.properties', + owner = 'falcon', + content = Template('client.properties.j2'), + mode = 0644, + ) + self.assertResourceCalled('PropertiesFile', '/usr/hdp/current/falcon-server/conf/runtime.properties', + owner = 'falcon', + mode = 0644, + properties = {u'*.domain': u'${falcon.app.type}', + u'*.log.cleanup.frequency.days.retention': u'days(7)', + u'*.log.cleanup.frequency.hours.retention': u'minutes(1)', + u'*.log.cleanup.frequency.minutes.retention': u'hours(6)', + u'*.log.cleanup.frequency.months.retention': u'months(3)'}, + ) + self.assertResourceCalled('PropertiesFile', '/usr/hdp/current/falcon-server/conf/startup.properties', + owner = 'falcon', + mode = 0644, + properties = self.getConfig()['configurations']['falcon-startup.properties'], + ) + self.assertResourceCalled('Directory', '/hadoop/falcon/data/lineage/graphdb', + owner = 'falcon', + recursive = True, + group = 'hadoop', + mode = 0775, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/hadoop/falcon/data/lineage', + owner = 'falcon', + recursive = True, + group = 'hadoop', + mode = 0775, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/hadoop/falcon/store', + owner = 'falcon', + recursive = True, + ) + self.assertResourceCalled('HdfsResource', '/apps/falcon', + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + owner = 'falcon', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + type = 'directory', + action = ['create_on_execute'], + mode = 0777, + ) + self.assertResourceCalled('Directory', '/hadoop/falcon/store', + owner = 'falcon', + recursive = True, + ) + self.assertResourceCalled('HdfsResource', '/apps/data-mirroring', + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + keytab = UnknownConfigurationMock(), + source = '/usr/hdp/current/falcon-server/data-mirroring', + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + user = 'hdfs', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + recursive_chmod = True, + recursive_chown = True, + owner = 'falcon', + group = 'users', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + type = 'directory', + action = ['create_on_execute'], + mode = 0770, + ) + self.assertResourceCalled('HdfsResource', None, + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + action = ['execute'], + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + ) + self.assertResourceCalled('Directory', '/hadoop/falcon', + owner = 'falcon', + recursive = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/hadoop/falcon/embeddedmq', + owner = 'falcon', + recursive = True, + ) + self.assertResourceCalled('Directory', '/hadoop/falcon/embeddedmq/data', + owner = 'falcon', + recursive = True, + ) + self.assertResourceCalled('Execute', '/usr/hdp/current/falcon-server/bin/falcon-start -port 15000', + environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client'}, + path = ['/usr/hdp/current/hadoop-client/bin'], + user = 'falcon', + ) + self.assertNoMoreResources() # 4 calls to tarfile.open (2 directories * read + write) self.assertTrue(tarfile_open_mock.called) - self.assertEqual(tarfile_open_mock.call_count,4) + self.assertEqual(tarfile_open_mock.call_count,2) # check the call args for creation of the tarfile call_args = tarfile_open_mock.call_args_list[0] @@ -219,7 +368,6 @@ class TestFalconServer(RMFTestCase): call_kwargs = call_args[1] self.assertTrue("falcon-upgrade-backup/falcon-conf-backup.tar" in call_argument_tarfile) - self.assertEquals(True, call_kwargs["dereference"]) @patch("resource_management.libraries.functions.security_commons.build_expectations") @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem") http://git-wip-us.apache.org/repos/asf/ambari/blob/6ea0b83c/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py index 0c4b292..5ca5251 100644 --- a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py +++ b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py @@ -18,31 +18,13 @@ See the License for the specific language governing permissions and limitations under the License. ''' import json +import tempfile from resource_management import * from stacks.utils.RMFTestCase import * from mock.mock import patch from mock.mock import MagicMock - -class TarfileFake: - """ - Dummy class to mock the calls to tarfile module. - """ - def __init__(self): - self.extractall_count = 0 - self.add_count = 0 - self.close_count = 0 - - def extractall(self, path=".", members=None): - self.extractall_count += 1 - - def add(self, name, arcname=None, recursive=True, exclude=None, filter=None): - self.add_count += 1 - - def close(self): - self.close_count += 1 - - +@patch.object(tempfile, "gettempdir", new=MagicMock(return_value="/tmp")) @patch("platform.linux_distribution", new = MagicMock(return_value="Linux")) class TestKnoxGateway(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "KNOX/0.5.0.2.2/package" @@ -238,9 +220,8 @@ class TestKnoxGateway(RMFTestCase): ) put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) - @patch("tarfile.open") @patch("os.path.isdir") - def test_pre_rolling_restart(self, isdir_mock, tarfile_open_mock): + def test_pre_rolling_restart(self, isdir_mock): isdir_mock.return_value = True config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/knox_upgrade.json" with open(config_file, "r") as f: @@ -255,16 +236,28 @@ class TestKnoxGateway(RMFTestCase): hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES) - self.assertTrue(tarfile_open_mock.called) - - self.assertResourceCalled("Execute", ('hdp-select', 'set', 'knox-server', version), sudo=True) + self.assertResourceCalled('Execute', ('tar', + '-zcvhf', + '/tmp/knox-upgrade-backup/knox-conf-backup.tar', + '/usr/hdp/current/knox-server/conf/'), + sudo = True, + ) + self.assertResourceCalled('Execute', ('tar', + '-zcvhf', + '/tmp/knox-upgrade-backup/knox-data-backup.tar', + '/var/lib/knox/data'), + sudo = True, + ) + self.assertResourceCalled('Execute', ('hdp-select', 'set', 'knox-server', '2.2.1.0-3242'), + sudo = True, + ) + self.assertNoMoreResources() @patch("os.remove") @patch("os.path.exists") - @patch("tarfile.open") @patch("os.path.isdir") @patch("resource_management.core.shell.call") - def test_pre_rolling_restart_23(self, call_mock, isdir_mock, tarfile_open_mock, path_exists_mock, remove_mock): + def test_pre_rolling_restart_23(self, call_mock, isdir_mock, path_exists_mock, remove_mock): isdir_mock.return_value = True config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/knox_upgrade.json" with open(config_file, "r") as f: @@ -273,8 +266,6 @@ class TestKnoxGateway(RMFTestCase): json_content['commandParams']['version'] = version path_exists_mock.return_value = True - knox_conf_tarfile = TarfileFake() - tarfile_open_mock.return_value = knox_conf_tarfile mocks_dict = {} self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py", @@ -286,9 +277,37 @@ class TestKnoxGateway(RMFTestCase): call_mocks = [(0, None), (0, None)], mocks_dict = mocks_dict) - self.assertTrue(tarfile_open_mock.called) - - self.assertResourceCalled("Execute", ('hdp-select', 'set', 'knox-server', version), sudo=True) + self.assertResourceCalled('Execute', ('tar', + '-zcvhf', + '/tmp/knox-upgrade-backup/knox-conf-backup.tar', + '/usr/hdp/current/knox-server/conf/'), + sudo = True, + ) + self.assertResourceCalled('Execute', ('tar', + '-zcvhf', + '/tmp/knox-upgrade-backup/knox-data-backup.tar', + '/var/lib/knox/data'), + sudo = True, + ) + self.assertResourceCalled('Execute', ('hdp-select', 'set', 'knox-server', '2.3.0.0-1234'), + sudo = True, + ) + self.assertResourceCalled('Execute', ('cp', + '/tmp/knox-upgrade-backup/knox-conf-backup.tar', + '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar'), + sudo = True, + ) + self.assertResourceCalled('Execute', ('tar', + '-xvf', + '/tmp/knox-upgrade-backup/knox-conf-backup.tar', + '-C', + '/usr/hdp/current/knox-server/conf/'), + sudo = True, + ) + self.assertResourceCalled('File', '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar', + action = ['delete'], + ) + self.assertNoMoreResources() self.assertEquals(1, mocks_dict['call'].call_count) self.assertEquals(1, mocks_dict['checked_call'].call_count) @@ -299,8 +318,6 @@ class TestKnoxGateway(RMFTestCase): ('conf-select', 'create-conf-dir', '--package', 'knox', '--stack-version', '2.3.0.0-1234', '--conf-version', '0'), mocks_dict['call'].call_args_list[0][0][0]) - self.assertTrue(2, knox_conf_tarfile.close_count) - @patch("os.path.islink") @patch("os.path.realpath") @patch("os.unlink")
