Cross-stack upgrade, Oozie restart fails with ext-2.2.zip missing error, stack_tools.py is missing get_stack_name in __all__, disable BigInsights in UI (Alejandro Fernandez via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0ad9d587 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0ad9d587 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0ad9d587 Branch: refs/heads/branch-feature-AMBARI-21450 Commit: 0ad9d587d0d7cf88c53ae08c8ce9d7351be4a505 Parents: 421f3c6 Author: Sumit Mohanty <[email protected]> Authored: Fri Jul 21 12:07:34 2017 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Fri Jul 21 12:07:34 2017 -0700 ---------------------------------------------------------------------- .../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 40 +++++++++++++------- .../package/scripts/oozie_server_upgrade.py | 35 ++++++++++------- .../stacks/2.0.6/OOZIE/test_oozie_server.py | 32 ++++++++++++---- 3 files changed, 74 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0ad9d587/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py index 142e962..aa5bc30 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py @@ -215,18 +215,25 @@ def oozie_ownership(): group = params.user_group ) -def get_oozie_ext_zip_source_path(upgrade_type, params): +def get_oozie_ext_zip_source_paths(upgrade_type, params): """ - Get the Oozie ext zip file path from the source stack. + Get an ordered list of Oozie ext zip file paths from the source stack. :param upgrade_type: Upgrade type will be None if not in the middle of a stack upgrade. :param params: Expected to contain fields for ext_js_path, upgrade_direction, source_stack_name, and ext_js_file - :return: Source path to use for Oozie extension zip file + :return: Source paths to use for Oozie extension zip file """ # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip + paths = [] source_ext_js_path = params.ext_js_path + # Preferred location used by HDP and BigInsights 4.2.5 if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE: source_ext_js_path = "/usr/share/" + params.source_stack_name.upper() + "-oozie/" + params.ext_js_file - return source_ext_js_path + paths.append(source_ext_js_path) + + # Alternate location used by BigInsights 4.2.0 when migrating to another stack. + paths.append("/var/lib/oozie/" + params.ext_js_file) + + return paths def oozie_server_specific(upgrade_type): import params @@ -262,16 +269,23 @@ def oozie_server_specific(upgrade_type): ) configure_cmds = [] - # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip - source_ext_zip_path = get_oozie_ext_zip_source_path(upgrade_type, params) - - configure_cmds.append(('cp', source_ext_zip_path, params.oozie_libext_dir)) - configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}'))) + # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path + source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params) + + # Copy the first oozie ext-2.2.zip file that is found. + # This uses a list to handle the cases when migrating from some versions of BigInsights to HDP. + if source_ext_zip_paths is not None: + for source_ext_zip_path in source_ext_zip_paths: + if os.path.isfile(source_ext_zip_path): + configure_cmds.append(('cp', source_ext_zip_path, params.oozie_libext_dir)) + configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}'))) + + Execute(configure_cmds, + not_if=no_op_test, + sudo=True, + ) + break - Execute( configure_cmds, - not_if = no_op_test, - sudo = True, - ) Directory(params.oozie_webapps_conf_dir, owner = params.oozie_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/0ad9d587/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py index 719fb32..eb57c22 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py @@ -86,22 +86,31 @@ class OozieUpgrade(Script): raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) - # Copy ext ZIP to libext dir - # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip - source_ext_zip_path = oozie.get_oozie_ext_zip_source_path(upgrade_type, params) - # something like <stack-root>/current/oozie-server/libext/ext-2.2.zip oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, params.ext_js_file) - if not os.path.isfile(source_ext_zip_path): - raise Fail("Unable to copy {0} because it does not exist".format(source_ext_zip_path)) - - Logger.info("Copying {0} to {1}".format(source_ext_zip_path, params.oozie_libext_dir)) - Execute(("cp", source_ext_zip_path, params.oozie_libext_dir), sudo=True) - Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True) - File(oozie_ext_zip_target_path, - mode=0644 - ) + # Copy ext ZIP to libext dir + # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path + source_ext_zip_paths = oozie.get_oozie_ext_zip_source_paths(upgrade_type, params) + + found_at_least_one_oozie_ext_file = False + + # Copy the first oozie ext-2.2.zip file that is found. + # This uses a list to handle the cases when migrating from some versions of BigInsights to HDP. + if source_ext_zip_paths is not None: + for source_ext_zip_path in source_ext_zip_paths: + if os.path.isfile(source_ext_zip_path): + found_at_least_one_oozie_ext_file = True + Logger.info("Copying {0} to {1}".format(source_ext_zip_path, params.oozie_libext_dir)) + Execute(("cp", source_ext_zip_path, params.oozie_libext_dir), sudo=True) + Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True) + File(oozie_ext_zip_target_path, + mode=0644 + ) + break + + if not found_at_least_one_oozie_ext_file: + raise Fail("Unable to find any Oozie source extension files from the following paths {0}".format(source_ext_zip_paths)) # Redownload jdbc driver to a new current location oozie.download_database_library_if_needed() http://git-wip-us.apache.org/repos/asf/ambari/blob/0ad9d587/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py index 426c36a..5ef6ad9 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py @@ -44,7 +44,9 @@ class TestOozieServer(RMFTestCase): @patch.object(shell, "call") @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True])) - def test_configure_default(self, call_mocks): + @patch("os.path.isfile") + def test_configure_default(self, isfile_mock, call_mocks): + isfile_mock.return_value = True call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py", classname = "OozieServer", @@ -59,7 +61,10 @@ class TestOozieServer(RMFTestCase): @patch.object(shell, "call") @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, False, True])) - def test_configure_default_mysql(self, call_mocks): + @patch("os.path.isfile") + def test_configure_default_mysql(self, isfile_mock, iscall_mocks): + # Mock call when checking if need to copy oozie ext-2.2.zip file + isfile_mock.side_effect = [True, False] call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py", classname = "OozieServer", @@ -281,7 +286,10 @@ class TestOozieServer(RMFTestCase): @patch.object(shell, "call") @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, False, True])) - def test_configure_existing_sqla(self, call_mocks): + @patch("os.path.isfile") + def test_configure_existing_sqla(self, isfile_mock, call_mocks): + # Mock call when checking if need to copy oozie ext-2.2.zip file + isfile_mock.side_effect = [True, False] call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py", classname = "OozieServer", @@ -602,7 +610,10 @@ class TestOozieServer(RMFTestCase): @patch.object(shell, "call") @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True, True])) - def test_configure_secured(self, call_mocks): + @patch("os.path.isfile") + def test_configure_secured(self, isfile_mock, call_mocks): + # Mock call when checking if need to copy oozie ext-2.2.zip file + isfile_mock.side_effect = [True, False] call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py", classname = "OozieServer", @@ -616,8 +627,11 @@ class TestOozieServer(RMFTestCase): self.assertNoMoreResources() @patch.object(shell, "call") + @patch("os.path.isfile") @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True, True])) - def test_configure_secured_ha(self, call_mocks): + def test_configure_secured_ha(self, isfile_mock, call_mocks): + # Mock call when checking if need to copy oozie ext-2.2.zip file + isfile_mock.side_effects = [True, False] call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) config_file = "stacks/2.0.6/configs/secured.json" @@ -650,7 +664,8 @@ class TestOozieServer(RMFTestCase): @patch("os.path.isfile") @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True, True])) def test_start_secured(self, isfile_mock, call_mocks): - isfile_mock.return_value = True + # Mock call when checking if need to copy oozie ext-2.2.zip file + isfile_mock.side_effects = [True, False] call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py", classname = "OozieServer", @@ -1133,7 +1148,10 @@ class TestOozieServer(RMFTestCase): @patch.object(shell, "call") @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True])) - def test_configure_default_hdp22(self, call_mocks): + @patch("os.path.isfile") + def test_configure_default_hdp22(self, isfile_mock, call_mocks): + # Mock call when checking if need to copy oozie ext-2.2.zip file + isfile_mock.side_effect = [True, False] call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) config_file = "stacks/2.0.6/configs/default.json" with open(config_file, "r") as f:
