AMBARI-21463. Cross-stack upgrade, Oozie restart fails with ext-2.2.zip missing error, stack_tools.py is missing get_stack_name in __all__, disable BigInsights in UI (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d2c6d53f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d2c6d53f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d2c6d53f Branch: refs/heads/branch-feature-AMBARI-21348 Commit: d2c6d53f70bcaa6aee789e6d026cc06990acd16c Parents: 113b381 Author: Alejandro Fernandez <[email protected]> Authored: Wed Jul 12 16:53:12 2017 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Thu Jul 13 16:10:26 2017 -0700 ---------------------------------------------------------------------- .../libraries/functions/stack_tools.py | 2 +- .../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 26 ++++++++++++++++---- .../4.0.0.2.0/package/scripts/oozie_server.py | 4 +-- .../package/scripts/oozie_server_upgrade.py | 15 +++++------ .../4.0.0.2.0/package/scripts/params_linux.py | 15 ++++++++++- .../stacks/BigInsights/4.2.5/metainfo.xml | 2 +- 6 files changed, 47 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d2c6d53f/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py index 830598b..de58021 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py @@ -19,7 +19,7 @@ limitations under the License. ''' __all__ = ["get_stack_tool", "get_stack_tool_name", "get_stack_tool_path", - "get_stack_tool_package", "STACK_SELECTOR_NAME", "CONF_SELECTOR_NAME"] + "get_stack_tool_package", "get_stack_name", "STACK_SELECTOR_NAME", "CONF_SELECTOR_NAME"] # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. import ambari_simplejson as json http://git-wip-us.apache.org/repos/asf/ambari/blob/d2c6d53f/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py index 0c38b0b..142e962 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py @@ -52,7 +52,7 @@ from ambari_commons.inet_utils import download_file from resource_management.core import Logger @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) -def oozie(is_server=False): +def oozie(is_server=False, upgrade_type=None): import params from status_params import oozie_server_win_service_name @@ -99,7 +99,7 @@ def oozie(is_server=False): # TODO: see if see can remove this @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) -def oozie(is_server=False): +def oozie(is_server=False, upgrade_type=None): import params if is_server: @@ -190,7 +190,7 @@ def oozie(is_server=False): oozie_ownership() if is_server: - oozie_server_specific() + oozie_server_specific(upgrade_type) def oozie_ownership(): import params @@ -215,7 +215,20 @@ def oozie_ownership(): group = params.user_group ) -def oozie_server_specific(): +def get_oozie_ext_zip_source_path(upgrade_type, params): + """ + Get the Oozie ext zip file path from the source stack. + :param upgrade_type: Upgrade type will be None if not in the middle of a stack upgrade. + :param params: Expected to contain fields for ext_js_path, upgrade_direction, source_stack_name, and ext_js_file + :return: Source path to use for Oozie extension zip file + """ + # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip + source_ext_js_path = params.ext_js_path + if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE: + source_ext_js_path = "/usr/share/" + params.source_stack_name.upper() + "-oozie/" + params.ext_js_file + return source_ext_js_path + +def oozie_server_specific(upgrade_type): import params no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user) @@ -249,7 +262,10 @@ def oozie_server_specific(): ) configure_cmds = [] - configure_cmds.append(('cp', params.ext_js_path, params.oozie_libext_dir)) + # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip + source_ext_zip_path = get_oozie_ext_zip_source_path(upgrade_type, params) + + configure_cmds.append(('cp', source_ext_zip_path, params.oozie_libext_dir)) configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}'))) Execute( configure_cmds, http://git-wip-us.apache.org/repos/asf/ambari/blob/d2c6d53f/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py index 2cf3313..b3a8643 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py @@ -75,7 +75,7 @@ class OozieServer(Script): conf_select.select(params.stack_name, "oozie", params.version) env.set_params(params) - oozie(is_server=True) + oozie(is_server=True, upgrade_type=upgrade_type) def start(self, env, upgrade_type=None): import params @@ -129,7 +129,7 @@ class OozieServerDefault(OozieServer): conf_select.select(params.stack_name, "oozie", params.version) stack_select.select("oozie-server", params.version) - OozieUpgrade.prepare_libext_directory() + OozieUpgrade.prepare_libext_directory(upgrade_type=upgrade_type) def disable_security(self, env): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/d2c6d53f/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py index a1f3336..719fb32 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py @@ -41,7 +41,7 @@ BACKUP_CONF_ARCHIVE = "oozie-conf-backup.tar" class OozieUpgrade(Script): @staticmethod - def prepare_libext_directory(): + def prepare_libext_directory(upgrade_type=None): """ Performs the following actions on libext: - creates <stack-root>/current/oozie/libext and recursively @@ -86,17 +86,18 @@ class OozieUpgrade(Script): raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) - # copy ext ZIP to libext dir - oozie_ext_zip_file = params.ext_js_path + # Copy ext ZIP to libext dir + # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip + source_ext_zip_path = oozie.get_oozie_ext_zip_source_path(upgrade_type, params) # something like <stack-root>/current/oozie-server/libext/ext-2.2.zip oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, params.ext_js_file) - if not os.path.isfile(oozie_ext_zip_file): - raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file)) + if not os.path.isfile(source_ext_zip_path): + raise Fail("Unable to copy {0} because it does not exist".format(source_ext_zip_path)) - Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_dir)) - Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True) + Logger.info("Copying {0} to {1}".format(source_ext_zip_path, params.oozie_libext_dir)) + Execute(("cp", source_ext_zip_path, params.oozie_libext_dir), sudo=True) Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True) File(oozie_ext_zip_target_path, mode=0644 http://git-wip-us.apache.org/repos/asf/ambari/blob/d2c6d53f/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py index 0c30c78..110d55a 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py @@ -30,11 +30,11 @@ from resource_management.libraries.functions import get_port_from_url from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from resource_management.libraries.script.script import Script - from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.functions.expect import expect from resource_management.libraries.functions.get_architecture import get_architecture from resource_management.libraries.functions.stack_features import get_stack_feature_version +from resource_management.libraries.functions.stack_tools import get_stack_name from resource_management.core.utils import PasswordString from ambari_commons.credential_store_helper import get_password_from_credential_store @@ -65,6 +65,13 @@ agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_ret agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int) stack_root = status_params.stack_root + +# The source stack will be present during a cross-stack upgrade. +# E.g., BigInsights-4.2.5 or HDP-2.6 +source_stack = default("/commandParams/source_stack", None) +# This variable name is important, do not change +source_stack_name = get_stack_name(source_stack) + stack_version_unformatted = status_params.stack_version_unformatted stack_version_formatted = status_params.stack_version_formatted version_for_stack_feature_checks = get_stack_feature_version(config) @@ -142,8 +149,14 @@ hadoop_jar_location = "/usr/lib/hadoop/" java_share_dir = "/usr/share/java" java64_home = config['hostLevelParams']['java_home'] java_exec = format("{java64_home}/bin/java") + +# This variable name is important, do not change ext_js_file = "ext-2.2.zip" + +# During a cross-stack migration, the source location will be different +# This variable name is important, do not change ext_js_path = format("/usr/share/{stack_name_uppercase}-oozie/{ext_js_file}") + security_enabled = config['configurations']['cluster-env']['security_enabled'] oozie_heapsize = config['configurations']['oozie-env']['oozie_heapsize'] oozie_permsize = config['configurations']['oozie-env']['oozie_permsize'] http://git-wip-us.apache.org/repos/asf/ambari/blob/d2c6d53f/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/metainfo.xml index 3cf364e..789c63f 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/metainfo.xml @@ -17,7 +17,7 @@ --> <metainfo> <versions> - <active>true</active> + <active>false</active> </versions> <extends>4.2</extends> <minJdk>1.7</minJdk>
