Repository: ambari Updated Branches: refs/heads/trunk b60e30561 -> a8dd52b19
AMBARI-14472: Stack Featurize Oozie Service (Juanjo Marron via jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a8dd52b1 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a8dd52b1 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a8dd52b1 Branch: refs/heads/trunk Commit: a8dd52b19aa440d8804d16e73bf445a2851d4674 Parents: b60e305 Author: Jayush Luniya <[email protected]> Authored: Mon Apr 4 10:28:33 2016 -0700 Committer: Jayush Luniya <[email protected]> Committed: Mon Apr 4 10:28:33 2016 -0700 ---------------------------------------------------------------------- .../libraries/functions/constants.py | 5 ++ .../libraries/functions/stack_features.py | 26 ++++++++++ .../package/alerts/alert_check_oozie_server.py | 8 ++- .../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 7 +-- .../4.0.0.2.0/package/scripts/oozie_client.py | 9 ++-- .../4.0.0.2.0/package/scripts/oozie_server.py | 21 ++++---- .../package/scripts/oozie_server_upgrade.py | 28 +++++------ .../4.0.0.2.0/package/scripts/params_linux.py | 51 ++++++++++---------- .../4.0.0.2.0/package/scripts/status_params.py | 11 ++++- .../HDP/2.0.6/properties/stack_features.json | 26 ++++++++++ 10 files changed, 132 insertions(+), 60 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a8dd52b1/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index 175b889..f7d817a 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -40,6 +40,7 @@ class StackFeature: Stack Feature supported """ SNAPPY = "snappy" + LZO = "lzo" EXPRESS_UPGRADE = "express_upgrade" ROLLING_UPGRADE = "rolling_upgrade" CONFIG_VERSIONING = "config_versioning" @@ -64,3 +65,7 @@ class StackFeature: KNOX_VERSIONED_DATA_DIR = "knox_versioned_data_dir" KNOX_SSO_TOPOLOGY = "knox_sso_topology" ATLAS_ROLLING_UPGRADE = "atlas_rolling_upgrade" + OOZIE_ADMIN_USER = "oozie_admin_user" + OOZIE_CREATE_HIVE_TEZ_CONFIGS = "oozie_create_hive_tez_configs" + OOZIE_SETUP_SHARED_LIB = "oozie_setup_shared_lib" + OOZIE_HOST_KERBEROS = "oozie_host_kerberos" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/a8dd52b1/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py index 6169699..41508b4 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py @@ -33,6 +33,11 @@ _DEFAULT_STACK_FEATURES = { "max_version": "2.2.0.0" }, { + "name": "lzo", + "description": "LZO libraries support", + "min_version": "2.2.1.0" + }, + { "name": "express_upgrade", "description": "Express upgrade support", "min_version": "2.1.0.0" @@ -153,6 +158,27 @@ _DEFAULT_STACK_FEATURES = { "name": "atlas_rolling_upgrade", "description": "Rolling upgrade support for Atlas", "min_version": "2.3.0.0" + }, + { + "name": "oozie_admin_user", + "description": "Oozie install user as an Oozie admin user (AMBARI-7976)", + "min_version": "2.2.0.0" + }, + { + "name": "oozie_create_hive_tez_configs", + "description": "Oozie create configs for Ambari Hive and Tez deployments (AMBARI-8074)", + "min_version": "2.2.0.0" + }, + { + "name": "oozie_setup_shared_lib", + "description": "Oozie setup tools used to shared Oozie lib to HDFS (AMBARI-7240)", + "min_version": "2.2.0.0" + }, + { + "name": "oozie_host_kerberos", + "description": "Oozie in secured clusters uses _HOST in Kerberos principal (AMBARI-9775)", + "min_version": "2.0.0.0", + "max_version": "2.2.0.0" } ] } http://git-wip-us.apache.org/repos/asf/ambari/blob/a8dd52b1/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py index 90851c8..381c43c 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py @@ -30,6 +30,10 @@ from ambari_commons.os_check import OSConst, OSCheck from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from urlparse import urlparse +import params + +stack_root = params.stack_root + RESULT_CODE_OK = 'OK' RESULT_CODE_CRITICAL = 'CRITICAL' RESULT_CODE_UNKNOWN = 'UNKNOWN' @@ -45,7 +49,7 @@ KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY = '{{kerberos-env/executable_search_paths}} OOZIE_URL_KEY = '{{oozie-site/oozie.base.url}}' SECURITY_ENABLED = '{{cluster-env/security_enabled}}' OOZIE_USER = '{{oozie-env/oozie_user}}' -OOZIE_CONF_DIR = '/usr/hdp/current/oozie-server/conf' +OOZIE_CONF_DIR = format("{stack_root}/current/oozie-server/conf") OOZIE_CONF_DIR_LEGACY = '/etc/oozie/conf' OOZIE_HTTPS_PORT = '{{oozie-site/oozie.https.port}}' OOZIE_ENV_CONTENT = '{{oozie-env/content}}' @@ -152,7 +156,7 @@ def get_check_command(oozie_url, host_name, configurations, parameters, only_kin finally: kinit_lock.release() - # oozie configuration directory uses a symlink when > HDP 2.2 + # oozie configuration directory using a symlink oozie_config_directory = OOZIE_CONF_DIR_LEGACY if os.path.exists(OOZIE_CONF_DIR): oozie_config_directory = OOZIE_CONF_DIR http://git-wip-us.apache.org/repos/asf/ambari/blob/a8dd52b1/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py index 3b01802..d69339e 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py @@ -25,7 +25,8 @@ from resource_management.core.source import DownloadSource from resource_management.core.source import InlineTemplate from resource_management.core.source import Template from resource_management.libraries.functions.format import format -from resource_management.libraries.functions.version import compare_versions +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.oozie_prepare_war import prepare_war from resource_management.libraries.resources.xml_config import XmlConfig from resource_management.libraries.script.script import Script @@ -144,7 +145,7 @@ def oozie(is_server=False): owner=params.oozie_user ) - if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.2') >= 0: + if params.stack_version_formatted and check_stack_feature(StackFeature.OOZIE_ADMIN_USER, params.stack_version_formatted): File(format("{params.conf_dir}/adminusers.txt"), mode=0644, group=params.user_group, @@ -268,7 +269,7 @@ def oozie_server_specific(): mode = 0644, ) - if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.2') >= 0: + if params.stack_version_formatted and check_stack_feature(StackFeature.OOZIE_CREATE_HIVE_TEZ_CONFIGS, params.stack_version_formatted): # Create hive-site and tez-site configs for oozie Directory(params.hive_conf_dir, create_parents = True, http://git-wip-us.apache.org/repos/asf/ambari/blob/a8dd52b1/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_client.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_client.py index 4fc50d2..dc08f0b 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_client.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_client.py @@ -22,6 +22,8 @@ import sys from resource_management import * from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from oozie import oozie from oozie_service import oozie_service @@ -30,7 +32,8 @@ from oozie_service import oozie_service class OozieClient(Script): def get_stack_to_component(self): - return {"HDP": "oozie-client"} + import params + return {params.stack_name: "oozie-client"} def install(self, env): self.install_packages(env) @@ -52,8 +55,8 @@ class OozieClient(Script): env.set_params(params) # this function should not execute if the version can't be determined or - # is not at least HDP 2.2.0.0 - if not params.version or compare_versions(format_stack_version(params.version), '2.2.0.0') < 0: + # the stack does not support rolling upgrade + if not (params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version)): return Logger.info("Executing Oozie Client Stack Upgrade pre-restart") http://git-wip-us.apache.org/repos/asf/ambari/blob/a8dd52b1/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py index e9da71b..c7107c9 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py @@ -20,10 +20,10 @@ limitations under the License. from resource_management.core import Logger from resource_management.libraries.script import Script -from resource_management.libraries.functions import compare_versions from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions import format_stack_version +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.format import format from resource_management.libraries.functions import default from resource_management.libraries.functions.constants import Direction @@ -47,7 +47,8 @@ from check_oozie_server_status import check_oozie_server_status class OozieServer(Script): def get_stack_to_component(self): - return {"HDP": "oozie-server"} + import params + return {params.stack_name: "oozie-server"} def install(self, env): self.install_packages(env) @@ -65,17 +66,17 @@ class OozieServer(Script): if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and params.version is not None: Logger.info(format("Configuring Oozie during upgrade type: {upgrade_type}, direction: {params.upgrade_direction}, and version {params.version}")) - if compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): # In order for the "<stack-root>/current/oozie-<client/server>" point to the new version of # oozie, we need to create the symlinks both for server and client. # This is required as both need to be pointing to new installed oozie version. - # Sets the symlink : eg: <stack-root>/current/oozie-client -> <stack-root>/2.3.x.y-<version>/oozie + # Sets the symlink : eg: <stack-root>/current/oozie-client -> <stack-root>/a.b.c.d-<version>/oozie stack_select.select("oozie-client", params.version) - # Sets the symlink : eg: <stack-root>/current/oozie-server -> <stack-root>/2.3.x.y-<version>/oozie + # Sets the symlink : eg: <stack-root>/current/oozie-server -> <stack-root>/a.b.c.d-<version>/oozie stack_select.select("oozie-server", params.version) - if compare_versions(format_stack_version(params.version), '2.3.0.0') >= 0: + if params.version and check_stack_feature(StackFeature.CONFIG_VERSIONING, params.version): conf_select.select(params.stack_name, "oozie", params.version) env.set_params(params) @@ -186,15 +187,15 @@ class OozieServerDefault(OozieServer): env.set_params(params) # this function should not execute if the version can't be determined or - # is not at least HDP 2.2.0.0 - if not params.version or compare_versions(format_stack_version(params.version), '2.2.0.0') < 0: + # the stack does not support rolling upgrade + if not (params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version)): return Logger.info("Executing Oozie Server Stack Upgrade pre-restart") OozieUpgrade.backup_configuration() - if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): conf_select.select(params.stack_name, "oozie", params.version) stack_select.select("oozie-server", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/a8dd52b1/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py index 2db3672..28d2991 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py @@ -29,12 +29,12 @@ from resource_management.core.resources.system import Directory from resource_management.core.resources.system import File from resource_management.libraries.functions import Direction from resource_management.libraries.functions import format -from resource_management.libraries.functions import compare_versions from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions import format_stack_version from resource_management.libraries.functions import tar_archive from resource_management.libraries.functions.oozie_prepare_war import prepare_war from resource_management.libraries.script.script import Script +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature import oozie @@ -107,9 +107,8 @@ class OozieUpgrade(Script): """ import params - # some versions of HDP don't need the lzo compression libraries - target_version_needs_compression_libraries = compare_versions( - format_stack_version(params.version), '2.2.1.0') >= 0 + # some stack versions don't need the lzo compression libraries + target_version_needs_compression_libraries = params.version and check_stack_feature(StackFeature.LZO, params.version) # ensure the directory exists Directory(params.oozie_libext_dir, mode = 0777) @@ -120,11 +119,11 @@ class OozieUpgrade(Script): # however, this only works when upgrading beyond 2.2.0.0; don't do this # for downgrade to 2.2.0.0 since hadoop-lzo will not be present # This can also be called during a Downgrade. - # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages + # When a version is Installed, it is responsible for downloading the hadoop-lzo packages # if lzo is enabled. if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries): hadoop_lzo_pattern = 'hadoop-lzo*.jar' - hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib") + hadoop_client_new_lib_dir = format("{stack_root}/{version}/hadoop/lib") files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern)) if not files: @@ -144,10 +143,10 @@ class OozieUpgrade(Script): hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy ext ZIP to libext dir - oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip' + oozie_ext_zip_file = params.ext_js_path # something like <stack-root>/current/oozie-server/libext/ext-2.2.zip - oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, "ext-2.2.zip") + oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, params.ext_js_file) if not os.path.isfile(oozie_ext_zip_file): raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file)) @@ -167,13 +166,12 @@ class OozieUpgrade(Script): if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None: raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.") - # something like 2.3.0.0-1234 stack_version = upgrade_stack[1] # copy the Falcon JAR if needed; falcon has not upgraded yet, so we must # use the versioned falcon directory if params.has_falcon_host: - versioned_falcon_jar_directory = "/usr/hdp/{0}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(stack_version) + versioned_falcon_jar_directory = "{0}/{1}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(params.stack_root, stack_version) Logger.info("Copying {0} to {1}".format(versioned_falcon_jar_directory, params.oozie_libext_dir)) Execute(format('{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}')) @@ -239,10 +237,10 @@ class OozieUpgrade(Script): # the database upgrade requires the db driver JAR, but since we have # not yet run <stack-selector-tool> to upgrade the current points, we have to use # the versioned libext directory as the location[[-vufdtffr, - versioned_libext_dir = "/usr/hdp/{0}/oozie/libext".format(stack_version) + versioned_libext_dir = "{0}/{1}/oozie/libext".format(params.stack_root, stack_version) oozie.download_database_library_if_needed(target_directory=versioned_libext_dir) - database_upgrade_command = "/usr/hdp/{0}/oozie/bin/ooziedb.sh upgrade -run".format(stack_version) + database_upgrade_command = "{0}/{1}/oozie/bin/ooziedb.sh upgrade -run".format(params.stack_root, stack_version) Execute(database_upgrade_command, user=params.oozie_user, logoutput=True) # install new sharelib to HDFS @@ -286,8 +284,8 @@ class OozieUpgrade(Script): stack_version = upgrade_stack[1] # install new sharelib to HDFS - sharelib_command = "/usr/hdp/{0}/oozie/bin/oozie-setup.sh sharelib create -fs {1}".format( - stack_version, params.fs_root) + sharelib_command = "{0}/{1}/oozie/bin/oozie-setup.sh sharelib create -fs {2}".format( + params.stack_root, stack_version, params.fs_root) Execute(sharelib_command, user=params.oozie_user, logoutput=True) http://git-wip-us.apache.org/repos/asf/ambari/blob/a8dd52b1/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py index ce44d5c..77dbc81 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py @@ -22,7 +22,8 @@ from ambari_commons.constants import AMBARI_SUDO_BINARY from resource_management.libraries.functions import format from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.default import default from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions import get_port_from_url @@ -47,20 +48,21 @@ hostname = config["hostname"] # New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade version = default("/commandParams/version", None) stack_name = default("/hostLevelParams/stack_name", None) +stack_name_uppercase = stack_name.upper() upgrade_direction = default("/commandParams/upgrade_direction", None) agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability'] agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int) -stack_version_unformatted = config['hostLevelParams']['stack_version'] -stack_version_formatted = format_stack_version(stack_version_unformatted) +stack_root = status_params.stack_root +stack_version_unformatted = status_params.stack_version_unformatted +stack_version_formatted = status_params.stack_version_formatted hadoop_conf_dir = conf_select.get_hadoop_conf_dir() hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_lib_home = stack_select.get_hadoop_dir("lib") #hadoop params -if Script.is_stack_greater_or_equal("2.2"): - # something like 2.3.0.0-1234 +if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE,stack_version_formatted): stack_version = None upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is not None and len(upgrade_stack) == 2 and upgrade_stack[1] is not None: @@ -70,24 +72,24 @@ if Script.is_stack_greater_or_equal("2.2"): oozie_root = status_params.component_directory # using the correct oozie root dir, format the correct location - oozie_lib_dir = format("/usr/hdp/current/{oozie_root}") - oozie_setup_sh = format("/usr/hdp/current/{oozie_root}/bin/oozie-setup.sh") - oozie_webapps_dir = format("/usr/hdp/current/{oozie_root}/oozie-server/webapps") - oozie_webapps_conf_dir = format("/usr/hdp/current/{oozie_root}/oozie-server/conf") - oozie_libext_dir = format("/usr/hdp/current/{oozie_root}/libext") - oozie_server_dir = format("/usr/hdp/current/{oozie_root}/oozie-server") - oozie_shared_lib = format("/usr/hdp/current/{oozie_root}/share") - oozie_home = format("/usr/hdp/current/{oozie_root}") - oozie_bin_dir = format("/usr/hdp/current/{oozie_root}/bin") - oozie_examples_regex = format("/usr/hdp/current/{oozie_root}/doc") + oozie_lib_dir = format("{stack_root}/current/{oozie_root}") + oozie_setup_sh = format("{stack_root}/current/{oozie_root}/bin/oozie-setup.sh") + oozie_webapps_dir = format("{stack_root}/current/{oozie_root}/oozie-server/webapps") + oozie_webapps_conf_dir = format("{stack_root}/current/{oozie_root}/oozie-server/conf") + oozie_libext_dir = format("{stack_root}/current/{oozie_root}/libext") + oozie_server_dir = format("{stack_root}/current/{oozie_root}/oozie-server") + oozie_shared_lib = format("{stack_root}/current/{oozie_root}/share") + oozie_home = format("{stack_root}/current/{oozie_root}") + oozie_bin_dir = format("{stack_root}/current/{oozie_root}/bin") + oozie_examples_regex = format("{stack_root}/current/{oozie_root}/doc") # set the falcon home for copying JARs; if in an upgrade, then use the version of falcon that # matches the version of oozie - falcon_home = '/usr/hdp/current/falcon-client' + falcon_home = format("{stack_root}/current/falcon-client") if stack_version is not None: - falcon_home = '/usr/hdp/{0}/falcon'.format(stack_version) + falcon_home = '{0}/{1}/falcon'.format(stack_root, stack_version) - conf_dir = format("/usr/hdp/current/{oozie_root}/conf") + conf_dir = format("{stack_root}/current/{oozie_root}/conf") hive_conf_dir = format("{conf_dir}/action-conf/hive") else: @@ -124,9 +126,8 @@ oozie_pid_dir = status_params.oozie_pid_dir pid_file = status_params.pid_file hadoop_jar_location = "/usr/lib/hadoop/" java_share_dir = "/usr/share/java" -# for HDP1 it's "/usr/share/HDP-oozie/ext.zip" ext_js_file = "ext-2.2.zip" -ext_js_path = format("/usr/share/HDP-oozie/{ext_js_file}") +ext_js_path = format("/usr/share/{stack_name_uppercase}-oozie/{ext_js_file}") security_enabled = config['configurations']['cluster-env']['security_enabled'] oozie_heapsize = config['configurations']['oozie-env']['oozie_heapsize'] oozie_permsize = config['configurations']['oozie-env']['oozie_permsize'] @@ -144,7 +145,7 @@ oozie_site = config['configurations']['oozie-site'] # Need this for yarn.nodemanager.recovery.dir in yarn-site yarn_log_dir_prefix = config['configurations']['yarn-env']['yarn_log_dir_prefix'] -if security_enabled and Script.is_stack_less_than("2.2"): +if security_enabled and stack_version_formatted and check_stack_feature(StackFeature.OOZIE_HOST_KERBEROS, stack_version_formatted): #older versions of oozie have problems when using _HOST in principal oozie_site = dict(config['configurations']['oozie-site']) oozie_site['oozie.service.HadoopAccessorService.kerberos.principal'] = \ @@ -196,11 +197,11 @@ oozie_setup_sh_current = oozie_setup_sh hdfs_site = config['configurations']['hdfs-site'] fs_root = config['configurations']['core-site']['fs.defaultFS'] -if Script.is_stack_less_than("2.2"): - put_shared_lib_to_hdfs_cmd = format("hadoop --config {hadoop_conf_dir} dfs -put {oozie_shared_lib} {oozie_hdfs_user_dir}") -# for newer -else: +if stack_version_formatted and check_stack_feature(StackFeature.OOZIE_SETUP_SHARED_LIB, stack_version_formatted): put_shared_lib_to_hdfs_cmd = format("{oozie_setup_sh} sharelib create -fs {fs_root} -locallib {oozie_shared_lib}") + # for older +else: + put_shared_lib_to_hdfs_cmd = format("hadoop --config {hadoop_conf_dir} dfs -put {oozie_shared_lib} {oozie_hdfs_user_dir}") jdbc_driver_name = default("/configurations/oozie-site/oozie.service.JPAService.jdbc.driver", "") # NOT SURE THAT IT'S A GOOD IDEA TO USE PATH TO CLASS IN DRIVER, MAYBE IT WILL BE BETTER TO USE DB TYPE. http://git-wip-us.apache.org/repos/asf/ambari/blob/a8dd52b1/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/status_params.py index a08ae3a..5790c54 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/status_params.py @@ -23,6 +23,9 @@ from resource_management.libraries.functions import format from resource_management.libraries.functions.default import default from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.script.script import Script +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.version import format_stack_version # a map of the Ambari role to the component name # for use with <stack-root>/current/<component> @@ -36,6 +39,10 @@ SERVER_ROLE_DIRECTORY_MAP = { component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "OOZIE_CLIENT") config = Script.get_config() +stack_root = Script.get_stack_root() + +stack_version_unformatted = config['hostLevelParams']['stack_version'] +stack_version_formatted = format_stack_version(stack_version_unformatted) if OSCheck.is_windows_family(): # windows service mapping @@ -48,8 +55,8 @@ else: kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) conf_dir = "/etc/oozie/conf" - if Script.is_stack_greater_or_equal("2.2"): - conf_dir = format("/usr/hdp/current/{component_directory}/conf") + if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): + conf_dir = format("{stack_root}/current/{component_directory}/conf") tmp_dir = Script.get_tmp_dir() oozie_user = config['configurations']['oozie-env']['oozie_user'] http://git-wip-us.apache.org/repos/asf/ambari/blob/a8dd52b1/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index fd3fbde..7a3c12e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -7,6 +7,11 @@ "max_version": "2.2.0.0" }, { + "name": "lzo", + "description": "LZO libraries support", + "min_version": "2.2.1.0" + }, + { "name": "express_upgrade", "description": "Express upgrade support", "min_version": "2.1.0.0" @@ -127,6 +132,27 @@ "name": "atlas_rolling_upgrade", "description": "Rolling upgrade support for Atlas", "min_version": "2.3.0.0" + }, + { + "name": "oozie_admin_user", + "description": "Oozie install user as an Oozie admin user (AMBARI-7976)", + "min_version": "2.2.0.0" + }, + { + "name": "oozie_create_hive_tez_configs", + "description": "Oozie create configs for Ambari Hive and Tez deployments (AMBARI-8074)", + "min_version": "2.2.0.0" + }, + { + "name": "oozie_setup_shared_lib", + "description": "Oozie setup tools used to shared Oozie lib to HDFS (AMBARI-7240)", + "min_version": "2.2.0.0" + }, + { + "name": "oozie_host_kerberos", + "description": "Oozie in secured clusters uses _HOST in Kerberos principal (AMBARI-9775)", + "min_version": "2.0.0.0", + "max_version": "2.2.0.0" } ] }
