Repository: ambari Updated Branches: refs/heads/branch-2.4 01b59ac28 -> 9406f54f2
AMBARI-17037. Falcon server fails to start, HDP 2.4 to use data-mirroring directory, HDP 2.5 to use extensions (alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9406f54f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9406f54f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9406f54f Branch: refs/heads/branch-2.4 Commit: 9406f54f2107c09b10fcb65843436679884c5d28 Parents: 01b59ac Author: Alejandro Fernandez <afernan...@hortonworks.com> Authored: Mon Jun 6 13:08:55 2016 -0700 Committer: Alejandro Fernandez <afernan...@hortonworks.com> Committed: Mon Jun 6 13:17:06 2016 -0700 ---------------------------------------------------------------------- .../libraries/functions/constants.py | 1 + .../libraries/functions/hdfs_utils.py | 2 +- .../libraries/functions/stack_features.py | 3 +++ .../FALCON/0.5.0.2.1/package/scripts/falcon.py | 15 ++++++++++++++- .../0.5.0.2.1/package/scripts/falcon_server.py | 18 ++++++++++++++++++ .../0.5.0.2.1/package/scripts/params_linux.py | 19 +++++++++++++++++++ .../HDP/2.0.6/properties/stack_features.json | 5 +++++ .../python/stacks/2.0.6/configs/default.json | 3 ++- 8 files changed, 63 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9406f54f/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index 34d1a1a..555a215 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -44,6 +44,7 @@ class StackFeature: EXPRESS_UPGRADE = "express_upgrade" ROLLING_UPGRADE = "rolling_upgrade" CONFIG_VERSIONING = "config_versioning" + FALCON_EXTENSIONS = "falcon_extensions" DATANODE_NON_ROOT = "datanode_non_root" REMOVE_RANGER_HDFS_PLUGIN_ENV = "remove_ranger_hdfs_plugin_env" RANGER = "ranger" http://git-wip-us.apache.org/repos/asf/ambari/blob/9406f54f/ambari-common/src/main/python/resource_management/libraries/functions/hdfs_utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/hdfs_utils.py b/ambari-common/src/main/python/resource_management/libraries/functions/hdfs_utils.py index 06bb65f..4018e13 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/hdfs_utils.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/hdfs_utils.py @@ -19,7 +19,7 @@ limitations under the License. Ambari Agent """ - +from resource_management.libraries.functions.is_empty import is_empty """ Check both dfs.http.policy and deprecated dfs.https.enable http://git-wip-us.apache.org/repos/asf/ambari/blob/9406f54f/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py index 4c13aaa..5db1718 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py @@ -21,10 +21,12 @@ limitations under the License. # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. import ambari_simplejson as json from resource_management.core.exceptions import Fail +from resource_management.core.logger import Logger def check_stack_feature(stack_feature, stack_version): """ Given a stack_feature and a specific stack_version, it validates that the feature is supported by the stack_version. + IMPORTANT, notice that the mapping of feature to version comes from cluster-env if it exists there. :param stack_feature: Feature name to check if it is supported by the stack. For example: "rolling_upgrade" :param stack_version: Version of the stack :return: Will return True if successful, otherwise, False. @@ -35,6 +37,7 @@ def check_stack_feature(stack_feature, stack_version): stack_features_config = default("/configurations/cluster-env/stack_features", None) if not stack_version: + Logger.debug("Cannot determine if feature %s is supported since did not provide a stack version." % stack_feature) return False if stack_features_config: http://git-wip-us.apache.org/repos/asf/ambari/blob/9406f54f/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py index 5e25325..b5b3a34 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py @@ -31,6 +31,7 @@ from resource_management.libraries.script import Script from resource_management.libraries.resources import PropertiesFile from resource_management.libraries.functions import format from resource_management.libraries.functions.show_logs import show_logs +from resource_management.core.logger import Logger from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl @@ -130,7 +131,7 @@ def falcon(type, action = None, upgrade_type=None): owner = params.falcon_user, mode = 0777) - if params.supports_hive_dr: + if params.supports_data_mirroring: params.HdfsResource(params.dfs_data_mirroring_dir, type = "directory", action = "create_on_execute", @@ -141,7 +142,19 @@ def falcon(type, action = None, upgrade_type=None): mode = 0770, source = params.local_data_mirroring_dir) + if params.supports_falcon_extensions: + params.HdfsResource(params.falcon_extensions_dest_dir, + type = "directory", + action = "create_on_execute", + owner = params.falcon_user, + group = params.proxyuser_group, + recursive_chown = True, + recursive_chmod = True, + mode = 0770, + source = params.falcon_extensions_source_dir) + # At least one HDFS Dir should be created, so execute the change now. params.HdfsResource(None, action = "execute") + Directory(params.falcon_local_dir, owner = params.falcon_user, create_parents = True, http://git-wip-us.apache.org/repos/asf/ambari/blob/9406f54f/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py index a201619..cf18420 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py @@ -24,6 +24,7 @@ from resource_management.libraries.script import Script from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import check_process_status +from resource_management.libraries.functions import Direction from resource_management.libraries.functions.security_commons import build_expectations from resource_management.libraries.functions.security_commons import cached_kinit_executor from resource_management.libraries.functions.security_commons import get_params_from_filesystem @@ -85,6 +86,23 @@ class FalconServerLinux(FalconServer): Logger.info("Executing Falcon Server Stack Upgrade pre-restart") conf_select.select(params.stack_name, "falcon", params.version) stack_select.select("falcon-server", params.version) + + # Must be an Upgrade direction from a version less than when the feature is supported to a + # version at or after the feature is supported. + if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and \ + check_stack_feature(StackFeature.FALCON_EXTENSIONS, params.stack_version_formatted) and \ + not check_stack_feature(StackFeature.FALCON_EXTENSIONS, params.current_version_formatted): + params.HdfsResource(params.falcon_extensions_dest_dir, + type = "directory", + action = "create_on_execute", + owner = params.falcon_user, + group = params.proxyuser_group, + recursive_chown = True, + recursive_chmod = True, + mode = 0770, + source = params.falcon_extensions_source_dir) + params.HdfsResource(None, action="execute") + falcon_server_upgrade.pre_start_restore() def security_status(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/9406f54f/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py index 8c2ad8e..22fb691 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py @@ -28,6 +28,7 @@ from resource_management.libraries.script.script import Script import os from resource_management.libraries.functions.expect import expect from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions import StackFeature config = Script.get_config() @@ -42,6 +43,12 @@ version = default("/commandParams/version", None) stack_version_unformatted = status_params.stack_version_unformatted stack_version_formatted = status_params.stack_version_formatted +upgrade_direction = default("/commandParams/upgrade_direction", None) + +# current host stack version +current_version = default("/hostLevelParams/current_version", None) +current_version_formatted = format_stack_version(current_version) + etc_prefix_dir = "/etc/falcon" # hadoop params @@ -61,6 +68,11 @@ if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, falcon_root = server_role_dir_mapping[command_role] falcon_webapp_dir = format('{stack_root}/current/{falcon_root}/webapp') falcon_home = format('{stack_root}/current/{falcon_root}') + + # Extensions dir is only available in HDP 2.5 and higher + falcon_extensions_source_dir = os.path.join(stack_root, "current", falcon_root, "extensions") + # Dir in HDFS + falcon_extensions_dest_dir = "/apps/falcon/extensions" else: falcon_webapp_dir = '/var/lib/falcon/webapp' falcon_home = '/usr/lib/falcon' @@ -110,6 +122,13 @@ smokeuser_principal = config['configurations']['cluster-env']['smokeuser_princi kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) supports_hive_dr = config['configurations']['falcon-env']['supports_hive_dr'] +# HDP 2.4 still supported the /usr/$STACK/$VERSION/falcon/data-mirroring folder. +# In HDP 2.5, it was replaced with Falcon Extensions. data mirroring XOR falcon extensions +supports_data_mirroring = supports_hive_dr and (stack_version_formatted and not check_stack_feature(StackFeature.FALCON_EXTENSIONS, stack_version_formatted)) + +# Falcon Extensions were supported in HDP 2.5 and higher. +supports_falcon_extensions = (stack_version_formatted and check_stack_feature(StackFeature.FALCON_EXTENSIONS, stack_version_formatted)) + local_data_mirroring_dir = format('{stack_root}/current/falcon-server/data-mirroring') dfs_data_mirroring_dir = "/apps/data-mirroring" http://git-wip-us.apache.org/repos/asf/ambari/blob/9406f54f/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index 8b669e8..734d5b4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -166,6 +166,11 @@ "max_version": "2.2.0.0" }, { + "name": "falcon_extensions", + "description": "Falcon Extension", + "min_version": "2.5.0.0" + }, + { "name": "hive_metastore_upgrade_schema", "description": "Hive metastore upgrade schema support (AMBARI-11176)", "min_version": "2.3.0.0" http://git-wip-us.apache.org/repos/asf/ambari/blob/9406f54f/ambari-server/src/test/python/stacks/2.0.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json index 8d04e36..86138d1 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json @@ -670,7 +670,8 @@ "content": "\n# The java implementation to use. If JAVA_HOME is not found we expect java and jar to be in path\nexport JAVA_HOME={{java_home}}\n\n# any additional java opts you want to set. This will apply to both client and server operations\n#export FALCON_OPTS=\n\n# any additional java opts that you want to set for client only\n#export FALCON_CLIENT_OPTS=\n\n# java heap size we want to set for the client. Default is 1024MB\n#export FALCON_CLIENT_HEAP=\n\n# any additional opts you want to set for prisim service.\n#export FALCON_PRISM_OPTS=\n\n# java heap size we want to set for the prisim service. Default is 1024MB\n#export FALCON_PRISM_HEAP=\n\n# any additional opts you want to set for falcon service.\nexport FALCON_SERVER_OPTS=\"-Dfalcon.embeddedmq={{falcon_embeddedmq_enabled}} -Dfalcon.emeddedmq.port={{falcon_emeddedmq_port}}\"\n\n# java heap size we want to set for the falcon server. Default is 1024MB\n#export FALCON_SERVER_HEAP=\n\n# What is is considered as falco n home dir. Default is the base locaion of the installed software\n#export FALCON_HOME_DIR=\n\n# Where log files are stored. Defatult is logs directory under the base install location\nexport FALCON_LOG_DIR={{falcon_log_dir}}\n\n# Where pid files are stored. Defatult is logs directory under the base install location\nexport FALCON_PID_DIR={{falcon_pid_dir}}\n\n# where the falcon active mq data is stored. Defatult is logs/data directory under the base install location\nexport FALCON_DATA_DIR={{falcon_embeddedmq_data}}\n\n# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.\n#export FALCON_EXPANDED_WEBAPP_DIR=", "falcon.embeddedmq.data": "/hadoop/falcon/embeddedmq/data", "falcon.embeddedmq": "true", - "falcon_store_uri": "file:///hadoop/falcon/store" + "falcon_store_uri": "file:///hadoop/falcon/store", + "supports_hive_dr": "true" }, "oozie-env": { "oozie_derby_database": "Derby",
