Repository: ambari Updated Branches: refs/heads/trunk e23ed7bf3 -> 5b4e663f8
AMBARI-14451: Stack Featurize HDFS service (Juanjo Marron via jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5b4e663f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5b4e663f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5b4e663f Branch: refs/heads/trunk Commit: 5b4e663f820390f7bc006509eb517c3477b8f453 Parents: e23ed7b Author: Jayush Luniya <[email protected]> Authored: Tue Mar 29 18:50:35 2016 -0700 Committer: Jayush Luniya <[email protected]> Committed: Tue Mar 29 18:50:35 2016 -0700 ---------------------------------------------------------------------- .../libraries/functions/constants.py | 2 ++ .../libraries/functions/stack_features.py | 10 ++++++++++ .../HDFS/2.1.0.2.0/configuration/hadoop-env.xml | 4 ++-- .../package/alerts/alert_ha_namenode_health.py | 6 +++--- .../package/alerts/alert_metrics_deviation.py | 6 +++--- .../HDFS/2.1.0.2.0/package/scripts/datanode.py | 8 +++++--- .../HDFS/2.1.0.2.0/package/scripts/hdfs_client.py | 7 +++++-- .../HDFS/2.1.0.2.0/package/scripts/journalnode.py | 9 +++++---- .../package/scripts/journalnode_upgrade.py | 2 +- .../HDFS/2.1.0.2.0/package/scripts/namenode.py | 8 +++++--- .../HDFS/2.1.0.2.0/package/scripts/nfsgateway.py | 8 +++++--- .../2.1.0.2.0/package/scripts/params_linux.py | 11 +++++++---- .../package/scripts/setup_ranger_hdfs.py | 8 +++++--- .../HDFS/2.1.0.2.0/package/scripts/snamenode.py | 8 +++++--- .../HDFS/2.1.0.2.0/package/scripts/utils.py | 18 ++++++++---------- .../HDP/2.0.6/properties/stack_features.json | 10 ++++++++++ 16 files changed, 81 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index f766a82..006b84c 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -43,6 +43,8 @@ class StackFeature: EXPRESS_UPGRADE = "express_upgrade" ROLLING_UPGRADE = "rolling_upgrade" CONFIG_VERSIONING = "config_versioning" + DATANODE_NON_ROOT = "datanode_non_root" + REMOVE_RANGER_HDFS_PLUGIN_ENV = "remove_ranger_hdfs_plugin_env" RANGER = "ranger" NFS = "nfs" TEZ_FOR_SPARK = "tez_for_spark" http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py index 2f0e6bf..9ad99dc 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py @@ -48,6 +48,16 @@ _DEFAULT_STACK_FEATURES = { "min_version": "2.3.0.0" }, { + "name": "datanode_non_root", + "description": "DataNode running as non-root support (AMBARI-7615)", + "min_version": "2.2.0.0" + }, + { + "name": "remove_ranger_hdfs_plugin_env", + "description": "HDFS removes Ranger env files (AMBARI-14299)", + "min_version": "2.3.0.0" + }, + { "name": "ranger", "description": "Ranger Service support", "min_version": "2.2.0.0" http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml index 98f20e7..fb541ed 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml @@ -241,8 +241,8 @@ export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop} #TODO: if env var set that can cause problems export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-{{hadoop_conf_dir}}} -{# this is different for HDP1 #} -# Path to jsvc required by secure HDP 2.0 datanode + +# Path to jsvc required by secure datanode export JSVC_HOME={{jsvc_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_ha_namenode_health.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_ha_namenode_health.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_ha_namenode_health.py index 70b1970..20d1717 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_ha_namenode_health.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_ha_namenode_health.py @@ -247,7 +247,7 @@ def get_jmx(query, connection_timeout): def _get_ha_state_from_json(string_json): """ - Searches through the specified JSON string looking for either the HDP 2.0 or 2.1+ HA state + Searches through the specified JSON string looking for HA state enumerations. :param string_json: the string JSON :return: the value of the HA state (active, standby, etc) @@ -255,7 +255,7 @@ def _get_ha_state_from_json(string_json): json_data = json.loads(string_json) jmx_beans = json_data["beans"] - # look for HDP 2.1+ first + # look for NameNodeStatus-State first for jmx_bean in jmx_beans: if "name" not in jmx_bean: continue @@ -264,7 +264,7 @@ def _get_ha_state_from_json(string_json): if jmx_bean_name == "Hadoop:service=NameNode,name=NameNodeStatus" and "State" in jmx_bean: return jmx_bean["State"] - # look for HDP 2.0 last + # look for FSNamesystem-tag.HAState last for jmx_bean in jmx_beans: if "name" not in jmx_bean: continue http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py index 9a122fa..038592f 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py @@ -379,7 +379,7 @@ def get_jmx(query, connection_timeout): def _get_ha_state_from_json(string_json): """ - Searches through the specified JSON string looking for either the HDP 2.0 or 2.1+ HA state + Searches through the specified JSON string looking for HA state enumerations. :param string_json: the string JSON :return: the value of the HA state (active, standby, etc) @@ -387,7 +387,7 @@ def _get_ha_state_from_json(string_json): json_data = json.loads(string_json) jmx_beans = json_data["beans"] - # look for HDP 2.1+ first + # look for NameNodeStatus-State first for jmx_bean in jmx_beans: if "name" not in jmx_bean: continue @@ -396,7 +396,7 @@ def _get_ha_state_from_json(string_json): if jmx_bean_name == "Hadoop:service=NameNode,name=NameNodeStatus" and "State" in jmx_bean: return jmx_bean["State"] - # look for HDP 2.0 last + # look for FSNamesystem-tag.HAState last for jmx_bean in jmx_beans: if "name" not in jmx_bean: continue http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py index 3cdfda9..ffaf4ea 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py @@ -21,7 +21,8 @@ from hdfs_datanode import datanode from resource_management import * from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions.version import compare_versions, format_stack_version +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.security_commons import build_expectations, \ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, FILE_TYPE_XML from hdfs import hdfs @@ -32,7 +33,8 @@ from utils import get_hdfs_binary class DataNode(Script): def get_stack_to_component(self): - return {"HDP": "hadoop-hdfs-datanode"} + import params + return {params.stack_name : "hadoop-hdfs-datanode"} def get_hdfs_binary(self): """ @@ -87,7 +89,7 @@ class DataNodeDefault(DataNode): Logger.info("Executing DataNode Stack Upgrade pre-restart") import params env.set_params(params) - if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): conf_select.select(params.stack_name, "hadoop", params.version) stack_select.select("hadoop-hdfs-datanode", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py index c5ae35e..9f62d29 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py @@ -20,6 +20,8 @@ limitations under the License. from resource_management import * from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.security_commons import build_expectations, \ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ FILE_TYPE_XML @@ -55,12 +57,13 @@ class HdfsClient(Script): class HdfsClientDefault(HdfsClient): def get_stack_to_component(self): - return {"HDP": "hadoop-client"} + import params + return { params.stack_name : "hadoop-client"} def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) - if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): conf_select.select(params.stack_name, "hadoop", params.version) stack_select.select("hadoop-client", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py index 6f26b40..ac73eaf 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py @@ -20,8 +20,8 @@ limitations under the License. from resource_management import * from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions.version import compare_versions, \ - format_stack_version +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.format import format from resource_management.libraries.functions.security_commons import build_expectations, \ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ @@ -43,14 +43,15 @@ class JournalNode(Script): class JournalNodeDefault(JournalNode): def get_stack_to_component(self): - return {"HDP": "hadoop-hdfs-journalnode"} + import params + return {params.stack_name : "hadoop-hdfs-journalnode"} def pre_upgrade_restart(self, env, upgrade_type=None): Logger.info("Executing Stack Upgrade pre-restart") import params env.set_params(params) - if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): conf_select.select(params.stack_name, "hadoop", params.version) stack_select.select("hadoop-hdfs-journalnode", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode_upgrade.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode_upgrade.py index d598840..bd9f014 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode_upgrade.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode_upgrade.py @@ -81,7 +81,7 @@ def hdfs_roll_edits(): """ import params - # TODO, this will be to be doc'ed since existing HDP 2.2 clusters will needs HDFS_CLIENT on all JOURNALNODE hosts + # TODO, this will need to be doc'ed since existing clusters will need HDFS_CLIENT on all JOURNALNODE hosts dfsadmin_base_command = get_dfsadmin_base_command('hdfs') command = dfsadmin_base_command + ' -rollEdits' Execute(command, user=params.hdfs_user, tries=1) http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py index acd10e8..c8acae3 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py @@ -31,7 +31,8 @@ from resource_management.core import shell from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import Direction -from resource_management.libraries.functions.version import compare_versions, format_stack_version +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.format import format from resource_management.libraries.functions.security_commons import build_expectations, \ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ @@ -68,7 +69,8 @@ except ImportError: class NameNode(Script): def get_stack_to_component(self): - return {"HDP": "hadoop-hdfs-namenode"} + import params + return {params.stack_name : "hadoop-hdfs-namenode"} def get_hdfs_binary(self): """ @@ -197,7 +199,7 @@ class NameNodeDefault(NameNode): import params env.set_params(params) - if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): # When downgrading an Express Upgrade, the first thing we do is to revert the symlinks. # Therefore, we cannot call this code in that scenario. call_if = [("rolling", "upgrade"), ("rolling", "downgrade"), ("nonrolling", "upgrade")] http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/nfsgateway.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/nfsgateway.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/nfsgateway.py index c705fca..ca2e3ef 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/nfsgateway.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/nfsgateway.py @@ -26,13 +26,15 @@ from hdfs_nfsgateway import nfsgateway from hdfs import hdfs from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions.version import compare_versions, format_stack_version +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature class NFSGateway(Script): def get_stack_to_component(self): - return {"HDP": "hadoop-hdfs-nfs3"} + import params + return {params.stack_name : "hadoop-hdfs-nfs3"} def install(self, env): import params @@ -45,7 +47,7 @@ class NFSGateway(Script): import params env.set_params(params) - if Script.is_stack_greater_or_equal('2.3.0.0'): + if params.stack_version_formatted and check_stack_feature(StackFeature.NFS, params.stack_version_formatted): conf_select.select(params.stack_name, "hadoop", params.version) stack_select.select("hadoop-hdfs-nfs3", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index 277536a..d6dec26 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -27,6 +27,8 @@ from ambari_commons.os_check import OSCheck from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions import format from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.default import default @@ -47,6 +49,7 @@ config = Script.get_config() tmp_dir = Script.get_tmp_dir() stack_name = default("/hostLevelParams/stack_name", None) +stack_root = Script.get_stack_root() upgrade_direction = default("/commandParams/upgrade_direction", None) stack_version_unformatted = config['hostLevelParams']['stack_version'] stack_version_formatted = format_stack_version(stack_version_unformatted) @@ -93,9 +96,9 @@ hadoop_conf_dir = conf_select.get_hadoop_conf_dir() hadoop_conf_secure_dir = os.path.join(hadoop_conf_dir, "secure") hadoop_lib_home = stack_select.get_hadoop_dir("lib") -# hadoop parameters for 2.2+ -if Script.is_stack_greater_or_equal("2.2"): - mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*" +# hadoop parameters for stacks that support rolling_upgrade +if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): + mapreduce_libs_path = format("{stack_root}/current/hadoop-mapreduce-client/*") if not security_enabled: hadoop_secure_dn_user = '""' @@ -121,7 +124,7 @@ limits_conf_dir = "/etc/security/limits.d" hdfs_user_nofile_limit = default("/configurations/hadoop-env/hdfs_user_nofile_limit", "128000") hdfs_user_nproc_limit = default("/configurations/hadoop-env/hdfs_user_nproc_limit", "65536") -create_lib_snappy_symlinks = not Script.is_stack_greater_or_equal("2.2") +create_lib_snappy_symlinks = check_stack_feature(StackFeature.SNAPPY, stack_version_formatted) jsvc_path = "/usr/lib/bigtop-utils" execute_path = os.environ['PATH'] + os.pathsep + hadoop_bin_dir http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py index 209ac91..858044c 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py @@ -20,9 +20,11 @@ limitations under the License. import os from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.constants import Direction from resource_management.libraries.functions.format import format -from resource_management.libraries.functions.version import compare_versions + def setup_ranger_hdfs(upgrade_type=None): import params @@ -61,8 +63,8 @@ def setup_ranger_hdfs(upgrade_type=None): stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble) if stack_version and params.upgrade_direction == Direction.UPGRADE: - # when upgrading to 2.3+, this env file must be removed - if compare_versions(stack_version, '2.3', format=True) > 0: + # when upgrading to stack remove_ranger_hdfs_plugin_env, this env file must be removed + if check_stack_feature(StackFeature.REMOVE_RANGER_HDFS_PLUGIN_ENV, stack_version): source_file = os.path.join(params.hadoop_conf_dir, 'set-hdfs-plugin-env.sh') target_file = source_file + ".bak" Execute(("mv", source_file, target_file), sudo=True, only_if=format("test -f {source_file}")) http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py index f96ac01..4a6f144 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py @@ -20,7 +20,8 @@ limitations under the License. from resource_management import * from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions.version import compare_versions, format_stack_version +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.security_commons import build_expectations, \ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ FILE_TYPE_XML @@ -64,14 +65,15 @@ class SNameNode(Script): class SNameNodeDefault(SNameNode): def get_stack_to_component(self): - return {"HDP": "hadoop-hdfs-secondarynamenode"} + import params + return { params.stack_name : "hadoop-hdfs-secondarynamenode"} def pre_upgrade_restart(self, env, upgrade_type=None): Logger.info("Executing Stack Upgrade pre-restart") import params env.set_params(params) - if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): conf_select.select(params.stack_name, "hadoop", params.version) stack_select.select("hadoop-hdfs-secondarynamenode", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py index c626028..339d52a 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py @@ -24,7 +24,8 @@ import ambari_simplejson as json # simplejson is much faster comparing to Python from resource_management.core.resources.system import Directory, File, Execute from resource_management.libraries.functions.format import format from resource_management.libraries.functions import check_process_status -from resource_management.libraries.functions.version import compare_versions +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.core import shell from resource_management.core.shell import as_user, as_sudo from resource_management.core.exceptions import ComponentIsNotRunning @@ -134,7 +135,7 @@ def kill_zkfc(zkfc_user): """ There are two potential methods for failing over the namenode, especially during a Rolling Upgrade. Option 1. Kill zkfc on primary namenode provided that the secondary is up and has zkfc running on it. - Option 2. Silent failover (not supported as of HDP 2.2.0.0) + Option 2. Silent failover :param zkfc_user: User that started the ZKFC process. :return: Return True if ZKFC was killed, otherwise, false. """ @@ -223,13 +224,13 @@ def service(action=None, name=None, user=None, options="", create_pid_dir=False, hadoop_secure_dn_pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}") hadoop_secure_dn_pid_file = format("{hadoop_secure_dn_pid_dir}/hadoop_secure_dn.pid") - # At Champlain stack and further, we may start datanode as a non-root even in secure cluster - if not (params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.2') >= 0) or params.secure_dn_ports_are_in_use: + # At datanode_non_root stack version and further, we may start datanode as a non-root even in secure cluster + if not (params.stack_version_formatted and check_stack_feature(StackFeature.DATANODE_NON_ROOT, params.stack_version_formatted)) or params.secure_dn_ports_are_in_use: user = "root" pid_file = format( "{hadoop_pid_dir_prefix}/{hdfs_user}/hadoop-{hdfs_user}-{name}.pid") - if action == 'stop' and (params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.2') >= 0) and \ + if action == 'stop' and (params.stack_version_formatted and check_stack_feature(StackFeature.DATANODE_NON_ROOT, params.stack_version_formatted)) and \ os.path.isfile(hadoop_secure_dn_pid_file): # We need special handling for this case to handle the situation # when we configure non-root secure DN and then restart it @@ -351,11 +352,8 @@ def get_hdfs_binary(distro_component_name): """ import params hdfs_binary = "hdfs" - if params.stack_name == "HDP": - # This was used in HDP 2.1 and earlier - hdfs_binary = "hdfs" - if Script.is_stack_greater_or_equal("2.2"): - hdfs_binary = "/usr/hdp/current/{0}/bin/hdfs".format(distro_component_name) + if params.stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version_formatted): + hdfs_binary = "{0}/current/{1}/bin/hdfs".format(params.stack_root, distro_component_name) return hdfs_binary http://git-wip-us.apache.org/repos/asf/ambari/blob/5b4e663f/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index 97bd19c..cfbc9fb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -22,6 +22,16 @@ "min_version": "2.3.0.0" }, { + "name": "datanode_non_root", + "description": "DataNode running as non-root support (AMBARI-7615)", + "min_version": "2.2.0.0" + }, + { + "name": "remove_ranger_hdfs_plugin_env", + "description": "HDFS removes Ranger env files (AMBARI-14299)", + "min_version": "2.3.0.0" + }, + { "name": "ranger", "description": "Ranger Service support", "min_version": "2.2.0.0"
