Repository: ambari Updated Branches: refs/heads/trunk 5c20a2488 -> 954a7072b
AMBARI-15114: Stack Featurize HBASE Service (Juanjo Marron via jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/954a7072 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/954a7072 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/954a7072 Branch: refs/heads/trunk Commit: 954a7072b0d3d82e5f818794b0a192a743b9c76a Parents: 5c20a24 Author: Jayush Luniya <[email protected]> Authored: Tue Mar 29 19:02:06 2016 -0700 Committer: Jayush Luniya <[email protected]> Committed: Tue Mar 29 19:02:06 2016 -0700 ---------------------------------------------------------------------- .../libraries/functions/constants.py | 1 + .../libraries/functions/stack_features.py | 5 +++ .../0.96.0.2.0/package/scripts/hbase_client.py | 7 ++-- .../0.96.0.2.0/package/scripts/hbase_master.py | 3 +- .../package/scripts/hbase_regionserver.py | 3 +- .../0.96.0.2.0/package/scripts/params_linux.py | 36 +++++++++++--------- .../package/scripts/phoenix_queryserver.py | 9 +++-- .../package/scripts/phoenix_service.py | 2 +- .../0.96.0.2.0/package/scripts/status_params.py | 11 ++++-- .../HBASE/0.96.0.2.0/package/scripts/upgrade.py | 5 +-- .../HDP/2.0.6/properties/stack_features.json | 5 +++ 11 files changed, 58 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index 7ec83b6..b6aa1b8 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -46,6 +46,7 @@ class StackFeature: DATANODE_NON_ROOT = "datanode_non_root" REMOVE_RANGER_HDFS_PLUGIN_ENV = "remove_ranger_hdfs_plugin_env" RANGER = "ranger" + PHOENIX = "phoenix" NFS = "nfs" TEZ_FOR_SPARK = "tez_for_spark" SPARK_16PLUS = "spark_16plus" http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py index d79f743..e5fdd89 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py @@ -63,6 +63,11 @@ _DEFAULT_STACK_FEATURES = { "min_version": "2.2.0.0" }, { + "name": "phoenix", + "description": "Phoenix Service support", + "min_version": "2.3.0.0" + }, + { "name": "nfs", "description": "NFS support", "min_version": "2.3.0.0" http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py index c31bbf6..b72e648 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py @@ -22,6 +22,8 @@ import sys from resource_management import * from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from hbase import hbase from ambari_commons import OSCheck, OSConst from ambari_commons.os_family_impl import OsFamilyImpl @@ -51,13 +53,14 @@ class HbaseClientWindows(HbaseClient): @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class HbaseClientDefault(HbaseClient): def get_stack_to_component(self): - return {"HDP": "hbase-client"} + import params + return {params.stack_name: "hbase-client"} def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) - if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): conf_select.select(params.stack_name, "hbase", params.version) stack_select.select("hbase-client", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py index 9515f61..5833773 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py @@ -71,7 +71,8 @@ class HbaseMasterWindows(HbaseMaster): @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class HbaseMasterDefault(HbaseMaster): def get_stack_to_component(self): - return {"HDP": "hbase-master"} + import params + return {params.stack_name: "hbase-master"} def pre_upgrade_restart(self, env, upgrade_type=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py index c17d219..37dd0a1 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py @@ -69,7 +69,8 @@ class HbaseRegionServerWindows(HbaseRegionServer): @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class HbaseRegionServerDefault(HbaseRegionServer): def get_stack_to_component(self): - return {"HDP": "hbase-regionserver"} + import params + return {params.stack_name: "hbase-regionserver"} def pre_upgrade_restart(self, env, upgrade_type=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index bc6fcde..31e1757 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -29,7 +29,8 @@ from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import format -from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.default import default from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions import is_empty @@ -51,8 +52,9 @@ version = default("/commandParams/version", None) component_directory = status_params.component_directory etc_prefix_dir = "/etc/hbase" -stack_version_unformatted = config['hostLevelParams']['stack_version'] -stack_version_formatted = format_stack_version(stack_version_unformatted) +stack_version_unformatted = status_params.stack_version_unformatted +stack_version_formatted = status_params.stack_version_formatted +stack_root = status_params.stack_root # hadoop default parameters hadoop_bin_dir = stack_select.get_hadoop_dir("bin") @@ -63,19 +65,19 @@ region_drainer = "/usr/lib/hbase/bin/draining_servers.rb" hbase_cmd = "/usr/lib/hbase/bin/hbase" hbase_max_direct_memory_size = None -# hadoop parameters for 2.2+ -if Script.is_stack_greater_or_equal("2.2"): - daemon_script = format('/usr/hdp/current/hbase-client/bin/hbase-daemon.sh') - region_mover = format('/usr/hdp/current/hbase-client/bin/region_mover.rb') - region_drainer = format('/usr/hdp/current/hbase-client/bin/draining_servers.rb') - hbase_cmd = format('/usr/hdp/current/hbase-client/bin/hbase') +# hadoop parameters for stacks supporting rolling_upgrade +if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): + daemon_script = format('{stack_root}/current/hbase-client/bin/hbase-daemon.sh') + region_mover = format('{stack_root}/current/hbase-client/bin/region_mover.rb') + region_drainer = format('{stack_root}/current/hbase-client/bin/draining_servers.rb') + hbase_cmd = format('{stack_root}/current/hbase-client/bin/hbase') hbase_max_direct_memory_size = default('configurations/hbase-env/hbase_max_direct_memory_size', None) - daemon_script=format("/usr/hdp/current/{component_directory}/bin/hbase-daemon.sh") - region_mover = format("/usr/hdp/current/{component_directory}/bin/region_mover.rb") - region_drainer = format("/usr/hdp/current/{component_directory}/bin/draining_servers.rb") - hbase_cmd = format("/usr/hdp/current/{component_directory}/bin/hbase") + daemon_script=format("{stack_root}/current/{component_directory}/bin/hbase-daemon.sh") + region_mover = format("{stack_root}/current/{component_directory}/bin/region_mover.rb") + region_drainer = format("{stack_root}/current/{component_directory}/bin/draining_servers.rb") + hbase_cmd = format("{stack_root}/current/{component_directory}/bin/hbase") hbase_conf_dir = status_params.hbase_conf_dir @@ -85,7 +87,7 @@ hbase_user_nofile_limit = default("/configurations/hbase-env/hbase_user_nofile_l hbase_user_nproc_limit = default("/configurations/hbase-env/hbase_user_nproc_limit", "16000") # no symlink for phoenix-server at this point -phx_daemon_script = '/usr/hdp/current/phoenix-server/bin/queryserver.py' +phx_daemon_script = format('{stack_root}/current/phoenix-server/bin/queryserver.py') hbase_excluded_hosts = config['commandParams']['excluded_hosts'] hbase_drain_only = default("/commandParams/mark_draining_only",False) @@ -313,7 +315,7 @@ if has_ranger_admin: downloaded_custom_connector = format("{exec_tmp_dir}/{jdbc_jar_name}") driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") - driver_curl_target = format("/usr/hdp/current/{component_directory}/lib/{jdbc_jar_name}") + driver_curl_target = format("{stack_root}/current/{component_directory}/lib/{jdbc_jar_name}") hbase_ranger_plugin_config = { 'username': repo_config_username, @@ -353,11 +355,11 @@ if security_enabled: hbase_coprocessor_master_classes = "org.apache.hadoop.hbase.security.access.AccessController" hbase_coprocessor_regionserver_classes = "org.apache.hadoop.hbase.security.access.AccessController" hbase_coprocessor_region_classes = "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.hadoop.hbase.security.access.AccessController" - elif xml_configurations_supported: # HDP stack 2.3+ ranger plugin enabled + elif xml_configurations_supported: # stack version ranger plugin enabled hbase_coprocessor_master_classes = "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor " hbase_coprocessor_regionserver_classes = "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor" hbase_coprocessor_region_classes = "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor" - else: # HDP Stack 2.2 and less / ranger plugin enabled + else: hbase_coprocessor_master_classes = "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor" hbase_coprocessor_regionserver_classes = "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor" hbase_coprocessor_region_classes = "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor" http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py index 87e4899..7d639ec 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py @@ -19,11 +19,13 @@ limitations under the License. from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.script import Script from phoenix_service import phoenix_service from hbase import hbase -# Note: Phoenix Query Server is only applicable to HDP-2.3 and above. +# Note: Phoenix Query Server is only applicable to stack version supporting Phoenix. class PhoenixQueryServer(Script): def install(self, env): @@ -33,7 +35,8 @@ class PhoenixQueryServer(Script): def get_stack_to_component(self): - return {"HDP": "phoenix-server"} + import params + return {params.stack_name: "phoenix-server"} def configure(self, env): @@ -59,7 +62,7 @@ class PhoenixQueryServer(Script): import params env.set_params(params) - if Script.is_stack_greater_or_equal("2.3"): + if params.stack_version_formatted and check_stack_feature(StackFeature.PHOENIX, params.stack_version_formatted): # phoenix uses hbase configs conf_select.select(params.stack_name, "hbase", params.version) stack_select.select("phoenix-server", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_service.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_service.py index e155cec..d1e2bbd 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_service.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_service.py @@ -22,7 +22,7 @@ from resource_management.core.resources.system import Execute from resource_management.core.resources.system import File from resource_management.libraries.functions import check_process_status, format -# Note: Phoenix Query Server is only applicable to HDP-2.3 and above. +# Note: Phoenix Query Server is only applicable to phoenix version stacks and above. def phoenix_service(action = 'start'): # 'start', 'stop', 'status' # Note: params/status_params should already be imported before calling phoenix_service() pid_file = format("{pid_dir}/phoenix-{hbase_user}-server.pid") http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py index 3ec84ee..8b60292 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py @@ -21,6 +21,9 @@ from ambari_commons.os_check import OSCheck from resource_management.libraries.functions import format from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.script.script import Script @@ -48,8 +51,12 @@ else: security_enabled = config['configurations']['cluster-env']['security_enabled'] kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) tmp_dir = Script.get_tmp_dir() + + stack_version_unformatted = str(config['hostLevelParams']['stack_version']) + stack_version_formatted = format_stack_version(stack_version_unformatted) + stack_root = Script.get_stack_root() hbase_conf_dir = "/etc/hbase/conf" limits_conf_dir = "/etc/security/limits.d" - if Script.is_stack_greater_or_equal("2.2"): - hbase_conf_dir = format("/usr/hdp/current/{component_directory}/conf") \ No newline at end of file + if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): + hbase_conf_dir = format("{stack_root}/current/{component_directory}/conf") \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/upgrade.py index 92c0f70..01a8156 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/upgrade.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/upgrade.py @@ -23,13 +23,14 @@ from resource_management.core.resources.system import Execute from resource_management.core import shell from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions.version import compare_versions, format_stack_version +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.decorator import retry def prestart(env, stack_component): import params - if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): conf_select.select(params.stack_name, "hbase", params.version) stack_select.select(stack_component, params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index d38985e..0234bba 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -37,6 +37,11 @@ "min_version": "2.2.0.0" }, { + "name": "phoenix", + "description": "Phoenix Service support", + "min_version": "2.3.0.0" + }, + { "name": "nfs", "description": "NFS support", "min_version": "2.3.0.0"
