AMBARI-21474. HBase REST Server is missing after Ambari upgrade
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b1c3784d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b1c3784d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b1c3784d Branch: refs/heads/branch-feature-AMBARI-21348 Commit: b1c3784dad1676bad22514e520075ebc267463fc Parents: 7764e38 Author: Attila Doroszlai <[email protected]> Authored: Fri Jul 14 13:28:59 2017 +0200 Committer: Attila Doroszlai <[email protected]> Committed: Mon Jul 17 13:34:52 2017 +0200 ---------------------------------------------------------------------- .../HBASE/package/files/draining_servers.rb | 164 +++++++ .../HBASE/package/files/hbase-smoke-cleanup.sh | 23 + .../HBASE/package/files/hbaseSmokeVerify.sh | 34 ++ .../services/HBASE/package/scripts/__init__.py | 19 + .../services/HBASE/package/scripts/functions.py | 54 +++ .../services/HBASE/package/scripts/hbase.py | 234 ++++++++++ .../HBASE/package/scripts/hbase_client.py | 82 ++++ .../HBASE/package/scripts/hbase_decommission.py | 93 ++++ .../HBASE/package/scripts/hbase_master.py | 163 +++++++ .../HBASE/package/scripts/hbase_regionserver.py | 166 +++++++ .../package/scripts/hbase_restgatewayserver.py | 83 ++++ .../HBASE/package/scripts/hbase_service.py | 93 ++++ .../HBASE/package/scripts/hbase_upgrade.py | 41 ++ .../services/HBASE/package/scripts/params.py | 29 ++ .../HBASE/package/scripts/params_linux.py | 447 +++++++++++++++++++ .../HBASE/package/scripts/params_windows.py | 43 ++ .../package/scripts/phoenix_queryserver.py | 88 ++++ .../HBASE/package/scripts/phoenix_service.py | 55 +++ .../HBASE/package/scripts/service_check.py | 95 ++++ .../HBASE/package/scripts/setup_ranger_hbase.py | 106 +++++ .../HBASE/package/scripts/status_params.py | 68 +++ .../services/HBASE/package/scripts/upgrade.py | 65 +++ ...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 117 +++++ ...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 116 +++++ .../HBASE/package/templates/hbase-smoke.sh.j2 | 44 ++ .../HBASE/package/templates/hbase.conf.j2 | 35 ++ .../package/templates/hbase_client_jaas.conf.j2 | 23 + .../templates/hbase_grant_permissions.j2 | 40 ++ .../package/templates/hbase_master_jaas.conf.j2 | 26 ++ .../templates/hbase_queryserver_jaas.conf.j2 | 26 ++ .../templates/hbase_regionserver_jaas.conf.j2 | 26 ++ .../package/templates/hbase_rest_jaas.conf.j2 | 26 ++ .../HBASE/package/templates/regionservers.j2 | 20 + 33 files changed, 2744 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/files/draining_servers.rb ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/files/draining_servers.rb b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/files/draining_servers.rb new file mode 100644 index 0000000..5bcb5b6 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/files/draining_servers.rb @@ -0,0 +1,164 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Add or remove servers from draining mode via zookeeper + +require 'optparse' +include Java + +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.client.HBaseAdmin +import org.apache.hadoop.hbase.zookeeper.ZKUtil +import org.apache.commons.logging.Log +import org.apache.commons.logging.LogFactory + +# Name of this script +NAME = "draining_servers" + +# Do command-line parsing +options = {} +optparse = OptionParser.new do |opts| + opts.banner = "Usage: ./hbase org.jruby.Main #{NAME}.rb [options] add|remove|list <hostname>|<host:port>|<servername> ..." + opts.separator 'Add remove or list servers in draining mode. Can accept either hostname to drain all region servers' + + 'in that host, a host:port pair or a host,port,startCode triplet. More than one server can be given separated by space' + opts.on('-h', '--help', 'Display usage information') do + puts opts + exit + end + options[:debug] = false + opts.on('-d', '--debug', 'Display extra debug logging') do + options[:debug] = true + end +end +optparse.parse! + +# Return array of servernames where servername is hostname+port+startcode +# comma-delimited +def getServers(admin) + serverInfos = admin.getClusterStatus().getServerInfo() + servers = [] + for server in serverInfos + servers << server.getServerName() + end + return servers +end + +def getServerNames(hostOrServers, config) + ret = [] + + for hostOrServer in hostOrServers + # check whether it is already serverName. No need to connect to cluster + parts = hostOrServer.split(',') + if parts.size() == 3 + ret << hostOrServer + else + admin = HBaseAdmin.new(config) if not admin + servers = getServers(admin) + + hostOrServer = hostOrServer.gsub(/:/, ",") + for server in servers + ret << server if server.start_with?(hostOrServer) + end + end + end + + admin.close() if admin + return ret +end + +def addServers(options, hostOrServers) + config = HBaseConfiguration.create() + servers = getServerNames(hostOrServers, config) + + zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(config, "draining_servers", nil) + parentZnode = zkw.drainingZNode + + begin + for server in servers + node = ZKUtil.joinZNode(parentZnode, server) + ZKUtil.createAndFailSilent(zkw, node) + end + ensure + zkw.close() + end +end + +def removeServers(options, hostOrServers) + config = HBaseConfiguration.create() + servers = getServerNames(hostOrServers, config) + + zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(config, "draining_servers", nil) + parentZnode = zkw.drainingZNode + + begin + for server in servers + node = ZKUtil.joinZNode(parentZnode, server) + ZKUtil.deleteNodeFailSilent(zkw, node) + end + ensure + zkw.close() + end +end + +# list servers in draining mode +def listServers(options) + config = HBaseConfiguration.create() + + zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(config, "draining_servers", nil) + parentZnode = zkw.drainingZNode + + servers = ZKUtil.listChildrenNoWatch(zkw, parentZnode) + servers.each {|server| puts server} +end + +hostOrServers = ARGV[1..ARGV.size()] + +# Create a logger and disable the DEBUG-level annoying client logging +def configureLogging(options) + apacheLogger = LogFactory.getLog(NAME) + # Configure log4j to not spew so much + unless (options[:debug]) + logger = org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase") + logger.setLevel(org.apache.log4j.Level::WARN) + logger = org.apache.log4j.Logger.getLogger("org.apache.zookeeper") + logger.setLevel(org.apache.log4j.Level::WARN) + end + return apacheLogger +end + +# Create a logger and save it to ruby global +$LOG = configureLogging(options) +case ARGV[0] + when 'add' + if ARGV.length < 2 + puts optparse + exit 1 + end + addServers(options, hostOrServers) + when 'remove' + if ARGV.length < 2 + puts optparse + exit 1 + end + removeServers(options, hostOrServers) + when 'list' + listServers(options) + else + puts optparse + exit 3 +end http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/files/hbase-smoke-cleanup.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/files/hbase-smoke-cleanup.sh b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/files/hbase-smoke-cleanup.sh new file mode 100644 index 0000000..cde19e4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/files/hbase-smoke-cleanup.sh @@ -0,0 +1,23 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +disable 'ambarismoketest' +drop 'ambarismoketest' +exit \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/files/hbaseSmokeVerify.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/files/hbaseSmokeVerify.sh b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/files/hbaseSmokeVerify.sh new file mode 100644 index 0000000..19276f3 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/files/hbaseSmokeVerify.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +conf_dir=$1 +data=$2 +hbase_cmd=$3 +echo "scan 'ambarismoketest'" | $hbase_cmd --config $conf_dir shell > /tmp/hbase_chk_verify +cat /tmp/hbase_chk_verify +echo "Looking for $data" +awk '/id/,/date/' /tmp/hbase_chk_verify +if [ "$?" -ne 0 ] +then + exit 1 +fi + +grep -q '1 row(s)' /tmp/hbase_chk_verify http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/__init__.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/__init__.py new file mode 100644 index 0000000..5561e10 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/__init__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/functions.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/functions.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/functions.py new file mode 100644 index 0000000..f98b9b9 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/functions.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os +import re +import math +import datetime + +from resource_management.core.shell import checked_call + +def calc_xmn_from_xms(heapsize_str, xmn_percent, xmn_max): + """ + @param heapsize_str: str (e.g '1000m') + @param xmn_percent: float (e.g 0.2) + @param xmn_max: integer (e.g 512) + """ + heapsize = int(re.search('\d+',heapsize_str).group(0)) + heapsize_unit = re.search('\D+',heapsize_str).group(0) + xmn_val = int(math.floor(heapsize*xmn_percent)) + xmn_val -= xmn_val % 8 + + result_xmn_val = xmn_max if xmn_val > xmn_max else xmn_val + return str(result_xmn_val) + heapsize_unit + +def ensure_unit_for_memory(memory_size): + memory_size_values = re.findall('\d+', str(memory_size)) + memory_size_unit = re.findall('\D+', str(memory_size)) + + if len(memory_size_values) > 0: + unit = 'm' + if len(memory_size_unit) > 0: + unit = memory_size_unit[0] + if unit not in ['b', 'k', 'm', 'g', 't', 'p']: + raise Exception("Memory size unit error. %s - wrong unit" % unit) + return "%s%s" % (memory_size_values[0], unit) + else: + raise Exception('Memory size can not be calculated') http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase.py new file mode 100644 index 0000000..837cf41 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os +from resource_management import * +import sys +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst + +from urlparse import urlparse + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def hbase(name=None): + import params + XmlConfig("hbase-site.xml", + conf_dir = params.hbase_conf_dir, + configurations = params.config['configurations']['hbase-site'], + configuration_attributes=params.config['configuration_attributes']['hbase-site'] + ) + + if params.service_map.has_key(name): + # Manually overriding service logon user & password set by the installation package + service_name = params.service_map[name] + ServiceConfig(service_name, + action="change_user", + username = params.hbase_user, + password = Script.get_password(params.hbase_user)) + +# name is 'master' or 'regionserver' or 'queryserver' or 'client' +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def hbase(name=None): + import params + + Directory( params.etc_prefix_dir, + mode=0755 + ) + + Directory( params.hbase_conf_dir, + owner = params.hbase_user, + group = params.user_group, + create_parents = True + ) + + Directory(params.java_io_tmpdir, + create_parents = True, + mode=0777 + ) + + # If a file location is specified in ioengine parameter, + # ensure that directory exists. Otherwise create the + # directory with permissions assigned to hbase:hadoop. + ioengine_input = params.ioengine_param + if ioengine_input != None: + if ioengine_input.startswith("file:/"): + ioengine_fullpath = ioengine_input[5:] + ioengine_dir = os.path.dirname(ioengine_fullpath) + Directory(ioengine_dir, + owner = params.hbase_user, + group = params.user_group, + create_parents = True, + mode = 0755 + ) + + parent_dir = os.path.dirname(params.tmp_dir) + # In case if we have several placeholders in path + while ("${" in parent_dir): + parent_dir = os.path.dirname(parent_dir) + if parent_dir != os.path.abspath(os.sep) : + Directory (parent_dir, + create_parents = True, + cd_access="a", + ) + Execute(("chmod", "1777", parent_dir), sudo=True) + + XmlConfig( "hbase-site.xml", + conf_dir = params.hbase_conf_dir, + configurations = params.config['configurations']['hbase-site'], + configuration_attributes=params.config['configuration_attributes']['hbase-site'], + owner = params.hbase_user, + group = params.user_group + ) + + XmlConfig( "core-site.xml", + conf_dir = params.hbase_conf_dir, + configurations = params.config['configurations']['core-site'], + configuration_attributes=params.config['configuration_attributes']['core-site'], + owner = params.hbase_user, + group = params.user_group + ) + + if 'hdfs-site' in params.config['configurations']: + XmlConfig( "hdfs-site.xml", + conf_dir = params.hbase_conf_dir, + configurations = params.config['configurations']['hdfs-site'], + configuration_attributes=params.config['configuration_attributes']['hdfs-site'], + owner = params.hbase_user, + group = params.user_group + ) + + XmlConfig("hdfs-site.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['hdfs-site'], + configuration_attributes=params.config['configuration_attributes']['hdfs-site'], + owner=params.hdfs_user, + group=params.user_group + ) + + if 'hbase-policy' in params.config['configurations']: + XmlConfig( "hbase-policy.xml", + conf_dir = params.hbase_conf_dir, + configurations = params.config['configurations']['hbase-policy'], + configuration_attributes=params.config['configuration_attributes']['hbase-policy'], + owner = params.hbase_user, + group = params.user_group + ) + # Manually overriding ownership of file installed by hadoop package + else: + File( format("{params.hbase_conf_dir}/hbase-policy.xml"), + owner = params.hbase_user, + group = params.user_group + ) + + File(format("{hbase_conf_dir}/hbase-env.sh"), + owner = params.hbase_user, + content=InlineTemplate(params.hbase_env_sh_template), + group = params.user_group, + ) + + # On some OS this folder could be not exists, so we will create it before pushing there files + Directory(params.limits_conf_dir, + create_parents = True, + owner='root', + group='root' + ) + + File(os.path.join(params.limits_conf_dir, 'hbase.conf'), + owner='root', + group='root', + mode=0644, + content=Template("hbase.conf.j2") + ) + + hbase_TemplateConfig( params.metric_prop_file_name, + tag = 'GANGLIA-MASTER' if name == 'master' else 'GANGLIA-RS' + ) + + hbase_TemplateConfig( 'regionservers') + + if params.security_enabled: + hbase_TemplateConfig( format("hbase_{name}_jaas.conf")) + + if name != "client": + Directory( params.pid_dir, + owner = params.hbase_user, + create_parents = True, + cd_access = "a", + mode = 0755, + ) + + Directory (params.log_dir, + owner = params.hbase_user, + create_parents = True, + cd_access = "a", + mode = 0755, + ) + + if (params.log4j_props != None): + File(format("{params.hbase_conf_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.hbase_user, + content=params.log4j_props + ) + elif (os.path.exists(format("{params.hbase_conf_dir}/log4j.properties"))): + File(format("{params.hbase_conf_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.hbase_user + ) + if name == "master": + if not params.hbase_root_dir_scheme or params.hbase_root_dir_scheme == urlparse(params.default_fs).scheme: # if hbase.rootdir has no scheme specified or is in HDFS + # Create hbase.rootdir + params.HdfsResource(params.hbase_hdfs_root_dir, + type="directory", + action="create_on_execute", + owner=params.hbase_user + ) + else: # hbase.rootdir is not in HDFS + # Do NOT create hbase.rootdir, but only log a message + Logger.info(format("hbase.rootidr '{params.hbase_hdfs_root_dir} not created', as its location is not in HDFS.")) + + params.HdfsResource(params.hbase_staging_dir, + type="directory", + action="create_on_execute", + owner=params.hbase_user, + mode=0711 + ) + if params.create_hbase_home_directory: + params.HdfsResource(params.hbase_home_directory, + type="directory", + action="create_on_execute", + owner=params.hbase_user, + mode=0755 + ) + params.HdfsResource(None, action="execute") + + if params.phoenix_enabled: + Package(params.phoenix_package, + retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, + retry_count=params.agent_stack_retry_count) + +def hbase_TemplateConfig(name, tag=None): + import params + + TemplateConfig( format("{hbase_conf_dir}/{name}"), + owner = params.hbase_user, + template_tag = tag + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_client.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_client.py new file mode 100644 index 0000000..f18a96a --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_client.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from hbase import hbase +from ambari_commons import OSCheck, OSConst +from ambari_commons.os_family_impl import OsFamilyImpl +from resource_management.core.exceptions import ClientComponentHasNoStatus + +class HbaseClient(Script): + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + hbase(name='client') + + def status(self, env): + raise ClientComponentHasNoStatus() + + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class HbaseClientWindows(HbaseClient): + pass + + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class HbaseClientDefault(HbaseClient): + def get_component_name(self): + return "hbase-client" + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + conf_select.select(params.stack_name, "hbase", params.version) + stack_select.select("hbase-client", params.version) + + # phoenix may not always be deployed + try: + stack_select.select("phoenix-client", params.version) + except Exception as e: + print "Ignoring error due to missing phoenix-client" + print str(e) + + + # set all of the hadoop clients since hbase client is upgraded as part + # of the final "CLIENTS" group and we need to ensure that hadoop-client + # is also set + conf_select.select(params.stack_name, "hadoop", params.version) + stack_select.select("hadoop-client", params.version) + + +if __name__ == "__main__": + HbaseClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_decommission.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_decommission.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_decommission.py new file mode 100644 index 0000000..022465a --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_decommission.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def hbase_decommission(env): + import params + + env.set_params(params) + File(params.region_drainer, content=StaticFile("draining_servers.rb"), owner=params.hbase_user, mode="f") + + hosts = params.hbase_excluded_hosts.split(",") + for host in hosts: + if host: + if params.hbase_drain_only == True: + regiondrainer_cmd = format("cmd /c {hbase_executable} org.jruby.Main {region_drainer} remove {host}") + Execute(regiondrainer_cmd, user=params.hbase_user, logoutput=True) + else: + regiondrainer_cmd = format("cmd /c {hbase_executable} org.jruby.Main {region_drainer} add {host}") + regionmover_cmd = format("cmd /c {hbase_executable} org.jruby.Main {region_mover} unload {host}") + Execute(regiondrainer_cmd, user=params.hbase_user, logoutput=True) + Execute(regionmover_cmd, user=params.hbase_user, logoutput=True) + + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def hbase_decommission(env): + import params + + env.set_params(params) + kinit_cmd = params.kinit_cmd_master + + File(params.region_drainer, + content=StaticFile("draining_servers.rb"), + mode=0755 + ) + + if params.hbase_excluded_hosts and params.hbase_excluded_hosts.split(","): + hosts = params.hbase_excluded_hosts.split(",") + elif params.hbase_included_hosts and params.hbase_included_hosts.split(","): + hosts = params.hbase_included_hosts.split(",") + + if params.hbase_drain_only: + for host in hosts: + if host: + regiondrainer_cmd = format( + "{kinit_cmd} {hbase_cmd} --config {hbase_conf_dir} {master_security_config} org.jruby.Main {region_drainer} remove {host}") + Execute(regiondrainer_cmd, + user=params.hbase_user, + logoutput=True + ) + pass + pass + + else: + for host in hosts: + if host: + regiondrainer_cmd = format( + "{kinit_cmd} {hbase_cmd} --config {hbase_conf_dir} {master_security_config} org.jruby.Main {region_drainer} add {host}") + regionmover_cmd = format( + "{kinit_cmd} {hbase_cmd} --config {hbase_conf_dir} {master_security_config} org.jruby.Main {region_mover} unload {host}") + + Execute(regiondrainer_cmd, + user=params.hbase_user, + logoutput=True + ) + + Execute(regionmover_cmd, + user=params.hbase_user, + logoutput=True + ) + pass + pass + pass http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_master.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_master.py new file mode 100644 index 0000000..ae0e0d9 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_master.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys + +from resource_management.libraries.script.script import Script +from resource_management.core.resources.service import Service +from resource_management.libraries.functions.security_commons import build_expectations, \ + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_XML +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.libraries.functions.format import format +from hbase import hbase +from hbase_service import hbase_service +from hbase_decommission import hbase_decommission +import upgrade +from setup_ranger_hbase import setup_ranger_hbase +from ambari_commons import OSCheck, OSConst +from ambari_commons.os_family_impl import OsFamilyImpl + +if OSCheck.is_windows_family(): + from resource_management.libraries.functions.windows_service_utils import check_windows_service_status + +class HbaseMaster(Script): + def configure(self, env): + import params + env.set_params(params) + hbase(name='master') + + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + + def decommission(self, env): + import params + env.set_params(params) + hbase_decommission(env) + + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class HbaseMasterWindows(HbaseMaster): + def start(self, env): + import status_params + self.configure(env) + Service(status_params.hbase_master_win_service_name, action="start") + + def stop(self, env): + import status_params + env.set_params(status_params) + Service(status_params.hbase_master_win_service_name, action="stop") + + def status(self, env): + import status_params + env.set_params(status_params) + check_windows_service_status(status_params.hbase_master_win_service_name) + + + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class HbaseMasterDefault(HbaseMaster): + def get_component_name(self): + return "hbase-master" + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + upgrade.prestart(env, "hbase-master") + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) # for security + setup_ranger_hbase(upgrade_type=upgrade_type, service_name="hbase-master") + hbase_service('master', action='start') + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + hbase_service('master', action = 'stop') + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = format("{pid_dir}/hbase-{hbase_user}-master.pid") + check_process_status(pid_file) + + def security_status(self, env): + import status_params + + env.set_params(status_params) + if status_params.security_enabled: + props_value_check = {"hbase.security.authentication" : "kerberos", + "hbase.security.authorization": "true"} + props_empty_check = ['hbase.master.keytab.file', + 'hbase.master.kerberos.principal'] + props_read_check = ['hbase.master.keytab.file'] + hbase_site_expectations = build_expectations('hbase-site', props_value_check, props_empty_check, + props_read_check) + + hbase_expectations = {} + hbase_expectations.update(hbase_site_expectations) + + security_params = get_params_from_filesystem(status_params.hbase_conf_dir, + {'hbase-site.xml': FILE_TYPE_XML}) + result_issues = validate_security_config_properties(security_params, hbase_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ( 'hbase-site' not in security_params + or 'hbase.master.keytab.file' not in security_params['hbase-site'] + or 'hbase.master.kerberos.principal' not in security_params['hbase-site']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.hbase_user, + security_params['hbase-site']['hbase.master.keytab.file'], + security_params['hbase-site']['hbase.master.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) + else: + self.put_structured_out({"securityState": "UNSECURED"}) + + def get_log_folder(self): + import params + return params.log_dir + + def get_user(self): + import params + return params.hbase_user + +if __name__ == "__main__": + HbaseMaster().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_regionserver.py new file mode 100644 index 0000000..370167b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_regionserver.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys + +from resource_management import * +from resource_management.core import shell +from resource_management.libraries.functions.security_commons import build_expectations, \ + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_XML +from hbase import hbase +from hbase_service import hbase_service +import upgrade +from setup_ranger_hbase import setup_ranger_hbase +from ambari_commons import OSCheck, OSConst +from ambari_commons.os_family_impl import OsFamilyImpl + + +class HbaseRegionServer(Script): + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + hbase(name='regionserver') + + def decommission(self, env): + print "Decommission not yet implemented!" + + + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class HbaseRegionServerWindows(HbaseRegionServer): + def start(self, env): + import status_params + self.configure(env) + Service(status_params.hbase_regionserver_win_service_name, action="start") + + def stop(self, env): + import status_params + env.set_params(status_params) + Service(status_params.hbase_regionserver_win_service_name, action="stop") + + def status(self, env): + import status_params + env.set_params(status_params) + check_windows_service_status(status_params.hbase_regionserver_win_service_name) + + + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class HbaseRegionServerDefault(HbaseRegionServer): + def get_component_name(self): + return "hbase-regionserver" + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + upgrade.prestart(env, "hbase-regionserver") + + def post_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + upgrade.post_regionserver(env) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) # for security + setup_ranger_hbase(upgrade_type=upgrade_type, service_name="hbase-regionserver") + + hbase_service('regionserver', action='start') + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + hbase_service( 'regionserver', + action = 'stop' + ) + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = format("{pid_dir}/hbase-{hbase_user}-regionserver.pid") + check_process_status(pid_file) + + def security_status(self, env): + import status_params + + env.set_params(status_params) + if status_params.security_enabled: + props_value_check = {"hbase.security.authentication" : "kerberos", + "hbase.security.authorization": "true"} + props_empty_check = ['hbase.regionserver.keytab.file', + 'hbase.regionserver.kerberos.principal'] + props_read_check = ['hbase.regionserver.keytab.file'] + hbase_site_expectations = build_expectations('hbase-site', props_value_check, props_empty_check, + props_read_check) + + hbase_expectations = {} + hbase_expectations.update(hbase_site_expectations) + + security_params = get_params_from_filesystem(status_params.hbase_conf_dir, + {'hbase-site.xml': FILE_TYPE_XML}) + result_issues = validate_security_config_properties(security_params, hbase_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ( 'hbase-site' not in security_params + or 'hbase.regionserver.keytab.file' not in security_params['hbase-site'] + or 'hbase.regionserver.kerberos.principal' not in security_params['hbase-site']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.hbase_user, + security_params['hbase-site']['hbase.regionserver.keytab.file'], + security_params['hbase-site']['hbase.regionserver.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) + else: + self.put_structured_out({"securityState": "UNSECURED"}) + + def get_log_folder(self): + import params + return params.log_dir + + def get_user(self): + import params + return params.hbase_user + +if __name__ == "__main__": + HbaseRegionServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_restgatewayserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_restgatewayserver.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_restgatewayserver.py new file mode 100644 index 0000000..1d028d6 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_restgatewayserver.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * + +from hbase import hbase +from hbase_service import hbase_service +from hbase_decommission import hbase_decommission +import upgrade + +class HbaseRestGatewayServer(Script): + def get_component_name(self): + return "hbase-restserver" + + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + + hbase(name='rest') + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + upgrade.prestart(env, "hbase-restserver") + + def post_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) # for security + + hbase_service( 'rest', + action = 'start' + ) + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + hbase_service( 'rest', + action = 'stop' + ) + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = format("{pid_dir}/hbase-{hbase_user}-rest.pid") + check_process_status(pid_file) + + def decommission(self, env): + import params + env.set_params(params) + + hbase_decommission(env) + + +if __name__ == "__main__": + HbaseRestGatewayServer().execute() + http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_service.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_service.py new file mode 100644 index 0000000..41daaeb --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_service.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from datetime import datetime + +from resource_management.core.resources.system import Execute +from resource_management.core.resources.system import File +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.format import format +from resource_management.core.shell import as_sudo +from resource_management.libraries.functions.show_logs import show_logs +from resource_management.core.logger import Logger + +def hbase_service(name, action = 'start'): + import params + + role = name + cmd = format("{daemon_script} --config {hbase_conf_dir}") + pid_file = format("{pid_dir}/hbase-{hbase_user}-{role}.pid") + pid_expression = as_sudo(["cat", pid_file]) + no_op_test = as_sudo(["test", "-f", pid_file]) + format(" && ps -p `{pid_expression}` >/dev/null 2>&1") + + # delete wal log if HBase version has moved down + if params.to_backup_wal_dir: + wal_directory = params.wal_directory + timestamp = datetime.now() + timestamp_format = '%Y%m%d%H%M%S' + wal_directory_backup = '%s_%s' % (wal_directory, timestamp.strftime(timestamp_format)) + + check_if_wal_dir_exists = format("hdfs dfs -ls {wal_directory}") + wal_dir_exists = False + try: + Execute(check_if_wal_dir_exists, + user=params.hbase_user + ) + wal_dir_exists = True + except Exception, e: + Logger.error(format("Did not find HBase WAL directory {wal_directory}. It's possible that it was already moved. Exception: {e.message}")) + + if wal_dir_exists: + move_wal_dir_cmd = format("hdfs dfs -mv {wal_directory} {wal_directory_backup}") + try: + Execute(move_wal_dir_cmd, + user=params.hbase_user + ) + except Exception, e: + Logger.error(format("Failed to backup HBase WAL directory, command: {move_wal_dir_cmd} . Exception: {e.message}")) + + if action == 'start': + daemon_cmd = format("{cmd} start {role}") + + try: + Execute ( daemon_cmd, + not_if = no_op_test, + user = params.hbase_user + ) + except: + show_logs(params.log_dir, params.hbase_user) + raise + elif action == 'stop': + daemon_cmd = format("{cmd} stop {role}") + + try: + Execute ( daemon_cmd, + user = params.hbase_user, + only_if = no_op_test, + # BUGFIX: hbase regionserver sometimes hangs when nn is in safemode + timeout = params.hbase_regionserver_shutdown_timeout, + on_timeout = format("! ( {no_op_test} ) || {sudo} -H -E kill -9 `{pid_expression}`"), + ) + except: + show_logs(params.log_dir, params.hbase_user) + raise + + File(pid_file, + action = "delete", + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_upgrade.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_upgrade.py new file mode 100644 index 0000000..2dc9883 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/hbase_upgrade.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management.libraries.script import Script +from resource_management.core.resources.system import Execute + +class HbaseMasterUpgrade(Script): + + def take_snapshot(self, env): + import params + + snap_cmd = "echo 'snapshot_all' | {0} shell".format(params.hbase_cmd) + + exec_cmd = "{0} {1}".format(params.kinit_cmd, snap_cmd) + + Execute(exec_cmd, user=params.hbase_user) + + def restore_snapshot(self, env): + import params + print "TODO AMBARI-12698" + +if __name__ == "__main__": + HbaseMasterUpgrade().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/params.py new file mode 100644 index 0000000..f3208ce --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/params.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from ambari_commons import OSCheck +from resource_management.libraries.functions.default import default + +if OSCheck.is_windows_family(): + from params_windows import * +else: + from params_linux import * + +host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) +retryAble = default("/commandParams/command_retry_enabled", False) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/params_linux.py new file mode 100644 index 0000000..bff022d --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/params_linux.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os +import status_params +import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. + +from urlparse import urlparse + +from functions import calc_xmn_from_xms, ensure_unit_for_memory + +from ambari_commons.constants import AMBARI_SUDO_BINARY +from ambari_commons.os_check import OSCheck +from ambari_commons.str_utils import string_set_intersection + +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import format +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.stack_features import get_stack_feature_version +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions import is_empty +from resource_management.libraries.functions import get_unique_id_and_date +from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.expect import expect +from resource_management.libraries.functions.constants import Direction + +# server configurations +config = Script.get_config() +exec_tmp_dir = Script.get_tmp_dir() +sudo = AMBARI_SUDO_BINARY + +stack_name = status_params.stack_name +agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability'] +agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int) +version = default("/commandParams/version", None) +component_directory = status_params.component_directory +etc_prefix_dir = "/etc/hbase" + +stack_version_unformatted = status_params.stack_version_unformatted +stack_version_formatted = status_params.stack_version_formatted +stack_root = status_params.stack_root + +# get the correct version to use for checking stack features +version_for_stack_feature_checks = get_stack_feature_version(config) + +stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks) +stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks) + +# hadoop default parameters +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +daemon_script = "/usr/lib/hbase/bin/hbase-daemon.sh" +region_mover = "/usr/lib/hbase/bin/region_mover.rb" +region_drainer = "/usr/lib/hbase/bin/draining_servers.rb" +hbase_cmd = "/usr/lib/hbase/bin/hbase" +hbase_max_direct_memory_size = None + +# hadoop parameters for stacks supporting rolling_upgrade +if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): + daemon_script = format('{stack_root}/current/hbase-client/bin/hbase-daemon.sh') + region_mover = format('{stack_root}/current/hbase-client/bin/region_mover.rb') + region_drainer = format('{stack_root}/current/hbase-client/bin/draining_servers.rb') + hbase_cmd = format('{stack_root}/current/hbase-client/bin/hbase') + + hbase_max_direct_memory_size = default('configurations/hbase-env/hbase_max_direct_memory_size', None) + + tmp_daemon_script = format("{stack_root}/current/{component_directory}/bin/hbase-daemon.sh") + if os.path.exists(tmp_daemon_script): + daemon_script = tmp_daemon_script + + region_mover = format("{stack_root}/current/{component_directory}/bin/region_mover.rb") + region_drainer = format("{stack_root}/current/{component_directory}/bin/draining_servers.rb") + hbase_cmd = format("{stack_root}/current/{component_directory}/bin/hbase") + + +hbase_conf_dir = status_params.hbase_conf_dir +limits_conf_dir = status_params.limits_conf_dir + +hbase_user_nofile_limit = default("/configurations/hbase-env/hbase_user_nofile_limit", "32000") +hbase_user_nproc_limit = default("/configurations/hbase-env/hbase_user_nproc_limit", "16000") + +# no symlink for phoenix-server at this point +phx_daemon_script = format('{stack_root}/current/phoenix-server/bin/queryserver.py') + +hbase_excluded_hosts = config['commandParams']['excluded_hosts'] +hbase_drain_only = default("/commandParams/mark_draining_only",False) +hbase_included_hosts = config['commandParams']['included_hosts'] + +hbase_user = status_params.hbase_user +hbase_principal_name = config['configurations']['hbase-env']['hbase_principal_name'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +_authentication = config['configurations']['core-site']['hadoop.security.authentication'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] + +# this is "hadoop-metrics.properties" for 1.x stacks +metric_prop_file_name = "hadoop-metrics2-hbase.properties" + +# not supporting 32 bit jdk. +java64_home = config['hostLevelParams']['java_home'] +java_version = expect("/hostLevelParams/java_version", int) + +log_dir = config['configurations']['hbase-env']['hbase_log_dir'] +java_io_tmpdir = default("/configurations/hbase-env/hbase_java_io_tmpdir", "/tmp") +master_heapsize = ensure_unit_for_memory(config['configurations']['hbase-env']['hbase_master_heapsize']) + +regionserver_heapsize = ensure_unit_for_memory(config['configurations']['hbase-env']['hbase_regionserver_heapsize']) +regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max'] +regionserver_xmn_percent = expect("/configurations/hbase-env/hbase_regionserver_xmn_ratio", float) +regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max) + +hbase_regionserver_shutdown_timeout = expect('/configurations/hbase-env/hbase_regionserver_shutdown_timeout', int, 30) + +phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', []) +phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False) +has_phoenix = len(phoenix_hosts) > 0 + +underscored_version = stack_version_unformatted.replace('.', '_') +dashed_version = stack_version_unformatted.replace('.', '-') +if OSCheck.is_redhat_family() or OSCheck.is_suse_family(): + phoenix_package = format("phoenix_{underscored_version}_*") +elif OSCheck.is_ubuntu_family(): + phoenix_package = format("phoenix-{dashed_version}-.*") + +pid_dir = status_params.pid_dir +tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] +local_dir = config['configurations']['hbase-site']['hbase.local.dir'] +ioengine_param = default('/configurations/hbase-site/hbase.bucketcache.ioengine', None) + +client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf") +master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf") +regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas.conf") +rest_jaas_config_file = format("{hbase_conf_dir}/hbase_rest_jaas.conf") +queryserver_jaas_config_file = format("{hbase_conf_dir}/hbase_queryserver_jaas.conf") + +ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed when ganglia is not present +has_ganglia_server = not len(ganglia_server_hosts) == 0 +if has_ganglia_server: + ganglia_server_host = ganglia_server_hosts[0] + +ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", []) +has_metric_collector = not len(ams_collector_hosts) == 0 +if has_metric_collector: + if 'cluster-env' in config['configurations'] and \ + 'metrics_collector_vip_host' in config['configurations']['cluster-env']: + metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host'] + else: + metric_collector_host = ams_collector_hosts[0] + if 'cluster-env' in config['configurations'] and \ + 'metrics_collector_vip_port' in config['configurations']['cluster-env']: + metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port'] + else: + metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "localhost:6188") + if metric_collector_web_address.find(':') != -1: + metric_collector_port = metric_collector_web_address.split(':')[1] + else: + metric_collector_port = '6188' + if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY": + metric_collector_protocol = 'https' + else: + metric_collector_protocol = 'http' + metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "") + metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "") + metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "") + + pass +metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60) +metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10) + +# if hbase is selected the hbase_rs_hosts, should not be empty, but still default just in case +if 'slave_hosts' in config['clusterHostInfo']: + rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/slave_hosts') #if hbase_rs_hosts not given it is assumed that region servers on same nodes as slaves +else: + rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/all_hosts') + +smoke_test_user = config['configurations']['cluster-env']['smokeuser'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +smokeuser_permissions = "RWXCA" + +titan_user = default('configurations/titan-env/titan_user', 'titan') +titan_user_permissions = "RWC" +titan_user_hbase_namespace = "@default" + +service_check_data = get_unique_id_and_date() +user_group = config['configurations']['cluster-env']["user_group"] + +if security_enabled: + _hostname_lowercase = config['hostname'].lower() + master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase) + master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file'] + regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase) + _rest_server_jaas_princ = config['configurations']['hbase-site']['hbase.rest.kerberos.principal'] + _rest_server_spnego_jaas_princ = config['configurations']['hbase-site']['hbase.rest.authentication.kerberos.principal'] + _queryserver_jaas_princ = config['configurations']['hbase-site']['phoenix.queryserver.kerberos.principal'] + if not is_empty(_queryserver_jaas_princ): + queryserver_jaas_princ =_queryserver_jaas_princ.replace('_HOST',_hostname_lowercase) + if not is_empty(_rest_server_jaas_princ): + rest_server_jaas_princ = _rest_server_jaas_princ.replace('_HOST',_hostname_lowercase) + if not is_empty(_rest_server_spnego_jaas_princ): + rest_server_spnego_jaas_princ = _rest_server_spnego_jaas_princ.replace('_HOST',_hostname_lowercase) + +regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file'] +rest_server_keytab_path = config['configurations']['hbase-site']['hbase.rest.keytab.file'] +rest_server_spnego_keytab_path = config['configurations']['hbase-site']['hbase.rest.authentication.kerberos.keytab'] +queryserver_keytab_path = config['configurations']['hbase-site']['phoenix.queryserver.keytab.file'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +hbase_user_keytab = config['configurations']['hbase-env']['hbase_user_keytab'] +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) +if security_enabled: + kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_principal_name};") + kinit_cmd_master = format("{kinit_path_local} -kt {master_keytab_path} {master_jaas_princ};") + master_security_config = format("-Djava.security.auth.login.config={hbase_conf_dir}/hbase_master_jaas.conf") +else: + kinit_cmd = "" + kinit_cmd_master = "" + master_security_config = "" + +#log4j.properties +if (('hbase-log4j' in config['configurations']) and ('content' in config['configurations']['hbase-log4j'])): + log4j_props = config['configurations']['hbase-log4j']['content'] +else: + log4j_props = None + +hbase_env_sh_template = config['configurations']['hbase-env']['content'] + +hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir'] +hbase_root_dir_scheme = urlparse(hbase_hdfs_root_dir).scheme + +hbase_staging_dir = "/apps/hbase/staging" +#for create_hdfs_directory +hostname = config["hostname"] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] + + + +hdfs_site = config['configurations']['hdfs-site'] +default_fs = config['configurations']['core-site']['fs.defaultFS'] + +dfs_type = default("/commandParams/dfs_type", "") + +import functools +#create partial functions with common arguments for every HdfsResource call +#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs, + immutable_paths = get_not_managed_resources(), + dfs_type = dfs_type +) + +# ranger host +ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) +has_ranger_admin = not len(ranger_admin_hosts) == 0 +xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] +ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] + +# ranger hbase properties +policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] +if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'): + policymgr_mgr_url = policymgr_mgr_url.rstrip('/') +xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits') +xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') +xa_db_host = config['configurations']['admin-properties']['db_host'] +repo_name = str(config['clusterName']) + '_hbase' + +common_name_for_certificate = config['configurations']['ranger-hbase-plugin-properties']['common.name.for.certificate'] + +zookeeper_znode_parent = config['configurations']['hbase-site']['zookeeper.znode.parent'] +hbase_zookeeper_quorum = config['configurations']['hbase-site']['hbase.zookeeper.quorum'] +hbase_zookeeper_property_clientPort = config['configurations']['hbase-site']['hbase.zookeeper.property.clientPort'] +hbase_security_authentication = config['configurations']['hbase-site']['hbase.security.authentication'] +hadoop_security_authentication = config['configurations']['core-site']['hadoop.security.authentication'] + +repo_config_username = config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] + +ranger_env = config['configurations']['ranger-env'] +ranger_plugin_properties = config['configurations']['ranger-hbase-plugin-properties'] +policy_user = config['configurations']['ranger-hbase-plugin-properties']['policy_user'] + +#For curl command in ranger plugin to get db connector +jdk_location = config['hostLevelParams']['jdk_location'] +java_share_dir = '/usr/share/java' +enable_ranger_hbase = False +if has_ranger_admin: + enable_ranger_hbase = (config['configurations']['ranger-hbase-plugin-properties']['ranger-hbase-plugin-enabled'].lower() == 'yes') + xa_audit_db_password = '' + if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db: + xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) + repo_config_password = unicode(config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']) + xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() + previous_jdbc_jar_name = None + + if stack_supports_ranger_audit_db: + if xa_audit_db_flavor == 'mysql': + jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) + audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}') + jdbc_driver = "com.mysql.jdbc.Driver" + elif xa_audit_db_flavor == 'oracle': + jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) + colon_count = xa_db_host.count(':') + if colon_count == 2 or colon_count == 0: + audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}') + else: + audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}') + jdbc_driver = "oracle.jdbc.OracleDriver" + elif xa_audit_db_flavor == 'postgres': + jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None) + audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}') + jdbc_driver = "org.postgresql.Driver" + elif xa_audit_db_flavor == 'mssql': + jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) + audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}') + jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" + elif xa_audit_db_flavor == 'sqla': + jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None) + audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}') + jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" + + downloaded_custom_connector = format("{exec_tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + driver_curl_target = format("{stack_root}/current/{component_directory}/lib/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + previous_jdbc_jar = format("{stack_root}/current/{component_directory}/lib/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None + sql_connector_jar = '' + + if security_enabled: + master_principal = config['configurations']['hbase-site']['hbase.master.kerberos.principal'] + + hbase_ranger_plugin_config = { + 'username': repo_config_username, + 'password': repo_config_password, + 'hadoop.security.authentication': hadoop_security_authentication, + 'hbase.security.authentication': hbase_security_authentication, + 'hbase.zookeeper.property.clientPort': hbase_zookeeper_property_clientPort, + 'hbase.zookeeper.quorum': hbase_zookeeper_quorum, + 'zookeeper.znode.parent': zookeeper_znode_parent, + 'commonNameForCertificate': common_name_for_certificate, + 'hbase.master.kerberos.principal': master_principal if security_enabled else '' + } + + hbase_ranger_plugin_repo = { + 'isActive': 'true', + 'config': json.dumps(hbase_ranger_plugin_config), + 'description': 'hbase repo', + 'name': repo_name, + 'repositoryType': 'hbase', + 'assetType': '2' + } + + if stack_supports_ranger_kerberos and security_enabled: + hbase_ranger_plugin_config['policy.download.auth.users'] = hbase_user + hbase_ranger_plugin_config['tag.download.auth.users'] = hbase_user + hbase_ranger_plugin_config['policy.grantrevoke.auth.users'] = hbase_user + + if stack_supports_ranger_kerberos: + hbase_ranger_plugin_config['ambari.service.check.user'] = policy_user + + hbase_ranger_plugin_repo = { + 'isEnabled': 'true', + 'configs': hbase_ranger_plugin_config, + 'description': 'hbase repo', + 'name': repo_name, + 'type': 'hbase' + } + + ranger_hbase_principal = None + ranger_hbase_keytab = None + if stack_supports_ranger_kerberos and security_enabled and 'hbase-master' in component_directory.lower(): + ranger_hbase_principal = master_jaas_princ + ranger_hbase_keytab = master_keytab_path + elif stack_supports_ranger_kerberos and security_enabled and 'hbase-regionserver' in component_directory.lower(): + ranger_hbase_principal = regionserver_jaas_princ + ranger_hbase_keytab = regionserver_keytab_path + + xa_audit_db_is_enabled = False + ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] + if xml_configurations_supported and stack_supports_ranger_audit_db: + xa_audit_db_is_enabled = config['configurations']['ranger-hbase-audit']['xasecure.audit.destination.db'] + xa_audit_hdfs_is_enabled = config['configurations']['ranger-hbase-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None + ssl_keystore_password = unicode(config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None + ssl_truststore_password = unicode(config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None + credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None + + #For SQLA explicitly disable audit to DB for Ranger + if xa_audit_db_flavor == 'sqla': + xa_audit_db_is_enabled = False + + +create_hbase_home_directory = check_stack_feature(StackFeature.HBASE_HOME_DIRECTORY, stack_version_formatted) +hbase_home_directory = format("/user/{hbase_user}") + +atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', []) +has_atlas = len(atlas_hosts) > 0 + +metadata_user = default('/configurations/atlas-env/metadata_user', None) +atlas_graph_storage_hostname = default('/configurations/application-properties/atlas.graph.storage.hostname', None) +atlas_graph_storage_hbase_table = default('/configurations/application-properties/atlas.graph.storage.hbase.table', None) +atlas_audit_hbase_tablename = default('/configurations/application-properties/atlas.audit.hbase.tablename', None) + +if has_atlas: + zk_hosts_matches = string_set_intersection(atlas_graph_storage_hostname, hbase_zookeeper_quorum) + atlas_with_managed_hbase = len(zk_hosts_matches) > 0 +else: + atlas_with_managed_hbase = False + +wal_directory = "/apps/hbase/data/MasterProcWALs" + +backup_wal_dir = default('/configurations/hbase-env/backup_wal_dir', False) + +#Need to make sure not to keep removing WAL logs once EU is finalized. +upgrade_direction = default("/commandParams/upgrade_direction", None) +to_backup_wal_dir = upgrade_direction is not None and upgrade_direction == Direction.UPGRADE and backup_wal_dir http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/params_windows.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/params_windows.py new file mode 100644 index 0000000..7d634cb --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/params_windows.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +import os +import status_params + +# server configurations +config = Script.get_config() +hbase_conf_dir = os.environ["HBASE_CONF_DIR"] +hbase_bin_dir = os.path.join(os.environ["HBASE_HOME"],'bin') +hbase_executable = os.path.join(hbase_bin_dir,"hbase.cmd") +stack_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],"..")) +hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"] +hbase_user = hadoop_user + +#decomm params +region_drainer = os.path.join(hbase_bin_dir,"draining_servers.rb") +region_mover = os.path.join(hbase_bin_dir,"region_mover.rb") +hbase_excluded_hosts = config['commandParams']['excluded_hosts'] +hbase_drain_only = config['commandParams']['mark_draining_only'] + +service_map = { + 'master' : status_params.hbase_master_win_service_name, + 'regionserver' : status_params.hbase_regionserver_win_service_name +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b1c3784d/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/phoenix_queryserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/phoenix_queryserver.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/phoenix_queryserver.py new file mode 100644 index 0000000..82113e9 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HBASE/package/scripts/phoenix_queryserver.py @@ -0,0 +1,88 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.script import Script +from phoenix_service import phoenix_service +from hbase import hbase + +# Note: Phoenix Query Server is only applicable to stack version supporting Phoenix. +class PhoenixQueryServer(Script): + + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + + + def get_component_name(self): + return "phoenix-server" + + + def configure(self, env): + import params + env.set_params(params) + hbase(name='queryserver') + + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) + phoenix_service('start') + + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + phoenix_service('stop') + + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + if params.stack_version_formatted and check_stack_feature(StackFeature.PHOENIX, params.stack_version_formatted): + # phoenix uses hbase configs + conf_select.select(params.stack_name, "hbase", params.version) + stack_select.select("phoenix-server", params.version) + + + def status(self, env): + import status_params + env.set_params(status_params) + phoenix_service('status') + + + def security_status(self, env): + self.put_structured_out({"securityState": "UNSECURED"}) + + def get_log_folder(self): + import params + return params.log_dir + + def get_user(self): + import params + return params.hbase_user + +if __name__ == "__main__": + PhoenixQueryServer().execute() \ No newline at end of file
