This is an automated email from the ASF dual-hosted git repository. vjasani pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push: new a0eb5316f7 AMBARI-25949 regionserver graceful start/stop should be compatible for hbase 2 (#3702) a0eb5316f7 is described below commit a0eb5316f75d7c7a90b04f605c94b99dce9355c3 Author: Viraj Jasani <vjas...@apache.org> AuthorDate: Mon Jul 31 11:58:53 2023 -0700 AMBARI-25949 regionserver graceful start/stop should be compatible for hbase 2 (#3702) --- .../HBASE/package/files/draining_servers.rb | 165 +++++++++++++++++++++ .../HBASE/package/files/draining_servers2.rb | 159 ++++++++++++++++++++ .../HBASE/package/scripts/hbase_decommission.py | 26 +++- .../HBASE/package/scripts/hbase_regionserver.py | 60 +++++++- .../services/HBASE/package/scripts/params_linux.py | 3 + 5 files changed, 401 insertions(+), 12 deletions(-) diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/files/draining_servers.rb b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/files/draining_servers.rb new file mode 100644 index 0000000000..ab3ed10447 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/files/draining_servers.rb @@ -0,0 +1,165 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Add or remove servers from draining mode via zookeeper + +require 'optparse' +include Java + +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.client.HBaseAdmin +import org.apache.hadoop.hbase.zookeeper.ZKUtil +import org.apache.commons.logging.Log +import org.apache.commons.logging.LogFactory + +# Name of this script +NAME = "draining_servers" + +# Do command-line parsing +options = {} +optparse = OptionParser.new do |opts| + opts.banner = "Usage: ./hbase org.jruby.Main #{NAME}.rb [options] add|remove|list <hostname>|<host:port>|<servername> ..." + opts.separator 'Add remove or list servers in draining mode. Can accept either hostname to drain all region servers' + + 'in that host, a host:port pair or a host,port,startCode triplet. More than one server can be given separated by space' + opts.on('-h', '--help', 'Display usage information') do + puts opts + exit + end + options[:debug] = false + opts.on('-d', '--debug', 'Display extra debug logging') do + options[:debug] = true + end +end +optparse.parse! + +# Return array of servernames where servername is hostname+port+startcode +# comma-delimited +def getServers(admin) + serverInfos = admin.getClusterStatus().getServerInfo() + servers = [] + for server in serverInfos + servers << server.getServerName() + end + return servers +end + +def getServerNames(hostOrServers, config) + ret = [] + + for hostOrServer in hostOrServers + # check whether it is already serverName. No need to connect to cluster + parts = hostOrServer.split(',') + if parts.size() == 3 + ret << hostOrServer + else + admin = HBaseAdmin.new(config) if not admin + servers = getServers(admin) + + hostOrServer = hostOrServer.gsub(/:/, ",") + for server in servers + ret << server if server.start_with?(hostOrServer) + end + end + end + + admin.close() if admin + return ret +end + +def addServers(options, hostOrServers) + config = HBaseConfiguration.create() + servers = getServerNames(hostOrServers, config) + + zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(config, "draining_servers", nil) + parentZnode = zkw.drainingZNode + + begin + for server in servers + node = ZKUtil.joinZNode(parentZnode, server) + ZKUtil.createAndFailSilent(zkw, node) + end + ensure + zkw.close() + end +end + +def removeServers(options, hostOrServers) + config = HBaseConfiguration.create() + zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(config, "draining_servers", nil) + parentZnode = zkw.drainingZNode + servers = ZKUtil.listChildrenNoWatch(zkw, parentZnode) + + begin + for server in servers + if hostOrServers.include?(server.split(',')[0]) + node = ZKUtil.joinZNode(parentZnode, server) + ZKUtil.deleteNodeFailSilent(zkw, node) + end + end + ensure + zkw.close() + end +end + +# list servers in draining mode +def listServers(options) + config = HBaseConfiguration.create() + + zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(config, "draining_servers", nil) + parentZnode = zkw.drainingZNode + + servers = ZKUtil.listChildrenNoWatch(zkw, parentZnode) + servers.each {|server| puts server} +end + +hostOrServers = ARGV[1..ARGV.size()] + +# Create a logger and disable the DEBUG-level annoying client logging +def configureLogging(options) + apacheLogger = LogFactory.getLog(NAME) + # Configure log4j to not spew so much + unless (options[:debug]) + logger = org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase") + logger.setLevel(org.apache.log4j.Level::WARN) + logger = org.apache.log4j.Logger.getLogger("org.apache.zookeeper") + logger.setLevel(org.apache.log4j.Level::WARN) + end + return apacheLogger +end + +# Create a logger and save it to ruby global +$LOG = configureLogging(options) +case ARGV[0] + when 'add' + if ARGV.length < 2 + puts optparse + exit 1 + end + addServers(options, hostOrServers) + when 'remove' + if ARGV.length < 2 + puts optparse + exit 1 + end + removeServers(options, hostOrServers) + when 'list' + listServers(options) + else + puts optparse + exit 3 +end diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/files/draining_servers2.rb b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/files/draining_servers2.rb new file mode 100644 index 0000000000..7e1a4e95c2 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/files/draining_servers2.rb @@ -0,0 +1,159 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Add or remove servers from draining mode via zookeeper +# Deprecated in 2.0, and will be removed in 3.0. Use Admin decommission +# API instead. + +require 'optparse' +include Java + +java_import org.apache.hadoop.hbase.HBaseConfiguration +java_import org.apache.hadoop.hbase.client.ConnectionFactory +java_import org.apache.hadoop.hbase.client.HBaseAdmin +java_import org.apache.hadoop.hbase.zookeeper.ZKUtil +java_import org.apache.hadoop.hbase.zookeeper.ZNodePaths +java_import org.slf4j.LoggerFactory + +# Name of this script +NAME = 'draining_servers'.freeze + +# Do command-line parsing +options = {} +optparse = OptionParser.new do |opts| + opts.banner = "Usage: ./hbase org.jruby.Main #{NAME}.rb [options] add|remove|list <hostname>|<host:port>|<servername> ..." + opts.separator 'Add remove or list servers in draining mode. Can accept either hostname to drain all region servers' \ + 'in that host, a host:port pair or a host,port,startCode triplet. More than one server can be given separated by space' + opts.on('-h', '--help', 'Display usage information') do + puts opts + exit + end +end +optparse.parse! + +# Return array of servernames where servername is hostname+port+startcode +# comma-delimited +def getServers(admin) + serverInfos = admin.getClusterStatus.getServers + servers = [] + serverInfos.each do |server| + servers << server.getServerName + end + servers +end + +# rubocop:disable Metrics/AbcSize +def getServerNames(hostOrServers, config) + ret = [] + connection = ConnectionFactory.createConnection(config) + admin = nil + + hostOrServers.each do |host_or_server| + # check whether it is already serverName. No need to connect to cluster + parts = host_or_server.split(',') + if parts.size == 3 + ret << host_or_server + else + admin ||= connection.getAdmin + servers = getServers(admin) + + host_or_server = host_or_server.tr(':', ',') + servers.each do |server| + ret << server if server.start_with?(host_or_server) + end + end + end + + admin.close if admin + connection.close + ret +end + +def addServers(_options, hostOrServers) + config = HBaseConfiguration.create + servers = getServerNames(hostOrServers, config) + + zkw = org.apache.hadoop.hbase.zookeeper.ZKWatcher.new(config, 'draining_servers', nil) + + begin + parentZnode = zkw.getZNodePaths.drainingZNode + servers.each do |server| + node = ZNodePaths.joinZNode(parentZnode, server) + ZKUtil.createAndFailSilent(zkw, node) + end + ensure + zkw.close + end +end + +def removeServers(_options, hostOrServers) + config = HBaseConfiguration.create + servers = getServerNames(hostOrServers, config) + + zkw = org.apache.hadoop.hbase.zookeeper.ZKWatcher.new(config, 'draining_servers', nil) + + begin + parentZnode = zkw.getZNodePaths.drainingZNode + servers.each do |server| + node = ZNodePaths.joinZNode(parentZnode, server) + ZKUtil.deleteNodeFailSilent(zkw, node) + end + ensure + zkw.close + end +end +# rubocop:enable Metrics/AbcSize + +# list servers in draining mode +def listServers(_options) + config = HBaseConfiguration.create + + zkw = org.apache.hadoop.hbase.zookeeper.ZKWatcher.new(config, 'draining_servers', nil) + + begin + parentZnode = zkw.getZNodePaths.drainingZNode + servers = ZKUtil.listChildrenNoWatch(zkw, parentZnode) + servers.each { |server| puts server } + ensure + zkw.close + end +end + +hostOrServers = ARGV[1..ARGV.size] + +# Create a logger and save it to ruby global +$LOG = LoggerFactory.getLogger(NAME) +case ARGV[0] +when 'add' + if ARGV.length < 2 + puts optparse + exit 1 + end + addServers(options, hostOrServers) +when 'remove' + if ARGV.length < 2 + puts optparse + exit 1 + end + removeServers(options, hostOrServers) +when 'list' + listServers(options) +else + puts optparse + exit 3 +end diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/scripts/hbase_decommission.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/scripts/hbase_decommission.py index 212eb768ac..7d69ba7f79 100755 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/scripts/hbase_decommission.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/scripts/hbase_decommission.py @@ -22,12 +22,15 @@ from resource_management.core.source import StaticFile from resource_management.libraries.functions.format import format from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst +from resource_management.core.logger import Logger + @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) def hbase_decommission(env): import params env.set_params(params) + File(params.region_drainer, content=StaticFile("draining_servers.rb"), owner=params.hbase_user, mode="f") hosts = params.hbase_excluded_hosts.split(",") for host in hosts: @@ -37,7 +40,7 @@ def hbase_decommission(env): Execute(regiondrainer_cmd, user=params.hbase_user, logoutput=True) else: regiondrainer_cmd = format("cmd /c {hbase_executable} org.jruby.Main {region_drainer} add {host}") - regionmover_cmd = format("cmd /c {hbase_executable} org.jruby.Main {region_mover} -o unload -r {host}") + regionmover_cmd = format("cmd /c {hbase_executable} org.jruby.Main {region_mover} -m 24 -o unload -r {host}") Execute(regiondrainer_cmd, user=params.hbase_user, logoutput=True) Execute(regionmover_cmd, user=params.hbase_user, logoutput=True) @@ -49,6 +52,15 @@ def hbase_decommission(env): env.set_params(params) kinit_cmd = params.kinit_cmd_master + File(params.region_drainer, + content=StaticFile("draining_servers.rb"), + mode=0755 + ) + File(params.region_drainer2, + content=StaticFile("draining_servers2.rb"), + mode=0755 + ) + if params.hbase_excluded_hosts and params.hbase_excluded_hosts.split(","): hosts = params.hbase_excluded_hosts.split(",") elif params.hbase_included_hosts and params.hbase_included_hosts.split(","): @@ -58,11 +70,11 @@ def hbase_decommission(env): for host in hosts: if host: regiondrainer_cmd = format( - "{kinit_cmd} HBASE_SERVER_JAAS_OPTS=\"{master_security_config}\" {hbase_cmd} --config {hbase_conf_dir} {hbase_decommission_auth_config} org.jruby.Main {region_drainer} remove {host}") + "{kinit_cmd} HBASE_SERVER_JAAS_OPTS=\"{master_security_config}\" {hbase_cmd} --config {hbase_conf_dir} {hbase_decommission_auth_config} org.jruby.Main {region_drainer2} remove {host}") Execute(regiondrainer_cmd, user=params.hbase_user, logoutput=True - ) + ) pass pass @@ -70,19 +82,19 @@ def hbase_decommission(env): for host in hosts: if host: regiondrainer_cmd = format( - "{kinit_cmd} HBASE_SERVER_JAAS_OPTS=\"{master_security_config}\" {hbase_cmd} --config {hbase_conf_dir} {hbase_decommission_auth_config} org.jruby.Main {region_drainer} add {host}") + "{kinit_cmd} HBASE_SERVER_JAAS_OPTS=\"{master_security_config}\" {hbase_cmd} --config {hbase_conf_dir} {hbase_decommission_auth_config} org.jruby.Main {region_drainer2} add {host}") regionmover_cmd = format( - "{kinit_cmd} HBASE_SERVER_JAAS_OPTS=\"{master_security_config}\" {hbase_cmd} --config {hbase_conf_dir} {hbase_decommission_auth_config} org.jruby.Main {region_mover} -o unload -r {host}") + "{kinit_cmd} HBASE_SERVER_JAAS_OPTS=\"{master_security_config}\" {hbase_cmd} --config {hbase_conf_dir} {hbase_decommission_auth_config} org.jruby.Main {region_mover} -m 24 -o unload -r {host}") Execute(regiondrainer_cmd, user=params.hbase_user, logoutput=True - ) + ) Execute(regionmover_cmd, user=params.hbase_user, logoutput=True - ) + ) pass pass pass diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/scripts/hbase_regionserver.py index e0c766e4fc..90568f2dfc 100755 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/scripts/hbase_regionserver.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/scripts/hbase_regionserver.py @@ -21,6 +21,7 @@ limitations under the License. import sys from resource_management.core import shell +from resource_management.core.resources.system import Execute from resource_management.libraries.script.script import Script from resource_management.libraries.functions.format import format from resource_management.libraries.functions.check_process_status import check_process_status @@ -29,12 +30,15 @@ from resource_management.libraries.functions.security_commons import build_expec FILE_TYPE_XML from ambari_commons import OSCheck, OSConst +from ambari_commons.constants import UPGRADE_TYPE_ROLLING from ambari_commons.os_family_impl import OsFamilyImpl from hbase import hbase from hbase_service import hbase_service import upgrade from setup_ranger_hbase import setup_ranger_hbase +from hbase_decommission import hbase_decommission +from resource_management.core.logger import Logger class HbaseRegionServer(Script): @@ -51,6 +55,44 @@ class HbaseRegionServer(Script): def decommission(self, env): print "Decommission not yet implemented!" + def graceful_stop(self, env, upgrade_type=None): + import params + + # Mark Draining ZNode + params.hbase_drain_only = False + params.hbase_excluded_hosts = params.hostname + env.set_params(params) + hbase_decommission(env) + + # Stop RegionServer + hbase_service('regionserver', action='stop') + + # Remove from Draining ZNode to make host useable on restarting regionserver + params.hbase_drain_only = True + env.set_params(params) + hbase_decommission(env) + + def graceful_start(self, env, upgrade_type=None): + import params + env.set_params(params) + + #Start RegionServer + hbase_service('regionserver', action='start') + + # Load Regions back + kinit_cmd = params.kinit_cmd_master + host = params.hostname + + try: + regionmover_cmd = format( + "{kinit_cmd} HBASE_SERVER_JAAS_OPTS=\"{master_security_config}\" {hbase_cmd} --config {hbase_conf_dir} {hbase_decommission_auth_config} org.jruby.Main {region_mover} -m 24 -o load -r {host}") + Execute(regionmover_cmd, + user=params.hbase_user, + logoutput=True + ) + except Exception as ex: + Logger.info("HBase 2: region_mover failed while loading regions back to source RS." + str(ex)) + @OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) @@ -90,15 +132,23 @@ class HbaseRegionServerDefault(HbaseRegionServer): self.configure(env) # for security setup_ranger_hbase(upgrade_type=upgrade_type, service_name="hbase-regionserver") - hbase_service('regionserver', action='start') + if upgrade_type == UPGRADE_TYPE_ROLLING and len(params.rs_hosts) > 5: + self.graceful_start(env) + else: + hbase_service('regionserver', + action='start' + ) def stop(self, env, upgrade_type=None): import params env.set_params(params) - hbase_service( 'regionserver', - action = 'stop' - ) + if upgrade_type == UPGRADE_TYPE_ROLLING and len(params.rs_hosts) > 5: + self.graceful_stop(env) + else: + hbase_service('regionserver', + action='stop' + ) def status(self, env): import status_params @@ -158,7 +208,7 @@ class HbaseRegionServerDefault(HbaseRegionServer): def get_log_folder(self): import params return params.log_dir - + def get_user(self): import params return params.hbase_user diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/scripts/params_linux.py index a82a83c80f..0aab19f057 100755 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HBASE/package/scripts/params_linux.py @@ -72,6 +72,7 @@ hadoop_conf_dir = conf_select.get_hadoop_conf_dir() daemon_script = "/usr/lib/hbase/bin/hbase-daemon.sh" region_mover = "/usr/lib/hbase/bin/region_mover.rb" region_drainer = "/usr/lib/hbase/bin/draining_servers.rb" +region_drainer2 = "/usr/lib/hbase/bin/draining_servers2.rb" hbase_cmd = "/usr/lib/hbase/bin/hbase" hbase_max_direct_memory_size = None @@ -80,6 +81,7 @@ if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, daemon_script = format('{stack_root}/current/hbase-client/bin/hbase-daemon.sh') region_mover = format('{stack_root}/current/hbase-client/bin/region_mover.rb') region_drainer = format('{stack_root}/current/hbase-client/bin/draining_servers.rb') + region_drainer2 = format('{stack_root}/current/hbase-client/bin/draining_servers2.rb') hbase_cmd = format('{stack_root}/current/hbase-client/bin/hbase') hbase_max_direct_memory_size = default('configurations/hbase-env/hbase_max_direct_memory_size', None) @@ -87,6 +89,7 @@ if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, daemon_script=format("{stack_root}/current/{component_directory}/bin/hbase-daemon.sh") region_mover = format("{stack_root}/current/{component_directory}/bin/region_mover.rb") region_drainer = format("{stack_root}/current/{component_directory}/bin/draining_servers.rb") + region_drainer2 = format("{stack_root}/current/{component_directory}/bin/draining_servers2.rb") hbase_cmd = format("{stack_root}/current/{component_directory}/bin/hbase") --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@ambari.apache.org For additional commands, e-mail: commits-h...@ambari.apache.org