Repository: hbase Updated Branches: refs/heads/master 9aeafe30b -> 211786e00
HBASE-13580 region_mover.rb broken with TypeError: no public constructors for Java::OrgApacheHadoopHbaseClient::HTable (Samir Ahmic) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/211786e0 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/211786e0 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/211786e0 Branch: refs/heads/master Commit: 211786e00ce194dfa3efb4b35eefd4aa3236984c Parents: 9aeafe3 Author: tedyu <[email protected]> Authored: Mon May 11 09:01:56 2015 -0700 Committer: tedyu <[email protected]> Committed: Mon May 11 09:01:56 2015 -0700 ---------------------------------------------------------------------- bin/region_mover.rb | 69 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/211786e0/bin/region_mover.rb ---------------------------------------------------------------------- diff --git a/bin/region_mover.rb b/bin/region_mover.rb index 3259564..a6b11db 100644 --- a/bin/region_mover.rb +++ b/bin/region_mover.rb @@ -26,9 +26,9 @@ include Java import org.apache.hadoop.hbase.HConstants import org.apache.hadoop.hbase.HBaseConfiguration import org.apache.hadoop.hbase.client.HBaseAdmin +import org.apache.hadoop.hbase.TableName import org.apache.hadoop.hbase.client.Get import org.apache.hadoop.hbase.client.Scan -import org.apache.hadoop.hbase.client.HTable import org.apache.hadoop.hbase.client.ConnectionFactory import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.apache.hadoop.hbase.filter.InclusiveStopFilter; @@ -44,6 +44,18 @@ import org.apache.hadoop.hbase.HRegionInfo # Name of this script NAME = "region_mover" +# Get configuration instance +def getConfiguration() + config = HBaseConfiguration.create() + # No prefetching on hbase:meta This is for versions pre 0.99. Newer versions do not prefetch. + config.setInt("hbase.client.prefetch.limit", 1) + # Make a config that retries at short intervals many times + config.setInt("hbase.client.pause", 500) + config.setInt("hbase.client.retries.number", 100) + return config +end + +$connection=ConnectionFactory.createConnection(getConfiguration()) # Returns true if passed region is still on 'original' when we look at hbase:meta. def isSameServer(admin, r, original) @@ -62,7 +74,7 @@ end # Get servername that is up in hbase:meta; this is hostname + port + startcode comma-delimited. # Can return nil def getServerNameForRegion(admin, r) - return nil unless admin.isTableEnabled(r.getTableName) + return nil unless admin.isTableEnabled(r.getTable()) if r.isMetaRegion() # Hack zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(admin.getConfiguration(), "region_mover", nil) @@ -78,7 +90,7 @@ def getServerNameForRegion(admin, r) zkw.close() end end - table = HTable.new(admin.getConfiguration(), HConstants::META_TABLE_NAME) + table = $connection.getTable(TableName.valueOf('hbase:meta')) begin g = Get.new(r.getRegionName()) g.addColumn(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER) @@ -102,12 +114,15 @@ def isSuccessfulScan(admin, r) scan.setCaching(1) scan.setFilter(FilterList.new(FirstKeyOnlyFilter.new(),InclusiveStopFilter.new(r.getStartKey()))) begin - table = HTable.new(admin.getConfiguration(), r.getTableName()) + table = $connection.getTable(r.getTable()) scanner = table.getScanner(scan) begin results = scanner.next() # We might scan into next region, this might be an empty table. # But if no exception, presume scanning is working. + rescue java.lang.NullPointerException => e + $LOG.warn("Unable to scan region=" + r.getRegionNameAsString() + + " start key is empty. " + e.message) ensure scanner.close() end @@ -169,7 +184,7 @@ end # Return array of servernames where servername is hostname+port+startcode # comma-delimited def getServers(admin) - serverInfos = admin.getClusterStatus().getServerInfo() + serverInfos = admin.getClusterStatus().getServers() servers = [] for server in serverInfos servers << server.getServerName() @@ -177,6 +192,12 @@ def getServers(admin) return servers end +# Get master hostname +def getMaster(admin) + return admin.getClusterStatus().getMaster().getHostname(), + admin.getClusterStatus().getMaster().getPort() +end + # Remove the servername whose hostname portion matches from the passed # array of servers. Returns as side-effect the servername removed. def stripServer(servers, hostname, port) @@ -193,13 +214,29 @@ def stripServer(servers, hostname, port) return servername end +# Removes master from servers list +def stripMaster(servers, masterHostname, masterPort) + for server in servers + hostFromServerName, portFromServerName = getHostPortFromServerName(server) + if hostFromServerName == masterHostname and portFromServerName == masterPort.to_s + servers.delete(server) + end + end + return servers +end + + # Returns a new serverlist that excludes the servername whose hostname portion # matches from the passed array of servers. def stripExcludes(servers, excludefile) excludes = readExcludes(excludefile) - servers = servers.find_all{|server| - !excludes.contains(getHostPortFromServerName(server).join(":")) - } + updatedservers = [] + servers.each_with_index do |val,indx| + if !excludes.to_a.include? val.split(",")[0].to_s + updatedservers << val + end + end + servers = updatedservers # return updated servers list return servers end @@ -230,17 +267,6 @@ def configureLogging(options) return apacheLogger end -# Get configuration instance -def getConfiguration() - config = HBaseConfiguration.create() - # No prefetching on hbase:meta This is for versions pre 0.99. Newer versions do not prefetch. - config.setInt("hbase.client.prefetch.limit", 1) - # Make a config that retries at short intervals many times - config.setInt("hbase.client.pause", 500) - config.setInt("hbase.client.retries.number", 100) - return config -end - # Now get list of regions on targetServer def getRegions(config, servername) connection = ConnectionFactory::createConnection(config); @@ -296,12 +322,13 @@ def unloadRegions(options, hostname, port) # Get an admin instance admin = HBaseAdmin.new(config) servers = getServers(admin) + master, masterPort = getMaster(admin) # Remove the server we are unloading from from list of servers. # Side-effect is the servername that matches this hostname servername = stripServer(servers, hostname, port) - # Remove the servers in our exclude list from list of servers. servers = stripExcludes(servers, options[:excludesFile]) + servers = stripMaster(servers, master, masterPort) puts "Valid region move targets: ", servers if servers.length == 0 puts "No regions were moved - there was no server available" @@ -486,3 +513,5 @@ case ARGV[0] puts optparse exit 3 end + +$connection.close()
