HBASE-18434 Address some alerts raised by lgtm.com Signed-off-by: Ramkrishna <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/421029d0 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/421029d0 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/421029d0 Branch: refs/heads/HBASE-18426 Commit: 421029d0c73c3a20fef4852692d3779149599f2c Parents: 2fd8e82 Author: Malcolm Taylor <[email protected]> Authored: Sat Jul 22 10:40:27 2017 +0100 Committer: Ramkrishna <[email protected]> Committed: Wed Jul 26 10:00:23 2017 +0530 ---------------------------------------------------------------------- .../hadoop/hbase/ipc/NettyRpcDuplexHandler.java | 31 +++--- .../apache/hadoop/hbase/zookeeper/ZKUtil.java | 26 ++--- .../hbase/io/hfile/bucket/BucketAllocator.java | 2 +- .../hadoop/hbase/master/MasterDumpServlet.java | 101 ++++++++++--------- .../hadoop/hbase/master/MasterRpcServices.java | 2 +- .../hbase/regionserver/RSDumpServlet.java | 85 ++++++++-------- .../compactions/FIFOCompactionPolicy.java | 10 +- .../apache/hadoop/hbase/util/RegionMover.java | 4 +- .../apache/hadoop/hbase/wal/WALSplitter.java | 3 +- .../hbase/io/hfile/bucket/TestBucketCache.java | 8 ++ 10 files changed, 141 insertions(+), 131 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/421029d0/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcDuplexHandler.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcDuplexHandler.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcDuplexHandler.java index 08533b4..78964d6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcDuplexHandler.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcDuplexHandler.java @@ -91,21 +91,22 @@ class NettyRpcDuplexHandler extends ChannelDuplexHandler { : sizeWithoutCellBlock; ByteBuf buf = ctx.alloc().buffer(sizeWithoutCellBlock + 4); buf.writeInt(totalSize); - ByteBufOutputStream bbos = new ByteBufOutputStream(buf); - requestHeader.writeDelimitedTo(bbos); - if (call.param != null) { - call.param.writeDelimitedTo(bbos); - } - if (cellBlock != null) { - ChannelPromise withoutCellBlockPromise = ctx.newPromise(); - ctx.write(buf, withoutCellBlockPromise); - ChannelPromise cellBlockPromise = ctx.newPromise(); - ctx.write(cellBlock, cellBlockPromise); - PromiseCombiner combiner = new PromiseCombiner(); - combiner.addAll(withoutCellBlockPromise, cellBlockPromise); - combiner.finish(promise); - } else { - ctx.write(buf, promise); + try (ByteBufOutputStream bbos = new ByteBufOutputStream(buf)) { + requestHeader.writeDelimitedTo(bbos); + if (call.param != null) { + call.param.writeDelimitedTo(bbos); + } + if (cellBlock != null) { + ChannelPromise withoutCellBlockPromise = ctx.newPromise(); + ctx.write(buf, withoutCellBlockPromise); + ChannelPromise cellBlockPromise = ctx.newPromise(); + ctx.write(cellBlock, cellBlockPromise); + PromiseCombiner combiner = new PromiseCombiner(); + combiner.addAll(withoutCellBlockPromise, cellBlockPromise); + combiner.finish(promise); + } else { + ctx.write(buf, promise); + } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/421029d0/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java index a31cab9..dafe421 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java @@ -1930,21 +1930,21 @@ public class ZKUtil { socket.connect(sockAddr, timeout); socket.setSoTimeout(timeout); - PrintWriter out = new PrintWriter(socket.getOutputStream(), true); - BufferedReader in = new BufferedReader(new InputStreamReader( - socket.getInputStream())); - out.println("stat"); - out.flush(); - ArrayList<String> res = new ArrayList<>(); - while (true) { - String line = in.readLine(); - if (line != null) { - res.add(line); - } else { - break; + try (PrintWriter out = new PrintWriter(socket.getOutputStream(), true); + BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream()))) { + out.println("stat"); + out.flush(); + ArrayList<String> res = new ArrayList<>(); + while (true) { + String line = in.readLine(); + if (line != null) { + res.add(line); + } else { + break; + } } + return res.toArray(new String[res.size()]); } - return res.toArray(new String[res.size()]); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/421029d0/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java index 5c8fa1b..d1b8bb7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java @@ -315,7 +315,7 @@ public final class BucketAllocator { this.bucketSizes = bucketSizes == null ? DEFAULT_BUCKET_SIZES : bucketSizes; Arrays.sort(this.bucketSizes); this.bigItemSize = Ints.max(this.bucketSizes); - this.bucketCapacity = FEWEST_ITEMS_IN_BUCKET * bigItemSize; + this.bucketCapacity = FEWEST_ITEMS_IN_BUCKET * (long) bigItemSize; buckets = new Bucket[(int) (availableSpace / bucketCapacity)]; if (buckets.length < this.bucketSizes.length) throw new BucketAllocatorException("Bucket allocator size too small (" + buckets.length + http://git-wip-us.apache.org/repos/asf/hbase/blob/421029d0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java index a48444c..86dc014 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java @@ -54,60 +54,61 @@ public class MasterDumpServlet extends StateDumpServlet { response.setContentType("text/plain"); OutputStream os = response.getOutputStream(); - PrintWriter out = new PrintWriter(os); + try (PrintWriter out = new PrintWriter(os)) { - out.println("Master status for " + master.getServerName() + out.println("Master status for " + master.getServerName() + " as of " + new Date()); - out.println("\n\nVersion Info:"); - out.println(LINE); - dumpVersionInfo(out); - - out.println("\n\nTasks:"); - out.println(LINE); - TaskMonitor.get().dumpAsText(out); - - out.println("\n\nServers:"); - out.println(LINE); - dumpServers(master, out); - - out.println("\n\nRegions-in-transition:"); - out.println(LINE); - dumpRIT(master, out); - - out.println("\n\nExecutors:"); - out.println(LINE); - dumpExecutors(master.getExecutorService(), out); - - out.println("\n\nStacks:"); - out.println(LINE); - out.flush(); - PrintStream ps = new PrintStream(response.getOutputStream(), false, "UTF-8"); - Threads.printThreadInfo(ps, ""); - ps.flush(); - - out.println("\n\nMaster configuration:"); - out.println(LINE); - Configuration conf = master.getConfiguration(); - out.flush(); - conf.writeXml(os); - os.flush(); - - out.println("\n\nRecent regionserver aborts:"); - out.println(LINE); - master.getRegionServerFatalLogBuffer().dumpTo(out); - - out.println("\n\nLogs"); - out.println(LINE); - long tailKb = getTailKbParam(request); - LogMonitoring.dumpTailOfLogs(out, tailKb); - - out.println("\n\nRS Queue:"); - out.println(LINE); - if(isShowQueueDump(conf)) { - RSDumpServlet.dumpQueue(master, out); + out.println("\n\nVersion Info:"); + out.println(LINE); + dumpVersionInfo(out); + + out.println("\n\nTasks:"); + out.println(LINE); + TaskMonitor.get().dumpAsText(out); + + out.println("\n\nServers:"); + out.println(LINE); + dumpServers(master, out); + + out.println("\n\nRegions-in-transition:"); + out.println(LINE); + dumpRIT(master, out); + + out.println("\n\nExecutors:"); + out.println(LINE); + dumpExecutors(master.getExecutorService(), out); + + out.println("\n\nStacks:"); + out.println(LINE); + out.flush(); + PrintStream ps = new PrintStream(response.getOutputStream(), false, "UTF-8"); + Threads.printThreadInfo(ps, ""); + ps.flush(); + + out.println("\n\nMaster configuration:"); + out.println(LINE); + Configuration conf = master.getConfiguration(); + out.flush(); + conf.writeXml(os); + os.flush(); + + out.println("\n\nRecent regionserver aborts:"); + out.println(LINE); + master.getRegionServerFatalLogBuffer().dumpTo(out); + + out.println("\n\nLogs"); + out.println(LINE); + long tailKb = getTailKbParam(request); + LogMonitoring.dumpTailOfLogs(out, tailKb); + + out.println("\n\nRS Queue:"); + out.println(LINE); + if (isShowQueueDump(conf)) { + RSDumpServlet.dumpQueue(master, out); + } + out.flush(); } - out.flush(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/421029d0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 8b9266c..aa64caa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -1291,7 +1291,7 @@ public class MasterRpcServices extends RSRpcServices throws ServiceException { try { master.checkInitialized(); - Boolean result = master.getHFileCleaner().runCleaner() && master.getLogCleaner().runCleaner(); + boolean result = master.getHFileCleaner().runCleaner() && master.getLogCleaner().runCleaner(); return ResponseConverter.buildRunCleanerChoreResponse(result); } catch (IOException ioe) { throw new ServiceException(ioe); http://git-wip-us.apache.org/repos/asf/hbase/blob/421029d0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java index 9499a79..997ffcc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java @@ -55,52 +55,53 @@ public class RSDumpServlet extends StateDumpServlet { } OutputStream os = response.getOutputStream(); - PrintWriter out = new PrintWriter(os); + try (PrintWriter out = new PrintWriter(os)) { - out.println("RegionServer status for " + hrs.getServerName() + out.println("RegionServer status for " + hrs.getServerName() + " as of " + new Date()); - out.println("\n\nVersion Info:"); - out.println(LINE); - dumpVersionInfo(out); - - out.println("\n\nTasks:"); - out.println(LINE); - TaskMonitor.get().dumpAsText(out); - - out.println("\n\nRowLocks:"); - out.println(LINE); - dumpRowLock(hrs, out); - - out.println("\n\nExecutors:"); - out.println(LINE); - dumpExecutors(hrs.getExecutorService(), out); - - out.println("\n\nStacks:"); - out.println(LINE); - PrintStream ps = new PrintStream(response.getOutputStream(), false, "UTF-8"); - Threads.printThreadInfo(ps, ""); - ps.flush(); - - out.println("\n\nRS Configuration:"); - out.println(LINE); - Configuration conf = hrs.getConfiguration(); - out.flush(); - conf.writeXml(os); - os.flush(); - - out.println("\n\nLogs"); - out.println(LINE); - long tailKb = getTailKbParam(request); - LogMonitoring.dumpTailOfLogs(out, tailKb); - - out.println("\n\nRS Queue:"); - out.println(LINE); - if(isShowQueueDump(conf)) { - dumpQueue(hrs, out); - } + out.println("\n\nVersion Info:"); + out.println(LINE); + dumpVersionInfo(out); + + out.println("\n\nTasks:"); + out.println(LINE); + TaskMonitor.get().dumpAsText(out); + + out.println("\n\nRowLocks:"); + out.println(LINE); + dumpRowLock(hrs, out); + + out.println("\n\nExecutors:"); + out.println(LINE); + dumpExecutors(hrs.getExecutorService(), out); + + out.println("\n\nStacks:"); + out.println(LINE); + PrintStream ps = new PrintStream(response.getOutputStream(), false, "UTF-8"); + Threads.printThreadInfo(ps, ""); + ps.flush(); + + out.println("\n\nRS Configuration:"); + out.println(LINE); + Configuration conf = hrs.getConfiguration(); + out.flush(); + conf.writeXml(os); + os.flush(); + + out.println("\n\nLogs"); + out.println(LINE); + long tailKb = getTailKbParam(request); + LogMonitoring.dumpTailOfLogs(out, tailKb); + + out.println("\n\nRS Queue:"); + out.println(LINE); + if (isShowQueueDump(conf)) { + dumpQueue(hrs, out); + } - out.flush(); + out.flush(); + } } public static void dumpRowLock(HRegionServer hrs, PrintWriter out) { http://git-wip-us.apache.org/repos/asf/hbase/blob/421029d0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/FIFOCompactionPolicy.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/FIFOCompactionPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/FIFOCompactionPolicy.java index 97b8387..64ca771 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/FIFOCompactionPolicy.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/FIFOCompactionPolicy.java @@ -101,10 +101,9 @@ public class FIFOCompactionPolicy extends ExploringCompactionPolicy { long currentTime = EnvironmentEdgeManager.currentTime(); for(StoreFile sf: files){ // Check MIN_VERSIONS is in HStore removeUnneededFiles - Long maxTs = sf.getReader().getMaxTimestamp(); + long maxTs = sf.getReader().getMaxTimestamp(); long maxTtl = storeConfigInfo.getStoreFileTtl(); - if(maxTs == null - || maxTtl == Long.MAX_VALUE + if (maxTtl == Long.MAX_VALUE || (currentTime - maxTtl < maxTs)){ continue; } else{ @@ -120,10 +119,9 @@ public class FIFOCompactionPolicy extends ExploringCompactionPolicy { Collection<StoreFile> expiredStores = new ArrayList<>(); for(StoreFile sf: files){ // Check MIN_VERSIONS is in HStore removeUnneededFiles - Long maxTs = sf.getReader().getMaxTimestamp(); + long maxTs = sf.getReader().getMaxTimestamp(); long maxTtl = storeConfigInfo.getStoreFileTtl(); - if(maxTs == null - || maxTtl == Long.MAX_VALUE + if (maxTtl == Long.MAX_VALUE || (currentTime - maxTtl < maxTs)){ continue; } else if(filesCompacting == null || filesCompacting.contains(sf) == false){ http://git-wip-us.apache.org/repos/asf/hbase/blob/421029d0/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java index ce018da..a9a53b8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java @@ -427,7 +427,7 @@ public class RegionMover extends AbstractHBaseTool { moveRegionsPool.shutdown(); long timeoutInSeconds = regionsToMove.size() - * admin.getConfiguration().getInt(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX); + * admin.getConfiguration().getLong(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX); try { if (!moveRegionsPool.awaitTermination(timeoutInSeconds, TimeUnit.SECONDS)) { moveRegionsPool.shutdownNow(); @@ -501,7 +501,7 @@ public class RegionMover extends AbstractHBaseTool { moveRegionsPool.shutdown(); long timeoutInSeconds = regionsToMove.size() - * admin.getConfiguration().getInt(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX); + * admin.getConfiguration().getLong(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX); try { if (!moveRegionsPool.awaitTermination(timeoutInSeconds, TimeUnit.SECONDS)) { moveRegionsPool.shutdownNow(); http://git-wip-us.apache.org/repos/asf/hbase/blob/421029d0/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java index da828de..df4c1bd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java @@ -711,7 +711,8 @@ public class WALSplitter { for (FileStatus status : files) { String fileName = status.getPath().getName(); try { - Long tmpSeqId = Long.parseLong(fileName.substring(0, fileName.length() + long tmpSeqId = + Long.parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)); maxSeqId = Math.max(tmpSeqId, maxSeqId); } catch (NumberFormatException ex) { http://git-wip-us.apache.org/repos/asf/hbase/blob/421029d0/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java index 0f16bfa..b0db13a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java @@ -268,4 +268,12 @@ public class TestBucketCache { TEST_UTIL.cleanupTestDir(); } + + @Test + public void testBucketAllocatorLargeBuckets() throws BucketAllocatorException { + long availableSpace = 20 * 1024L * 1024 * 1024; + int[] bucketSizes = new int[] { 1024, 1024 * 1024, 1024 * 1024 * 1024 }; + BucketAllocator allocator = new BucketAllocator(availableSpace, bucketSizes); + assertTrue(allocator.getBuckets().length > 0); + } }
