[hbase-operator-tools] branch master updated: Add an HBCK2 tool. Add command-line processing for the one call that is available in Hbck Service Interface currently. More to come.
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase-operator-tools.git The following commit(s) were added to refs/heads/master by this push: new 0cf0e0e Add an HBCK2 tool. Add command-line processing for the one call that is available in Hbck Service Interface currently. More to come. 0cf0e0e is described below commit 0cf0e0ecf2d4a33522e0e273f9310f11aa2eaee6 Author: Michael Stack AuthorDate: Thu Sep 13 22:21:20 2018 -0700 Add an HBCK2 tool. Add command-line processing for the one call that is available in Hbck Service Interface currently. More to come. --- hbase-hbck2/pom.xml| 129 +--- .../src/main/java/org/apache/hbase/HBCK2.java | 130 + hbase-hbck2/src/main/resources/log4j2.xml | 13 +++ .../java/org/apache/hbase/TestHBCK2.java} | 45 --- pom.xml| 76 ++-- 5 files changed, 294 insertions(+), 99 deletions(-) diff --git a/hbase-hbck2/pom.xml b/hbase-hbck2/pom.xml index 82e80c9..250b0bc 100644 --- a/hbase-hbck2/pom.xml +++ b/hbase-hbck2/pom.xml @@ -32,10 +32,7 @@ Apache HBase - HBCK2 HBCK for HBase 2+ - - -${project.basedir}/target/java @@ -53,84 +50,72 @@ maven-remote-resources-plugin - -maven-assembly-plugin - - true - - - maven-surefire-plugin - - - - - org.apache.maven.plugins maven-source-plugin - + + +org.apache.maven.plugins +maven-shade-plugin +3.2.0 + + +package + + shade + + + + + classworlds:classworlds + junit:junit + jmock:* + *:xml-apis + org.apache.maven:lib:tests + log4j:log4j:jar: + + + + + - - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - -org.apache.maven.plugins -maven-antrun-plugin -[${maven.antrun.version}] - - run - - - - - - - - -org.apache.maven.plugins -maven-dependency-plugin -[2.8,) - - build-classpath - - - - - - - - -org.apache.maven.plugins -maven-compiler-plugin -[3.2,) - - compile - - - - - - - - - - - - + + + +commons-cli +commons-cli +1.4 + + + junit + junit + 4.12 + + + + org.apache.logging.log4j + log4j-api + 2.11.1 + + +org.apache.logging.log4j +log4j-core +2.11.1 + + + + + + org.apache.hbase + hbase-shaded-client + 2.1.1-SNAPSHOT + + diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java new file mode 100644 index 000..d88a2de --- /dev/null +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package
[08/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/testdevapidocs/index-all.html -- diff --git a/testdevapidocs/index-all.html b/testdevapidocs/index-all.html index 491160c..6e0259d 100644 --- a/testdevapidocs/index-all.html +++ b/testdevapidocs/index-all.html @@ -15286,6 +15286,10 @@ GaussianRandomReadBenchmark(Configuration, FileSystem, Path, int) - Constructor for class org.apache.hadoop.hbase.HFilePerformanceEvaluation.GaussianRandomReadBenchmark +GCCountResourceAnalyzer() - Constructor for class org.apache.hadoop.hbase.ResourceCheckerJUnitListener.GCCountResourceAnalyzer + +GCTimeSecondResourceAnalyzer() - Constructor for class org.apache.hadoop.hbase.ResourceCheckerJUnitListener.GCTimeSecondResourceAnalyzer + gen - Variable in class org.apache.hadoop.hbase.util.TestLoadTestKVGenerator GENERATE_VERIFY_GAP_OPT - Variable in class org.apache.hadoop.hbase.test.IntegrationTestReplication @@ -19208,8 +19212,14 @@ getVal(ResourceChecker.Phase) - Method in class org.apache.hadoop.hbase.ResourceCheckerJUnitListener.AvailableMemoryMBResourceAnalyzer +getVal(ResourceChecker.Phase) - Method in class org.apache.hadoop.hbase.ResourceCheckerJUnitListener.GCCountResourceAnalyzer + +getVal(ResourceChecker.Phase) - Method in class org.apache.hadoop.hbase.ResourceCheckerJUnitListener.GCTimeSecondResourceAnalyzer + getVal(ResourceChecker.Phase) - Method in class org.apache.hadoop.hbase.ResourceCheckerJUnitListener.MaxFileDescriptorResourceAnalyzer +getVal(ResourceChecker.Phase) - Method in class org.apache.hadoop.hbase.ResourceCheckerJUnitListener.MaxHeapMemoryMBResourceAnalyzer + getVal(ResourceChecker.Phase) - Method in class org.apache.hadoop.hbase.ResourceCheckerJUnitListener.OpenFileDescriptorResourceAnalyzer getVal(ResourceChecker.Phase) - Method in class org.apache.hadoop.hbase.ResourceCheckerJUnitListener.ProcessCountResourceAnalyzer @@ -19218,6 +19228,8 @@ getVal(ResourceChecker.Phase) - Method in class org.apache.hadoop.hbase.ResourceCheckerJUnitListener.ThreadResourceAnalyzer +getVal(ResourceChecker.Phase) - Method in class org.apache.hadoop.hbase.ResourceCheckerJUnitListener.UsedHeapMemoryMBResourceAnalyzer + getValue(int, int, int) - Static method in class org.apache.hadoop.hbase.io.encoding.TestChangingEncoding getValue() - Method in class org.apache.hadoop.hbase.metrics.impl.TestGauge.SimpleGauge @@ -25329,6 +25341,8 @@ maxFullTableSplits - Variable in class org.apache.hadoop.hbase.chaos.actions.SplitAllRegionOfTableAction +MaxHeapMemoryMBResourceAnalyzer() - Constructor for class org.apache.hadoop.hbase.ResourceCheckerJUnitListener.MaxHeapMemoryMBResourceAnalyzer + maxHeapSize - Variable in class org.apache.hadoop.hbase.regionserver.TestHeapMemoryManager maxKeyLen - Variable in class org.apache.hadoop.hbase.io.hfile.TestHFileSeek.MyOptions @@ -33435,8 +33449,14 @@ ResourceCheckerJUnitListener.AvailableMemoryMBResourceAnalyzer - Class in org.apache.hadoop.hbase +ResourceCheckerJUnitListener.GCCountResourceAnalyzer - Class in org.apache.hadoop.hbase + +ResourceCheckerJUnitListener.GCTimeSecondResourceAnalyzer - Class in org.apache.hadoop.hbase + ResourceCheckerJUnitListener.MaxFileDescriptorResourceAnalyzer - Class in org.apache.hadoop.hbase +ResourceCheckerJUnitListener.MaxHeapMemoryMBResourceAnalyzer - Class in org.apache.hadoop.hbase + ResourceCheckerJUnitListener.OpenFileDescriptorResourceAnalyzer - Class in org.apache.hadoop.hbase ResourceCheckerJUnitListener.ProcessCountResourceAnalyzer - Class in org.apache.hadoop.hbase @@ -33445,6 +33465,8 @@ ResourceCheckerJUnitListener.ThreadResourceAnalyzer - Class in org.apache.hadoop.hbase +ResourceCheckerJUnitListener.UsedHeapMemoryMBResourceAnalyzer - Class in org.apache.hadoop.hbase + resourcePathFilter - Variable in class org.apache.hadoop.hbase.ClassFinder.Not resourcePathFilter - Variable in class org.apache.hadoop.hbase.ClassFinder @@ -66677,6 +66699,8 @@ USE_TAG - Static variable in class org.apache.hadoop.hbase.regionserver.DataBlockEncodingTool +UsedHeapMemoryMBResourceAnalyzer() - Constructor for class org.apache.hadoop.hbase.ResourceCheckerJUnitListener.UsedHeapMemoryMBResourceAnalyzer + useDifferentDFSClient() - Method in class org.apache.hadoop.hbase.wal.TestWALSplit usedRegions - Variable in class org.apache.hadoop.hbase.client.TestAsyncProcess.MyConnectionImpl2 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/testdevapidocs/org/apache/hadoop/hbase/RESTApiClusterManager.html -- diff --git a/testdevapidocs/org/apache/hadoop/hbase/RESTApiClusterManager.html b/testdevapidocs/org/apache/hadoop/hbase/RESTApiClusterManager.html index 7ed3c4d..5b683ed 100644 --- a/testdevapidocs/org/apache/hadoop/hbase/RESTApiClusterManager.html +++
[02/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.UsedHeapMemoryMBResourceAnalyzer.html -- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.UsedHeapMemoryMBResourceAnalyzer.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.UsedHeapMemoryMBResourceAnalyzer.html new file mode 100644 index 000..47aac2c --- /dev/null +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.UsedHeapMemoryMBResourceAnalyzer.html @@ -0,0 +1,307 @@ +http://www.w3.org/TR/html4/loose.dtd;> + + +Source code + + + + +001/** +002 * Licensed to the Apache Software Foundation (ASF) under one +003 * or more contributor license agreements. See the NOTICE file +004 * distributed with this work for additional information +005 * regarding copyright ownership. The ASF licenses this file +006 * to you under the Apache License, Version 2.0 (the +007 * "License"); you may not use this file except in compliance +008 * with the License. You may obtain a copy of the License at +009 * +010 * http://www.apache.org/licenses/LICENSE-2.0 +011 * +012 * Unless required by applicable law or agreed to in writing, software +013 * distributed under the License is distributed on an "AS IS" BASIS, +014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +015 * See the License for the specific language governing permissions and +016 * limitations under the License. +017 */ +018 +019package org.apache.hadoop.hbase; +020 +021import java.lang.management.ManagementFactory; +022import java.lang.management.MemoryUsage; +023import java.util.ArrayList; +024import java.util.HashSet; +025import java.util.List; +026import java.util.Map; +027import java.util.Set; +028import java.util.concurrent.ConcurrentHashMap; +029import java.util.concurrent.TimeUnit; +030import org.apache.hadoop.hbase.ResourceChecker.Phase; +031import org.apache.hadoop.hbase.util.JVM; +032import org.junit.runner.notification.RunListener; +033 +034/** +035 * Listen to the test progress and check the usage of: +036 * ul +037 * lithreads/li +038 * liopen file descriptor/li +039 * limax open file descriptor/li +040 * /ul +041 * p +042 * When surefire forkMode=once/always/perthread, this code is executed on the forked process. +043 */ +044public class ResourceCheckerJUnitListener extends RunListener { +045 private MapString, ResourceChecker rcs = new ConcurrentHashMap(); +046 +047 static class ThreadResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { +048private static SetString initialThreadNames = new HashSet(); +049private static ListString stringsToLog = null; +050 +051@Override +052public int getVal(Phase phase) { +053 MapThread, StackTraceElement[] stackTraces = Thread.getAllStackTraces(); +054 if (phase == Phase.INITIAL) { +055stringsToLog = null; +056for (Thread t : stackTraces.keySet()) { +057 initialThreadNames.add(t.getName()); +058} +059 } else if (phase == Phase.END) { +060if (stackTraces.size() initialThreadNames.size()) { +061 stringsToLog = new ArrayList(); +062 for (Thread t : stackTraces.keySet()) { +063if (!initialThreadNames.contains(t.getName())) { +064 stringsToLog.add("\nPotentially hanging thread: " + t.getName() + "\n"); +065 StackTraceElement[] stackElements = stackTraces.get(t); +066 for (StackTraceElement ele : stackElements) { +067stringsToLog.add("\t" + ele + "\n"); +068 } +069} +070 } +071} +072 } +073 return stackTraces.size(); +074} +075 +076@Override +077public int getMax() { +078 return 500; +079} +080 +081@Override +082public ListString getStringsToLog() { +083 return stringsToLog; +084} +085 } +086 +087 +088 static class OpenFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { +089@Override +090public int getVal(Phase phase) { +091 if (!JVM.isUnix()) { +092return 0; +093 } +094 JVM jvm = new JVM(); +095 return (int) jvm.getOpenFileDescriptorCount(); +096} +097 +098@Override +099public int getMax() { +100 return 1024; +101} +102 } +103 +104 static class MaxFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { +105@Override +106public int getVal(Phase phase) { +107 if (!JVM.isUnix()) { +108return 0; +109 } +110 JVM jvm = new JVM(); +111 return (int) jvm.getMaxFileDescriptorCount(); +112} +113 } +114 +115 static class SystemLoadAverageResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { +116@Override +117public int
hbase-site git commit: INFRA-10751 Empty commit
Repository: hbase-site Updated Branches: refs/heads/asf-site 836bc028d -> 7bf9e1ef5 INFRA-10751 Empty commit Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/7bf9e1ef Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/7bf9e1ef Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/7bf9e1ef Branch: refs/heads/asf-site Commit: 7bf9e1ef510afe2021daafbf134dcb4482196185 Parents: 836bc02 Author: jenkins Authored: Thu Sep 13 14:53:48 2018 + Committer: jenkins Committed: Thu Sep 13 14:53:48 2018 + -- --
[06/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/testdevapidocs/org/apache/hadoop/hbase/package-tree.html -- diff --git a/testdevapidocs/org/apache/hadoop/hbase/package-tree.html b/testdevapidocs/org/apache/hadoop/hbase/package-tree.html index d9e587f..4dd2a36 100644 --- a/testdevapidocs/org/apache/hadoop/hbase/package-tree.html +++ b/testdevapidocs/org/apache/hadoop/hbase/package-tree.html @@ -395,11 +395,15 @@ org.apache.hadoop.hbase.ResourceChecker.ResourceAnalyzer org.apache.hadoop.hbase.ResourceCheckerJUnitListener.AvailableMemoryMBResourceAnalyzer +org.apache.hadoop.hbase.ResourceCheckerJUnitListener.GCCountResourceAnalyzer +org.apache.hadoop.hbase.ResourceCheckerJUnitListener.GCTimeSecondResourceAnalyzer org.apache.hadoop.hbase.ResourceCheckerJUnitListener.MaxFileDescriptorResourceAnalyzer +org.apache.hadoop.hbase.ResourceCheckerJUnitListener.MaxHeapMemoryMBResourceAnalyzer org.apache.hadoop.hbase.ResourceCheckerJUnitListener.OpenFileDescriptorResourceAnalyzer org.apache.hadoop.hbase.ResourceCheckerJUnitListener.ProcessCountResourceAnalyzer org.apache.hadoop.hbase.ResourceCheckerJUnitListener.SystemLoadAverageResourceAnalyzer org.apache.hadoop.hbase.ResourceCheckerJUnitListener.ThreadResourceAnalyzer +org.apache.hadoop.hbase.ResourceCheckerJUnitListener.UsedHeapMemoryMBResourceAnalyzer org.apache.hadoop.hbase.ipc.RpcScheduler @@ -578,14 +582,14 @@ java.lang.https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true; title="class or interface in java.lang">EnumE (implements java.lang.https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true; title="class or interface in java.lang">ComparableT, java.io.https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true; title="class or interface in java.io">Serializable) -org.apache.hadoop.hbase.ClusterManager.ServiceType org.apache.hadoop.hbase.ScanPerformanceEvaluation.ScanCounter -org.apache.hadoop.hbase.RESTApiClusterManager.Service -org.apache.hadoop.hbase.HBaseClusterManager.CommandProvider.Operation +org.apache.hadoop.hbase.IntegrationTestDDLMasterFailover.ACTION +org.apache.hadoop.hbase.ResourceChecker.Phase +org.apache.hadoop.hbase.ClusterManager.ServiceType org.apache.hadoop.hbase.RESTApiClusterManager.RoleCommand org.apache.hadoop.hbase.IntegrationTestRegionReplicaPerf.Stat -org.apache.hadoop.hbase.ResourceChecker.Phase -org.apache.hadoop.hbase.IntegrationTestDDLMasterFailover.ACTION +org.apache.hadoop.hbase.HBaseClusterManager.CommandProvider.Operation +org.apache.hadoop.hbase.RESTApiClusterManager.Service org.apache.hadoop.hbase.PerformanceEvaluation.Counter http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/testdevapidocs/org/apache/hadoop/hbase/procedure/package-tree.html -- diff --git a/testdevapidocs/org/apache/hadoop/hbase/procedure/package-tree.html b/testdevapidocs/org/apache/hadoop/hbase/procedure/package-tree.html index 28973e0..a1c9b1d 100644 --- a/testdevapidocs/org/apache/hadoop/hbase/procedure/package-tree.html +++ b/testdevapidocs/org/apache/hadoop/hbase/procedure/package-tree.html @@ -81,14 +81,14 @@ java.lang.https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true; title="class or interface in java.lang">Object -org.apache.hadoop.hbase.procedure2.ProcedureTEnvironment (implements java.lang.https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true; title="class or interface in java.lang">ComparableT) +org.apache.hadoop.hbase.procedure.Procedure (implements java.util.concurrent.https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/Callable.html?is-external=true; title="class or interface in java.util.concurrent">CallableV, org.apache.hadoop.hbase.errorhandling.ForeignExceptionListener) -org.apache.hadoop.hbase.procedure.TestProcedureDescriber.TestProcedure +org.apache.hadoop.hbase.procedure.TestProcedure.LatchedProcedure -org.apache.hadoop.hbase.procedure.Procedure (implements java.util.concurrent.https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/Callable.html?is-external=true; title="class or interface in java.util.concurrent">CallableV, org.apache.hadoop.hbase.errorhandling.ForeignExceptionListener) +org.apache.hadoop.hbase.procedure2.ProcedureTEnvironment (implements java.lang.https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true; title="class or interface in java.lang">ComparableT) -org.apache.hadoop.hbase.procedure.TestProcedure.LatchedProcedure +org.apache.hadoop.hbase.procedure.TestProcedureDescriber.TestProcedure org.apache.hadoop.hbase.procedure.ProcedureManager http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/testdevapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
[09/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/devapidocs/src-html/org/apache/hadoop/hbase/util/BloomFilterUtil.html -- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/util/BloomFilterUtil.html b/devapidocs/src-html/org/apache/hadoop/hbase/util/BloomFilterUtil.html index 430aa77..ae6ba83 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/util/BloomFilterUtil.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/util/BloomFilterUtil.html @@ -29,263 +29,270 @@ 021import java.util.Random; 022 023import org.apache.hadoop.hbase.Cell; -024import org.apache.yetus.audience.InterfaceAudience; -025import org.apache.hadoop.hbase.nio.ByteBuff; -026import org.apache.hadoop.hbase.regionserver.BloomType; +024import org.apache.hadoop.hbase.nio.ByteBuff; +025import org.apache.hadoop.hbase.regionserver.BloomType; +026import org.apache.yetus.audience.InterfaceAudience; 027 -028/** -029 * Utility methods related to BloomFilters -030 */ -031@InterfaceAudience.Private -032public final class BloomFilterUtil { -033 -034 /** Record separator for the Bloom filter statistics human-readable string */ -035 public static final String STATS_RECORD_SEP = "; "; -036 /** -037 * Used in computing the optimal Bloom filter size. This approximately equals -038 * 0.480453. -039 */ -040 public static final double LOG2_SQUARED = Math.log(2) * Math.log(2); -041 -042 /** -043 * A random number generator to use for "fake lookups" when testing to -044 * estimate the ideal false positive rate. -045 */ -046 private static Random randomGeneratorForTest; -047 -048 /** Bit-value lookup array to prevent doing the same work over and over */ -049 public static final byte [] bitvals = { -050(byte) 0x01, -051(byte) 0x02, -052(byte) 0x04, -053(byte) 0x08, -054(byte) 0x10, -055(byte) 0x20, -056(byte) 0x40, -057(byte) 0x80 -058 }; -059 -060 /** -061 * Private constructor to keep this class from being instantiated. -062 */ -063 private BloomFilterUtil() { -064 } -065 -066 /** -067 * @param maxKeys -068 * @param errorRate -069 * @return the number of bits for a Bloom filter than can hold the given -070 * number of keys and provide the given error rate, assuming that the -071 * optimal number of hash functions is used and it does not have to -072 * be an integer. -073 */ -074 public static long computeBitSize(long maxKeys, double errorRate) { -075return (long) Math.ceil(maxKeys * (-Math.log(errorRate) / LOG2_SQUARED)); -076 } -077 -078 public static void setFakeLookupMode(boolean enabled) { -079if (enabled) { -080 randomGeneratorForTest = new Random(283742987L); -081} else { -082 randomGeneratorForTest = null; -083} -084 } -085 -086 /** -087 * The maximum number of keys we can put into a Bloom filter of a certain -088 * size to maintain the given error rate, assuming the number of hash -089 * functions is chosen optimally and does not even have to be an integer -090 * (hence the "ideal" in the function name). -091 * -092 * @param bitSize -093 * @param errorRate -094 * @return maximum number of keys that can be inserted into the Bloom filter -095 * @see #computeMaxKeys(long, double, int) for a more precise estimate -096 */ -097 public static long idealMaxKeys(long bitSize, double errorRate) { -098// The reason we need to use floor here is that otherwise we might put -099// more keys in a Bloom filter than is allowed by the target error rate. -100return (long) (bitSize * (LOG2_SQUARED / -Math.log(errorRate))); -101 } -102 -103 /** -104 * The maximum number of keys we can put into a Bloom filter of a certain -105 * size to get the given error rate, with the given number of hash functions. -106 * -107 * @param bitSize -108 * @param errorRate -109 * @param hashCount -110 * @return the maximum number of keys that can be inserted in a Bloom filter -111 * to maintain the target error rate, if the number of hash functions -112 * is provided. -113 */ -114 public static long computeMaxKeys(long bitSize, double errorRate, -115 int hashCount) { -116return (long) (-bitSize * 1.0 / hashCount * -117Math.log(1 - Math.exp(Math.log(errorRate) / hashCount))); -118 } -119 -120 /** -121 * Computes the actual error rate for the given number of elements, number -122 * of bits, and number of hash functions. Taken directly from the -123 * a href= -124 * "http://en.wikipedia.org/wiki/Bloom_filter#Probability_of_false_positives; -125 * Wikipedia Bloom filter article/a. -126 * -127 * @param maxKeys -128 * @param bitSize -129 * @param functionCount -130 * @return the actual error rate -131 */ -132 public static double actualErrorRate(long maxKeys, long bitSize, -133 int functionCount) { -134
[18/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/checkstyle-aggregate.html -- diff --git a/checkstyle-aggregate.html b/checkstyle-aggregate.html index 5bc4312..46b1067 100644 --- a/checkstyle-aggregate.html +++ b/checkstyle-aggregate.html @@ -7,7 +7,7 @@ - + Apache HBase Checkstyle Results @@ -284,7 +284,7 @@ 3739 0 0 -15225 +15224 Files @@ -8887,7 +8887,7 @@ org/apache/hadoop/hbase/util/BloomFilterUtil.java 0 0 -16 +15 org/apache/hadoop/hbase/util/BloomFilterWriter.java 0 @@ -9783,7 +9783,7 @@ sortStaticImportsAlphabetically: true groups: *,org.apache.hbase.thirdparty,org.apache.hadoop.hbase.shaded option: top -1150 +1149 Error @@ -9813,12 +9813,12 @@ http://checkstyle.sourceforge.net/config_javadoc.html#JavadocTagContinuationIndentation;>JavadocTagContinuationIndentation offset: 2 -734 +733 Error http://checkstyle.sourceforge.net/config_javadoc.html#NonEmptyAtclauseDescription;>NonEmptyAtclauseDescription -3539 +3540 Error misc @@ -19148,7 +19148,7 @@ Error javadoc -JavadocTagContinuationIndentation +NonEmptyAtclauseDescription Javadoc comment at column 0 has parse error. Details: no viable alternative at input ' *' while parsing JAVADOC_TAG 117 @@ -72094,43 +72094,43 @@ blocks NeedBraces 'if' construct must use '{}'s. -202 +205 Error blocks NeedBraces 'if' construct must use '{}'s. -207 +210 Error indentation Indentation 'throws' has incorrect indentation level 2, expected level should be 4. -227 +230 Error javadoc JavadocTagContinuationIndentation Line continuation have incorrect indentation level, expected level should be 2. -467 +470 Error javadoc JavadocTagContinuationIndentation Line continuation have incorrect indentation level, expected level should be 2. -468 +471 Error indentation Indentation 'method call' child has incorrect indentation level 10, expected level should be 12. -725 +728 Error blocks NeedBraces 'if' construct must use '{}'s. -815 +818 org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALLoaderPerformanceEvaluation.java @@ -83446,13 +83446,13 @@ blocks NeedBraces 'if' construct must use '{}'s. -330 +332 Error sizes LineLength Line is longer than 100 characters (found 109). -362 +364 org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java @@ -107629,110 +107629,104 @@ Line Error -imports -ImportOrder -Wrong order for 'org.apache.hadoop.hbase.nio.ByteBuff' import. -25 +javadoc +NonEmptyAtclauseDescription +At-clause should have a non-empty description. +69 Error javadoc NonEmptyAtclauseDescription At-clause should have a non-empty description. -67 +70 Error javadoc NonEmptyAtclauseDescription At-clause should have a non-empty description. -68 +99 Error javadoc NonEmptyAtclauseDescription At-clause should have a non-empty description. -92 +100 Error javadoc NonEmptyAtclauseDescription At-clause should have a non-empty description. -93 +114 Error javadoc NonEmptyAtclauseDescription At-clause should have a non-empty description. -107 +115 Error javadoc NonEmptyAtclauseDescription At-clause should have a non-empty description. -108 +116 Error javadoc NonEmptyAtclauseDescription At-clause should have a non-empty description. -109 +134 Error javadoc NonEmptyAtclauseDescription At-clause should have a non-empty description. -127 +135 Error javadoc NonEmptyAtclauseDescription At-clause should have a non-empty description. -128 +136 Error javadoc NonEmptyAtclauseDescription At-clause should have a non-empty description. -129 +149 Error javadoc NonEmptyAtclauseDescription At-clause should have a non-empty description. -142 +150 Error javadoc NonEmptyAtclauseDescription At-clause should have a non-empty description. -143 +184 Error javadoc NonEmptyAtclauseDescription At-clause should have a non-empty description. -177 +185 Error -javadoc -NonEmptyAtclauseDescription -At-clause should have a non-empty description. -178 - -Error indentation Indentation 'method def modifier' has incorrect indentation level 3, expected level should be 2. -246 +253 org/apache/hadoop/hbase/util/BloomFilterWriter.java - + Severity Category Rule Message Line - + Error imports ImportOrder @@ -107741,73 +107735,73 @@ org/apache/hadoop/hbase/util/BoundedPriorityBlockingQueue.java - + Severity Category Rule Message Line - + Error imports ImportOrder Wrong order for 'java.util.concurrent.BlockingQueue' import. 23 - + Error imports ImportOrder Wrong order for 'java.util.Collection' import. 25 - + Error imports ImportOrder Wrong order for 'java.util.AbstractQueue' import. 28 - + Error blocks NeedBraces 'if' construct must use '{}'s. 95 - + Error blocks NeedBraces 'if' construct must use '{}'s. 160 - + Error blocks NeedBraces 'if'
[11/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.WalProcedureMap.html -- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.WalProcedureMap.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.WalProcedureMap.html index ba63645..6253d3f 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.WalProcedureMap.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.WalProcedureMap.html @@ -148,725 +148,728 @@ 140 LOG.info("Rebuilding tracker for " + log); 141} 142 -143FSDataInputStream stream = log.getStream(); -144try { -145 boolean hasMore = true; -146 while (hasMore) { -147ProcedureWALEntry entry = ProcedureWALFormat.readEntry(stream); -148if (entry == null) { -149 LOG.warn("Nothing left to decode. Exiting with missing EOF, log=" + log); -150 break; -151} -152switch (entry.getType()) { -153 case PROCEDURE_WAL_INIT: -154readInitEntry(entry); -155break; -156 case PROCEDURE_WAL_INSERT: -157readInsertEntry(entry); -158break; -159 case PROCEDURE_WAL_UPDATE: -160 case PROCEDURE_WAL_COMPACT: -161readUpdateEntry(entry); -162break; -163 case PROCEDURE_WAL_DELETE: -164readDeleteEntry(entry); -165break; -166 case PROCEDURE_WAL_EOF: -167hasMore = false; -168break; -169 default: -170throw new CorruptedWALProcedureStoreException("Invalid entry: " + entry); -171} -172 } -173} catch (InvalidProtocolBufferException e) { -174 LOG.error("While reading procedure from " + log, e); -175 loader.markCorruptedWAL(log, e); -176} -177 -178if (localTracker != null) { -179 localTracker.setPartialFlag(false); -180} -181if (!localProcedureMap.isEmpty()) { -182 log.setProcIds(localProcedureMap.getMinProcId(), localProcedureMap.getMaxProcId()); -183 procedureMap.mergeTail(localProcedureMap); -184 -185 //if (hasFastStartSupport) { -186 // TODO: Some procedure may be already runnables (see readInitEntry()) -187 // (we can also check the "update map" in the log trackers) -188 // -- -189 //EntryIterator iter = procedureMap.fetchReady(); -190 //if (iter != null) loader.load(iter); +143long count = 0; +144FSDataInputStream stream = log.getStream(); +145try { +146 boolean hasMore = true; +147 while (hasMore) { +148ProcedureWALEntry entry = ProcedureWALFormat.readEntry(stream); +149if (entry == null) { +150 LOG.warn("Nothing left to decode. Exiting with missing EOF, log=" + log); +151 break; +152} +153count++; +154switch (entry.getType()) { +155 case PROCEDURE_WAL_INIT: +156readInitEntry(entry); +157break; +158 case PROCEDURE_WAL_INSERT: +159readInsertEntry(entry); +160break; +161 case PROCEDURE_WAL_UPDATE: +162 case PROCEDURE_WAL_COMPACT: +163readUpdateEntry(entry); +164break; +165 case PROCEDURE_WAL_DELETE: +166readDeleteEntry(entry); +167break; +168 case PROCEDURE_WAL_EOF: +169hasMore = false; +170break; +171 default: +172throw new CorruptedWALProcedureStoreException("Invalid entry: " + entry); +173} +174 } +175 LOG.info("Read {} entries in {}", count, log); +176} catch (InvalidProtocolBufferException e) { +177 LOG.error("While reading entry #{} in {}", count, log, e); +178 loader.markCorruptedWAL(log, e); +179} +180 +181if (localTracker != null) { +182 localTracker.setPartialFlag(false); +183} +184if (!localProcedureMap.isEmpty()) { +185 log.setProcIds(localProcedureMap.getMinProcId(), localProcedureMap.getMaxProcId()); +186 procedureMap.mergeTail(localProcedureMap); +187 +188 //if (hasFastStartSupport) { +189 // TODO: Some procedure may be already runnables (see readInitEntry()) +190 // (we can also check the "update map" in the log trackers) 191 // -- -192 //} -193} -194 } -195 -196 public void finish() throws IOException { -197// notify the loader about the max proc ID -198loader.setMaxProcId(maxProcId); -199 -200// fetch the procedure ready to run. -201ProcedureIterator procIter =
[17/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/checkstyle.rss -- diff --git a/checkstyle.rss b/checkstyle.rss index a54afb5..39d3439 100644 --- a/checkstyle.rss +++ b/checkstyle.rss @@ -26,7 +26,7 @@ under the License. 2007 - 2018 The Apache Software Foundation File: 3739, - Errors: 15225, + Errors: 15224, Warnings: 0, Infos: 0 @@ -35657,7 +35657,7 @@ under the License. 0 - 16 + 15 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/coc.html -- diff --git a/coc.html b/coc.html index 5358760..bc5cd45 100644 --- a/coc.html +++ b/coc.html @@ -7,7 +7,7 @@ - + Apache HBase Code of Conduct Policy @@ -375,7 +375,7 @@ email to mailto:priv...@hbase.apache.org;>the priv https://www.apache.org/;>The Apache Software Foundation. All rights reserved. - Last Published: 2018-09-12 + Last Published: 2018-09-13 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/dependencies.html -- diff --git a/dependencies.html b/dependencies.html index 57d0669..e153205 100644 --- a/dependencies.html +++ b/dependencies.html @@ -7,7 +7,7 @@ - + Apache HBase Project Dependencies @@ -440,7 +440,7 @@ https://www.apache.org/;>The Apache Software Foundation. All rights reserved. - Last Published: 2018-09-12 + Last Published: 2018-09-13 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/dependency-convergence.html -- diff --git a/dependency-convergence.html b/dependency-convergence.html index b771aae..c49a190 100644 --- a/dependency-convergence.html +++ b/dependency-convergence.html @@ -7,7 +7,7 @@ - + Apache HBase Reactor Dependency Convergence @@ -879,7 +879,7 @@ https://www.apache.org/;>The Apache Software Foundation. All rights reserved. - Last Published: 2018-09-12 + Last Published: 2018-09-13 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/dependency-info.html -- diff --git a/dependency-info.html b/dependency-info.html index 9b10c65..b154e8d 100644 --- a/dependency-info.html +++ b/dependency-info.html @@ -7,7 +7,7 @@ - + Apache HBase Dependency Information @@ -313,7 +313,7 @@ https://www.apache.org/;>The Apache Software Foundation. All rights reserved. - Last Published: 2018-09-12 + Last Published: 2018-09-13 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/dependency-management.html -- diff --git a/dependency-management.html b/dependency-management.html index 07b5bdf..c41bbff 100644 --- a/dependency-management.html +++ b/dependency-management.html @@ -7,7 +7,7 @@ - + Apache HBase Project Dependency Management @@ -1005,7 +1005,7 @@ https://www.apache.org/;>The Apache Software Foundation. All rights reserved. - Last Published: 2018-09-12 + Last Published: 2018-09-13 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/devapidocs/constant-values.html -- diff --git a/devapidocs/constant-values.html b/devapidocs/constant-values.html index 3541ab6..61be308 100644 --- a/devapidocs/constant-values.html +++ b/devapidocs/constant-values.html @@ -3824,21 +3824,21 @@ publicstaticfinalhttps://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true; title="class or interface in java.lang">String date -"Wed Sep 12 14:44:52 UTC 2018" +"Thu Sep 13 14:44:44 UTC 2018" publicstaticfinalhttps://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true; title="class or interface in java.lang">String revision
[16/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.WalProcedureMap.html -- diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.WalProcedureMap.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.WalProcedureMap.html index 9ec698d..9cd1601 100644 --- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.WalProcedureMap.html +++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.WalProcedureMap.html @@ -113,7 +113,7 @@ var activeTableTab = "activeTableTab"; -private static class ProcedureWALFormatReader.WalProcedureMap +private static class ProcedureWALFormatReader.WalProcedureMap extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true; title="class or interface in java.lang">Object @@ -325,7 +325,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html procedureMap -privateProcedureWALFormatReader.Entry[] procedureMap +privateProcedureWALFormatReader.Entry[] procedureMap @@ -334,7 +334,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html replayOrderHead -privateProcedureWALFormatReader.Entry replayOrderHead +privateProcedureWALFormatReader.Entry replayOrderHead @@ -343,7 +343,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html replayOrderTail -privateProcedureWALFormatReader.Entry replayOrderTail +privateProcedureWALFormatReader.Entry replayOrderTail @@ -352,7 +352,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html rootHead -privateProcedureWALFormatReader.Entry rootHead +privateProcedureWALFormatReader.Entry rootHead @@ -361,7 +361,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html childUnlinkedHead -privateProcedureWALFormatReader.Entry childUnlinkedHead +privateProcedureWALFormatReader.Entry childUnlinkedHead @@ -370,7 +370,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html minProcId -privatelong minProcId +privatelong minProcId @@ -379,7 +379,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html maxProcId -privatelong maxProcId +privatelong maxProcId @@ -396,7 +396,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html WalProcedureMap -publicWalProcedureMap(intsize) +publicWalProcedureMap(intsize) @@ -413,7 +413,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html add -publicvoidadd(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureprocProto) +publicvoidadd(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureprocProto) @@ -422,7 +422,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html isIncreasing -private staticbooleanisIncreasing(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.Procedurecurrent, +private staticbooleanisIncreasing(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.Procedurecurrent, org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.Procedurecandidate) Returns: @@ -438,7 +438,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html remove -publicbooleanremove(longprocId) +publicbooleanremove(longprocId) @@ -447,7 +447,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html trackProcIds -privatevoidtrackProcIds(longprocId) +privatevoidtrackProcIds(longprocId) @@ -456,7 +456,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html getMinProcId -publiclonggetMinProcId() +publiclonggetMinProcId() @@ -465,7 +465,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html getMaxProcId -publiclonggetMaxProcId() +publiclonggetMaxProcId() @@ -474,7 +474,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html contains -publicbooleancontains(longprocId) +publicbooleancontains(longprocId) @@ -483,7 +483,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html isEmpty -publicbooleanisEmpty() +publicbooleanisEmpty() @@ -492,7 +492,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html clear -publicvoidclear() +publicvoidclear() @@ -501,7 +501,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html mergeTail -publicvoidmergeTail(ProcedureWALFormatReader.WalProcedureMapother) +publicvoidmergeTail(ProcedureWALFormatReader.WalProcedureMapother) @@ -510,7 +510,7 @@ extends
[15/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.InvalidWALDataException.html -- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.InvalidWALDataException.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.InvalidWALDataException.html index 5815be5..4a42bda 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.InvalidWALDataException.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.InvalidWALDataException.html @@ -91,178 +91,177 @@ 083 // Ignore the last log which is current active log. 084 while (logs.hasNext()) { 085ProcedureWALFile log = logs.next(); -086LOG.debug("Loading WAL id={}", log.getLogId()); -087log.open(); -088try { -089 reader.read(log); -090} finally { -091 log.close(); -092} -093 } -094 reader.finish(); -095 -096 // The tracker is now updated with all the procedures read from the logs -097 tracker.setPartialFlag(false); -098 tracker.resetUpdates(); -099} finally { -100 tracker.setKeepDeletes(false); -101} -102 } -103 -104 public static void writeHeader(OutputStream stream, ProcedureWALHeader header) -105 throws IOException { -106header.writeDelimitedTo(stream); -107 } -108 -109 /* -110 * +-+ -111 * | END OF WAL DATA | ---+ -112 * +-+ | -113 * | | | -114 * | Tracker | | -115 * | | | -116 * +-+ | -117 * | version | | -118 * +-+ | -119 * | TRAILER_MAGIC | | -120 * +-+ | -121 * | offset |-+ -122 * +-+ -123 */ -124 public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker) -125 throws IOException { -126long offset = stream.getPos(); -127 -128// Write EOF Entry -129ProcedureWALEntry.newBuilder() -130 .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) -131 .build().writeDelimitedTo(stream); -132 -133// Write Tracker -134 tracker.toProto().writeDelimitedTo(stream); -135 -136stream.write(TRAILER_VERSION); -137StreamUtils.writeLong(stream, TRAILER_MAGIC); -138StreamUtils.writeLong(stream, offset); -139return stream.getPos() - offset; -140 } -141 -142 public static ProcedureWALHeader readHeader(InputStream stream) -143 throws IOException { -144ProcedureWALHeader header; -145try { -146 header = ProcedureWALHeader.parseDelimitedFrom(stream); -147} catch (InvalidProtocolBufferException e) { -148 throw new InvalidWALDataException(e); -149} -150 -151if (header == null) { -152 throw new InvalidWALDataException("No data available to read the Header"); -153} -154 -155if (header.getVersion() 0 || header.getVersion() != HEADER_VERSION) { -156 throw new InvalidWALDataException("Invalid Header version. got " + header.getVersion() + -157 " expected " + HEADER_VERSION); -158} -159 -160if (header.getType() 0 || header.getType() LOG_TYPE_MAX_VALID) { -161 throw new InvalidWALDataException("Invalid header type. got " + header.getType()); -162} -163 -164return header; -165 } -166 -167 public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size) -168 throws IOException { -169// Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset -170long trailerPos = size - 17; -171 -172if (trailerPos startPos) { -173 throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos); -174} -175 -176stream.seek(trailerPos); -177int version = stream.read(); -178if (version != TRAILER_VERSION) { -179 throw new InvalidWALDataException("Invalid Trailer version. got " + version + -180 " expected " + TRAILER_VERSION); -181} -182 -183long magic = StreamUtils.readLong(stream); -184if (magic != TRAILER_MAGIC) { -185 throw new InvalidWALDataException("Invalid Trailer magic. got " + magic + -186 " expected " + TRAILER_MAGIC); -187} -188 -189long trailerOffset = StreamUtils.readLong(stream); -190stream.seek(trailerOffset); -191 -192ProcedureWALEntry entry = readEntry(stream); -193if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) { -194 throw new InvalidWALDataException("Invalid Trailer begin"); -195} -196 -197ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder() -198 .setVersion(version) -199
[12/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.EntryIterator.html -- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.EntryIterator.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.EntryIterator.html index ba63645..6253d3f 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.EntryIterator.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.EntryIterator.html @@ -148,725 +148,728 @@ 140 LOG.info("Rebuilding tracker for " + log); 141} 142 -143FSDataInputStream stream = log.getStream(); -144try { -145 boolean hasMore = true; -146 while (hasMore) { -147ProcedureWALEntry entry = ProcedureWALFormat.readEntry(stream); -148if (entry == null) { -149 LOG.warn("Nothing left to decode. Exiting with missing EOF, log=" + log); -150 break; -151} -152switch (entry.getType()) { -153 case PROCEDURE_WAL_INIT: -154readInitEntry(entry); -155break; -156 case PROCEDURE_WAL_INSERT: -157readInsertEntry(entry); -158break; -159 case PROCEDURE_WAL_UPDATE: -160 case PROCEDURE_WAL_COMPACT: -161readUpdateEntry(entry); -162break; -163 case PROCEDURE_WAL_DELETE: -164readDeleteEntry(entry); -165break; -166 case PROCEDURE_WAL_EOF: -167hasMore = false; -168break; -169 default: -170throw new CorruptedWALProcedureStoreException("Invalid entry: " + entry); -171} -172 } -173} catch (InvalidProtocolBufferException e) { -174 LOG.error("While reading procedure from " + log, e); -175 loader.markCorruptedWAL(log, e); -176} -177 -178if (localTracker != null) { -179 localTracker.setPartialFlag(false); -180} -181if (!localProcedureMap.isEmpty()) { -182 log.setProcIds(localProcedureMap.getMinProcId(), localProcedureMap.getMaxProcId()); -183 procedureMap.mergeTail(localProcedureMap); -184 -185 //if (hasFastStartSupport) { -186 // TODO: Some procedure may be already runnables (see readInitEntry()) -187 // (we can also check the "update map" in the log trackers) -188 // -- -189 //EntryIterator iter = procedureMap.fetchReady(); -190 //if (iter != null) loader.load(iter); +143long count = 0; +144FSDataInputStream stream = log.getStream(); +145try { +146 boolean hasMore = true; +147 while (hasMore) { +148ProcedureWALEntry entry = ProcedureWALFormat.readEntry(stream); +149if (entry == null) { +150 LOG.warn("Nothing left to decode. Exiting with missing EOF, log=" + log); +151 break; +152} +153count++; +154switch (entry.getType()) { +155 case PROCEDURE_WAL_INIT: +156readInitEntry(entry); +157break; +158 case PROCEDURE_WAL_INSERT: +159readInsertEntry(entry); +160break; +161 case PROCEDURE_WAL_UPDATE: +162 case PROCEDURE_WAL_COMPACT: +163readUpdateEntry(entry); +164break; +165 case PROCEDURE_WAL_DELETE: +166readDeleteEntry(entry); +167break; +168 case PROCEDURE_WAL_EOF: +169hasMore = false; +170break; +171 default: +172throw new CorruptedWALProcedureStoreException("Invalid entry: " + entry); +173} +174 } +175 LOG.info("Read {} entries in {}", count, log); +176} catch (InvalidProtocolBufferException e) { +177 LOG.error("While reading entry #{} in {}", count, log, e); +178 loader.markCorruptedWAL(log, e); +179} +180 +181if (localTracker != null) { +182 localTracker.setPartialFlag(false); +183} +184if (!localProcedureMap.isEmpty()) { +185 log.setProcIds(localProcedureMap.getMinProcId(), localProcedureMap.getMaxProcId()); +186 procedureMap.mergeTail(localProcedureMap); +187 +188 //if (hasFastStartSupport) { +189 // TODO: Some procedure may be already runnables (see readInitEntry()) +190 // (we can also check the "update map" in the log trackers) 191 // -- -192 //} -193} -194 } -195 -196 public void finish() throws IOException { -197// notify the loader about the max proc ID -198loader.setMaxProcId(maxProcId); -199 -200// fetch the procedure ready to run. -201ProcedureIterator procIter = procedureMap.fetchReady(); -202
[04/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.OpenFileDescriptorResourceAnalyzer.html -- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.OpenFileDescriptorResourceAnalyzer.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.OpenFileDescriptorResourceAnalyzer.html index cb4c472..47aac2c 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.OpenFileDescriptorResourceAnalyzer.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.OpenFileDescriptorResourceAnalyzer.html @@ -26,179 +26,221 @@ 018 019package org.apache.hadoop.hbase; 020 -021import java.util.ArrayList; -022import java.util.HashSet; -023import java.util.List; -024import java.util.Map; -025import java.util.Set; -026import java.util.concurrent.ConcurrentHashMap; -027import org.apache.hadoop.hbase.ResourceChecker.Phase; -028import org.apache.hadoop.hbase.util.JVM; -029import org.junit.runner.notification.RunListener; -030 -031/** -032 * Listen to the test progress and check the usage of: -033 * ul -034 * lithreads/li -035 * liopen file descriptor/li -036 * limax open file descriptor/li -037 * /ul -038 * p -039 * When surefire forkMode=once/always/perthread, this code is executed on the forked process. -040 */ -041public class ResourceCheckerJUnitListener extends RunListener { -042 private MapString, ResourceChecker rcs = new ConcurrentHashMap(); -043 -044 static class ThreadResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { -045private static SetString initialThreadNames = new HashSet(); -046private static ListString stringsToLog = null; -047 -048@Override -049public int getVal(Phase phase) { -050 MapThread, StackTraceElement[] stackTraces = Thread.getAllStackTraces(); -051 if (phase == Phase.INITIAL) { -052stringsToLog = null; -053for (Thread t : stackTraces.keySet()) { -054 initialThreadNames.add(t.getName()); -055} -056 } else if (phase == Phase.END) { -057if (stackTraces.size() initialThreadNames.size()) { -058 stringsToLog = new ArrayList(); -059 for (Thread t : stackTraces.keySet()) { -060if (!initialThreadNames.contains(t.getName())) { -061 stringsToLog.add("\nPotentially hanging thread: " + t.getName() + "\n"); -062 StackTraceElement[] stackElements = stackTraces.get(t); -063 for (StackTraceElement ele : stackElements) { -064stringsToLog.add("\t" + ele + "\n"); -065 } -066} -067 } -068} -069 } -070 return stackTraces.size(); -071} -072 -073@Override -074public int getMax() { -075 return 500; -076} -077 -078@Override -079public ListString getStringsToLog() { -080 return stringsToLog; -081} -082 } -083 -084 -085 static class OpenFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { -086@Override -087public int getVal(Phase phase) { -088 if (!JVM.isUnix()) { -089return 0; -090 } -091 JVM jvm = new JVM(); -092 return (int) jvm.getOpenFileDescriptorCount(); -093} -094 -095@Override -096public int getMax() { -097 return 1024; -098} -099 } -100 -101 static class MaxFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { -102@Override -103public int getVal(Phase phase) { -104 if (!JVM.isUnix()) { -105return 0; -106 } -107 JVM jvm = new JVM(); -108 return (int) jvm.getMaxFileDescriptorCount(); -109} -110 } -111 -112 static class SystemLoadAverageResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { -113@Override -114public int getVal(Phase phase) { -115 if (!JVM.isUnix()) { -116return 0; -117 } -118 return (int) (new JVM().getSystemLoadAverage() * 100); -119} -120 } -121 -122 static class ProcessCountResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { -123@Override -124public int getVal(Phase phase) { -125 if (!JVM.isUnix()) { -126return 0; -127 } -128 return new JVM().getNumberOfRunningProcess(); -129} -130 } -131 -132 static class AvailableMemoryMBResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { -133@Override -134public int getVal(Phase phase) { -135 if (!JVM.isUnix()) { -136return 0; -137 } -138 return (int) (new JVM().getFreeMemory() / (1024L * 1024L)); -139} -140 } -141 -142 -143 /** -144 * To be implemented by sub classes if they want to add specific ResourceAnalyzer. -145 */ -146 protected void addResourceAnalyzer(ResourceChecker rc) { -147 } -148 -149 -150
[13/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.Entry.html -- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.Entry.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.Entry.html index ba63645..6253d3f 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.Entry.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.Entry.html @@ -148,725 +148,728 @@ 140 LOG.info("Rebuilding tracker for " + log); 141} 142 -143FSDataInputStream stream = log.getStream(); -144try { -145 boolean hasMore = true; -146 while (hasMore) { -147ProcedureWALEntry entry = ProcedureWALFormat.readEntry(stream); -148if (entry == null) { -149 LOG.warn("Nothing left to decode. Exiting with missing EOF, log=" + log); -150 break; -151} -152switch (entry.getType()) { -153 case PROCEDURE_WAL_INIT: -154readInitEntry(entry); -155break; -156 case PROCEDURE_WAL_INSERT: -157readInsertEntry(entry); -158break; -159 case PROCEDURE_WAL_UPDATE: -160 case PROCEDURE_WAL_COMPACT: -161readUpdateEntry(entry); -162break; -163 case PROCEDURE_WAL_DELETE: -164readDeleteEntry(entry); -165break; -166 case PROCEDURE_WAL_EOF: -167hasMore = false; -168break; -169 default: -170throw new CorruptedWALProcedureStoreException("Invalid entry: " + entry); -171} -172 } -173} catch (InvalidProtocolBufferException e) { -174 LOG.error("While reading procedure from " + log, e); -175 loader.markCorruptedWAL(log, e); -176} -177 -178if (localTracker != null) { -179 localTracker.setPartialFlag(false); -180} -181if (!localProcedureMap.isEmpty()) { -182 log.setProcIds(localProcedureMap.getMinProcId(), localProcedureMap.getMaxProcId()); -183 procedureMap.mergeTail(localProcedureMap); -184 -185 //if (hasFastStartSupport) { -186 // TODO: Some procedure may be already runnables (see readInitEntry()) -187 // (we can also check the "update map" in the log trackers) -188 // -- -189 //EntryIterator iter = procedureMap.fetchReady(); -190 //if (iter != null) loader.load(iter); +143long count = 0; +144FSDataInputStream stream = log.getStream(); +145try { +146 boolean hasMore = true; +147 while (hasMore) { +148ProcedureWALEntry entry = ProcedureWALFormat.readEntry(stream); +149if (entry == null) { +150 LOG.warn("Nothing left to decode. Exiting with missing EOF, log=" + log); +151 break; +152} +153count++; +154switch (entry.getType()) { +155 case PROCEDURE_WAL_INIT: +156readInitEntry(entry); +157break; +158 case PROCEDURE_WAL_INSERT: +159readInsertEntry(entry); +160break; +161 case PROCEDURE_WAL_UPDATE: +162 case PROCEDURE_WAL_COMPACT: +163readUpdateEntry(entry); +164break; +165 case PROCEDURE_WAL_DELETE: +166readDeleteEntry(entry); +167break; +168 case PROCEDURE_WAL_EOF: +169hasMore = false; +170break; +171 default: +172throw new CorruptedWALProcedureStoreException("Invalid entry: " + entry); +173} +174 } +175 LOG.info("Read {} entries in {}", count, log); +176} catch (InvalidProtocolBufferException e) { +177 LOG.error("While reading entry #{} in {}", count, log, e); +178 loader.markCorruptedWAL(log, e); +179} +180 +181if (localTracker != null) { +182 localTracker.setPartialFlag(false); +183} +184if (!localProcedureMap.isEmpty()) { +185 log.setProcIds(localProcedureMap.getMinProcId(), localProcedureMap.getMaxProcId()); +186 procedureMap.mergeTail(localProcedureMap); +187 +188 //if (hasFastStartSupport) { +189 // TODO: Some procedure may be already runnables (see readInitEntry()) +190 // (we can also check the "update map" in the log trackers) 191 // -- -192 //} -193} -194 } -195 -196 public void finish() throws IOException { -197// notify the loader about the max proc ID -198loader.setMaxProcId(maxProcId); -199 -200// fetch the procedure ready to run. -201ProcedureIterator procIter = procedureMap.fetchReady(); -202if (procIter != null)
[14/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html -- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html index 5815be5..4a42bda 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html @@ -91,178 +91,177 @@ 083 // Ignore the last log which is current active log. 084 while (logs.hasNext()) { 085ProcedureWALFile log = logs.next(); -086LOG.debug("Loading WAL id={}", log.getLogId()); -087log.open(); -088try { -089 reader.read(log); -090} finally { -091 log.close(); -092} -093 } -094 reader.finish(); -095 -096 // The tracker is now updated with all the procedures read from the logs -097 tracker.setPartialFlag(false); -098 tracker.resetUpdates(); -099} finally { -100 tracker.setKeepDeletes(false); -101} -102 } -103 -104 public static void writeHeader(OutputStream stream, ProcedureWALHeader header) -105 throws IOException { -106header.writeDelimitedTo(stream); -107 } -108 -109 /* -110 * +-+ -111 * | END OF WAL DATA | ---+ -112 * +-+ | -113 * | | | -114 * | Tracker | | -115 * | | | -116 * +-+ | -117 * | version | | -118 * +-+ | -119 * | TRAILER_MAGIC | | -120 * +-+ | -121 * | offset |-+ -122 * +-+ -123 */ -124 public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker) -125 throws IOException { -126long offset = stream.getPos(); -127 -128// Write EOF Entry -129ProcedureWALEntry.newBuilder() -130 .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) -131 .build().writeDelimitedTo(stream); -132 -133// Write Tracker -134 tracker.toProto().writeDelimitedTo(stream); -135 -136stream.write(TRAILER_VERSION); -137StreamUtils.writeLong(stream, TRAILER_MAGIC); -138StreamUtils.writeLong(stream, offset); -139return stream.getPos() - offset; -140 } -141 -142 public static ProcedureWALHeader readHeader(InputStream stream) -143 throws IOException { -144ProcedureWALHeader header; -145try { -146 header = ProcedureWALHeader.parseDelimitedFrom(stream); -147} catch (InvalidProtocolBufferException e) { -148 throw new InvalidWALDataException(e); -149} -150 -151if (header == null) { -152 throw new InvalidWALDataException("No data available to read the Header"); -153} -154 -155if (header.getVersion() 0 || header.getVersion() != HEADER_VERSION) { -156 throw new InvalidWALDataException("Invalid Header version. got " + header.getVersion() + -157 " expected " + HEADER_VERSION); -158} -159 -160if (header.getType() 0 || header.getType() LOG_TYPE_MAX_VALID) { -161 throw new InvalidWALDataException("Invalid header type. got " + header.getType()); -162} -163 -164return header; -165 } -166 -167 public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size) -168 throws IOException { -169// Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset -170long trailerPos = size - 17; -171 -172if (trailerPos startPos) { -173 throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos); -174} -175 -176stream.seek(trailerPos); -177int version = stream.read(); -178if (version != TRAILER_VERSION) { -179 throw new InvalidWALDataException("Invalid Trailer version. got " + version + -180 " expected " + TRAILER_VERSION); -181} -182 -183long magic = StreamUtils.readLong(stream); -184if (magic != TRAILER_MAGIC) { -185 throw new InvalidWALDataException("Invalid Trailer magic. got " + magic + -186 " expected " + TRAILER_MAGIC); -187} -188 -189long trailerOffset = StreamUtils.readLong(stream); -190stream.seek(trailerOffset); -191 -192ProcedureWALEntry entry = readEntry(stream); -193if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) { -194 throw new InvalidWALDataException("Invalid Trailer begin"); -195} -196 -197ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder() -198 .setVersion(version) -199 .setTrackerPos(stream.getPos()) -200 .build(); -201return trailer; -202 } -203 -204 public static ProcedureWALEntry
[07/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/testdevapidocs/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.SystemLoadAverageResourceAnalyzer.html -- diff --git a/testdevapidocs/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.SystemLoadAverageResourceAnalyzer.html b/testdevapidocs/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.SystemLoadAverageResourceAnalyzer.html index 0aa0fa4..65829a0 100644 --- a/testdevapidocs/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.SystemLoadAverageResourceAnalyzer.html +++ b/testdevapidocs/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.SystemLoadAverageResourceAnalyzer.html @@ -118,7 +118,7 @@ var activeTableTab = "activeTableTab"; -static class ResourceCheckerJUnitListener.SystemLoadAverageResourceAnalyzer +static class ResourceCheckerJUnitListener.SystemLoadAverageResourceAnalyzer extends ResourceChecker.ResourceAnalyzer @@ -196,7 +196,7 @@ extends SystemLoadAverageResourceAnalyzer -SystemLoadAverageResourceAnalyzer() +SystemLoadAverageResourceAnalyzer() @@ -213,7 +213,7 @@ extends getVal -publicintgetVal(ResourceChecker.Phasephase) +publicintgetVal(ResourceChecker.Phasephase) Description copied from class:ResourceChecker.ResourceAnalyzer The value for the resource. http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/testdevapidocs/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.ThreadResourceAnalyzer.html -- diff --git a/testdevapidocs/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.ThreadResourceAnalyzer.html b/testdevapidocs/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.ThreadResourceAnalyzer.html index 95f0293..346ff77 100644 --- a/testdevapidocs/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.ThreadResourceAnalyzer.html +++ b/testdevapidocs/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.ThreadResourceAnalyzer.html @@ -50,7 +50,7 @@ var activeTableTab = "activeTableTab"; PrevClass -NextClass +NextClass Frames @@ -118,7 +118,7 @@ var activeTableTab = "activeTableTab"; -static class ResourceCheckerJUnitListener.ThreadResourceAnalyzer +static class ResourceCheckerJUnitListener.ThreadResourceAnalyzer extends ResourceChecker.ResourceAnalyzer @@ -229,7 +229,7 @@ extends initialThreadNames -private statichttps://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true; title="class or interface in java.util">Sethttps://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true; title="class or interface in java.lang">String initialThreadNames +private statichttps://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true; title="class or interface in java.util">Sethttps://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true; title="class or interface in java.lang">String initialThreadNames @@ -238,7 +238,7 @@ extends stringsToLog -private statichttps://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true; title="class or interface in java.util">Listhttps://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true; title="class or interface in java.lang">String stringsToLog +private statichttps://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true; title="class or interface in java.util">Listhttps://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true; title="class or interface in java.lang">String stringsToLog @@ -255,7 +255,7 @@ extends ThreadResourceAnalyzer -ThreadResourceAnalyzer() +ThreadResourceAnalyzer() @@ -272,7 +272,7 @@ extends getVal -publicintgetVal(ResourceChecker.Phasephase) +publicintgetVal(ResourceChecker.Phasephase) Description copied from class:ResourceChecker.ResourceAnalyzer The value for the resource. @@ -287,7 +287,7 @@ extends getMax -publicintgetMax() +publicintgetMax() Description copied from class:ResourceChecker.ResourceAnalyzer Maximum we set for the resource. Will get a warning in logs if we go over this limit. @@ -303,7 +303,7 @@ extends getStringsToLog -publichttps://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true; title="class or interface in java.util">Listhttps://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true; title="class or interface in java.lang">StringgetStringsToLog() +publichttps://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true; title="class or interface in java.util">Listhttps://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true; title="class or interface in java.lang">StringgetStringsToLog() Overrides: getStringsToLogin classResourceChecker.ResourceAnalyzer @@ -339,7 +339,7 @@ extends PrevClass -NextClass +NextClass Frames
[05/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.GCTimeSecondResourceAnalyzer.html -- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.GCTimeSecondResourceAnalyzer.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.GCTimeSecondResourceAnalyzer.html new file mode 100644 index 000..47aac2c --- /dev/null +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.GCTimeSecondResourceAnalyzer.html @@ -0,0 +1,307 @@ +http://www.w3.org/TR/html4/loose.dtd;> + + +Source code + + + + +001/** +002 * Licensed to the Apache Software Foundation (ASF) under one +003 * or more contributor license agreements. See the NOTICE file +004 * distributed with this work for additional information +005 * regarding copyright ownership. The ASF licenses this file +006 * to you under the Apache License, Version 2.0 (the +007 * "License"); you may not use this file except in compliance +008 * with the License. You may obtain a copy of the License at +009 * +010 * http://www.apache.org/licenses/LICENSE-2.0 +011 * +012 * Unless required by applicable law or agreed to in writing, software +013 * distributed under the License is distributed on an "AS IS" BASIS, +014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +015 * See the License for the specific language governing permissions and +016 * limitations under the License. +017 */ +018 +019package org.apache.hadoop.hbase; +020 +021import java.lang.management.ManagementFactory; +022import java.lang.management.MemoryUsage; +023import java.util.ArrayList; +024import java.util.HashSet; +025import java.util.List; +026import java.util.Map; +027import java.util.Set; +028import java.util.concurrent.ConcurrentHashMap; +029import java.util.concurrent.TimeUnit; +030import org.apache.hadoop.hbase.ResourceChecker.Phase; +031import org.apache.hadoop.hbase.util.JVM; +032import org.junit.runner.notification.RunListener; +033 +034/** +035 * Listen to the test progress and check the usage of: +036 * ul +037 * lithreads/li +038 * liopen file descriptor/li +039 * limax open file descriptor/li +040 * /ul +041 * p +042 * When surefire forkMode=once/always/perthread, this code is executed on the forked process. +043 */ +044public class ResourceCheckerJUnitListener extends RunListener { +045 private MapString, ResourceChecker rcs = new ConcurrentHashMap(); +046 +047 static class ThreadResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { +048private static SetString initialThreadNames = new HashSet(); +049private static ListString stringsToLog = null; +050 +051@Override +052public int getVal(Phase phase) { +053 MapThread, StackTraceElement[] stackTraces = Thread.getAllStackTraces(); +054 if (phase == Phase.INITIAL) { +055stringsToLog = null; +056for (Thread t : stackTraces.keySet()) { +057 initialThreadNames.add(t.getName()); +058} +059 } else if (phase == Phase.END) { +060if (stackTraces.size() initialThreadNames.size()) { +061 stringsToLog = new ArrayList(); +062 for (Thread t : stackTraces.keySet()) { +063if (!initialThreadNames.contains(t.getName())) { +064 stringsToLog.add("\nPotentially hanging thread: " + t.getName() + "\n"); +065 StackTraceElement[] stackElements = stackTraces.get(t); +066 for (StackTraceElement ele : stackElements) { +067stringsToLog.add("\t" + ele + "\n"); +068 } +069} +070 } +071} +072 } +073 return stackTraces.size(); +074} +075 +076@Override +077public int getMax() { +078 return 500; +079} +080 +081@Override +082public ListString getStringsToLog() { +083 return stringsToLog; +084} +085 } +086 +087 +088 static class OpenFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { +089@Override +090public int getVal(Phase phase) { +091 if (!JVM.isUnix()) { +092return 0; +093 } +094 JVM jvm = new JVM(); +095 return (int) jvm.getOpenFileDescriptorCount(); +096} +097 +098@Override +099public int getMax() { +100 return 1024; +101} +102 } +103 +104 static class MaxFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { +105@Override +106public int getVal(Phase phase) { +107 if (!JVM.isUnix()) { +108return 0; +109 } +110 JVM jvm = new JVM(); +111 return (int) jvm.getMaxFileDescriptorCount(); +112} +113 } +114 +115 static class SystemLoadAverageResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { +116@Override +117public int getVal(Phase phase) { +118
[01/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
Repository: hbase-site Updated Branches: refs/heads/asf-site f1997f363 -> 836bc028d http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.html -- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.html index 5f7f1cc..1890d8b 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.html @@ -233,155 +233,157 @@ 225// Test for false positives (some percentage allowed). We test in two modes: 226// "fake lookup" which ignores the key distribution, and production mode. 227for (boolean fakeLookupEnabled : new boolean[] { true, false }) { -228 BloomFilterUtil.setFakeLookupMode(fakeLookupEnabled); -229 try { -230String fakeLookupModeStr = ", fake lookup is " + (fakeLookupEnabled ? -231"enabled" : "disabled"); -232CompoundBloomFilter cbf = (CompoundBloomFilter) r.getGeneralBloomFilter(); -233cbf.enableTestingStats(); -234int numFalsePos = 0; -235Random rand = new Random(EVALUATION_SEED); -236int nTrials = NUM_KV[t] * 10; -237for (int i = 0; i nTrials; ++i) { -238 byte[] query = RandomKeyValueUtil.randomRowOrQualifier(rand); -239 if (isInBloom(scanner, query, bt, rand)) { -240numFalsePos += 1; -241 } -242} -243double falsePosRate = numFalsePos * 1.0 / nTrials; -244 LOG.debug(String.format(testIdMsg -245+ " False positives: %d out of %d (%f)", -246numFalsePos, nTrials, falsePosRate) + fakeLookupModeStr); -247 -248// Check for obvious Bloom filter crashes. -249assertTrue("False positive is too high: " + falsePosRate + " (greater " -250+ "than " + TOO_HIGH_ERROR_RATE + ")" + fakeLookupModeStr, -251falsePosRate TOO_HIGH_ERROR_RATE); -252 -253// Now a more precise check to see if the false positive rate is not -254// too high. The reason we use a relaxed restriction for the real-world -255// case as opposed to the "fake lookup" case is that our hash functions -256// are not completely independent. -257 -258double maxZValue = fakeLookupEnabled ? 1.96 : 2.5; -259 validateFalsePosRate(falsePosRate, nTrials, maxZValue, cbf, -260fakeLookupModeStr); -261 -262// For checking the lower bound we need to eliminate the last chunk, -263// because it is frequently smaller and the false positive rate in it -264// is too low. This does not help if there is only one under-sized -265// chunk, though. -266int nChunks = cbf.getNumChunks(); -267if (nChunks 1) { -268 numFalsePos -= cbf.getNumPositivesForTesting(nChunks - 1); -269 nTrials -= cbf.getNumQueriesForTesting(nChunks - 1); -270 falsePosRate = numFalsePos * 1.0 / nTrials; -271 LOG.info(testIdMsg + " False positive rate without last chunk is " + -272 falsePosRate + fakeLookupModeStr); -273} -274 -275 validateFalsePosRate(falsePosRate, nTrials, -2.58, cbf, -276fakeLookupModeStr); -277 } finally { -278 BloomFilterUtil.setFakeLookupMode(false); -279 } -280} -281 -282r.close(true); // end of test so evictOnClose -283 } -284 -285 private boolean isInBloom(StoreFileScanner scanner, byte[] row, BloomType bt, -286 Random rand) { -287return isInBloom(scanner, row, RandomKeyValueUtil.randomRowOrQualifier(rand)); -288 } -289 -290 private boolean isInBloom(StoreFileScanner scanner, byte[] row, -291 byte[] qualifier) { -292Scan scan = new Scan().withStartRow(row).withStopRow(row, true); -293 scan.addColumn(Bytes.toBytes(RandomKeyValueUtil.COLUMN_FAMILY_NAME), qualifier); -294HStore store = mock(HStore.class); -295 when(store.getColumnFamilyDescriptor()) -296 .thenReturn(ColumnFamilyDescriptorBuilder.of(RandomKeyValueUtil.COLUMN_FAMILY_NAME)); -297return scanner.shouldUseScanner(scan, store, Long.MIN_VALUE); -298 } -299 -300 private Path writeStoreFile(int t, BloomType bt, ListKeyValue kvs) -301 throws IOException { -302 conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE, -303BLOOM_BLOCK_SIZES[t]); -304 conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true); -305cacheConf = new CacheConfig(conf); -306HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCK_SIZES[t]).build(); -307StoreFileWriter w = new StoreFileWriter.Builder(conf, cacheConf, fs) -308
[10/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html -- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html index ba63645..6253d3f 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html @@ -148,725 +148,728 @@ 140 LOG.info("Rebuilding tracker for " + log); 141} 142 -143FSDataInputStream stream = log.getStream(); -144try { -145 boolean hasMore = true; -146 while (hasMore) { -147ProcedureWALEntry entry = ProcedureWALFormat.readEntry(stream); -148if (entry == null) { -149 LOG.warn("Nothing left to decode. Exiting with missing EOF, log=" + log); -150 break; -151} -152switch (entry.getType()) { -153 case PROCEDURE_WAL_INIT: -154readInitEntry(entry); -155break; -156 case PROCEDURE_WAL_INSERT: -157readInsertEntry(entry); -158break; -159 case PROCEDURE_WAL_UPDATE: -160 case PROCEDURE_WAL_COMPACT: -161readUpdateEntry(entry); -162break; -163 case PROCEDURE_WAL_DELETE: -164readDeleteEntry(entry); -165break; -166 case PROCEDURE_WAL_EOF: -167hasMore = false; -168break; -169 default: -170throw new CorruptedWALProcedureStoreException("Invalid entry: " + entry); -171} -172 } -173} catch (InvalidProtocolBufferException e) { -174 LOG.error("While reading procedure from " + log, e); -175 loader.markCorruptedWAL(log, e); -176} -177 -178if (localTracker != null) { -179 localTracker.setPartialFlag(false); -180} -181if (!localProcedureMap.isEmpty()) { -182 log.setProcIds(localProcedureMap.getMinProcId(), localProcedureMap.getMaxProcId()); -183 procedureMap.mergeTail(localProcedureMap); -184 -185 //if (hasFastStartSupport) { -186 // TODO: Some procedure may be already runnables (see readInitEntry()) -187 // (we can also check the "update map" in the log trackers) -188 // -- -189 //EntryIterator iter = procedureMap.fetchReady(); -190 //if (iter != null) loader.load(iter); +143long count = 0; +144FSDataInputStream stream = log.getStream(); +145try { +146 boolean hasMore = true; +147 while (hasMore) { +148ProcedureWALEntry entry = ProcedureWALFormat.readEntry(stream); +149if (entry == null) { +150 LOG.warn("Nothing left to decode. Exiting with missing EOF, log=" + log); +151 break; +152} +153count++; +154switch (entry.getType()) { +155 case PROCEDURE_WAL_INIT: +156readInitEntry(entry); +157break; +158 case PROCEDURE_WAL_INSERT: +159readInsertEntry(entry); +160break; +161 case PROCEDURE_WAL_UPDATE: +162 case PROCEDURE_WAL_COMPACT: +163readUpdateEntry(entry); +164break; +165 case PROCEDURE_WAL_DELETE: +166readDeleteEntry(entry); +167break; +168 case PROCEDURE_WAL_EOF: +169hasMore = false; +170break; +171 default: +172throw new CorruptedWALProcedureStoreException("Invalid entry: " + entry); +173} +174 } +175 LOG.info("Read {} entries in {}", count, log); +176} catch (InvalidProtocolBufferException e) { +177 LOG.error("While reading entry #{} in {}", count, log, e); +178 loader.markCorruptedWAL(log, e); +179} +180 +181if (localTracker != null) { +182 localTracker.setPartialFlag(false); +183} +184if (!localProcedureMap.isEmpty()) { +185 log.setProcIds(localProcedureMap.getMinProcId(), localProcedureMap.getMaxProcId()); +186 procedureMap.mergeTail(localProcedureMap); +187 +188 //if (hasFastStartSupport) { +189 // TODO: Some procedure may be already runnables (see readInitEntry()) +190 // (we can also check the "update map" in the log trackers) 191 // -- -192 //} -193} -194 } -195 -196 public void finish() throws IOException { -197// notify the loader about the max proc ID -198loader.setMaxProcId(maxProcId); -199 -200// fetch the procedure ready to run. -201ProcedureIterator procIter = procedureMap.fetchReady(); -202if (procIter != null) loader.load(procIter); -203 -204//
[19/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
Published site at 5d14c1af65c02f4e87059337c35e4431505de91c. Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/836bc028 Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/836bc028 Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/836bc028 Branch: refs/heads/asf-site Commit: 836bc028dea422e6232b47403cf4c4204a66c19d Parents: f1997f3 Author: jenkins Authored: Thu Sep 13 14:53:28 2018 + Committer: jenkins Committed: Thu Sep 13 14:53:28 2018 + -- acid-semantics.html |4 +- apache_hbase_reference_guide.pdf|4 +- book.html |2 +- bulk-loads.html |4 +- checkstyle-aggregate.html | 3386 +- checkstyle.rss |4 +- coc.html|4 +- dependencies.html |4 +- dependency-convergence.html |4 +- dependency-info.html|4 +- dependency-management.html |4 +- devapidocs/constant-values.html |6 +- devapidocs/index-all.html |6 +- .../hadoop/hbase/backup/package-tree.html |2 +- .../hadoop/hbase/client/package-tree.html | 24 +- .../hadoop/hbase/filter/package-tree.html |6 +- .../hadoop/hbase/io/hfile/package-tree.html |6 +- .../apache/hadoop/hbase/ipc/package-tree.html |2 +- .../hadoop/hbase/mapreduce/package-tree.html|2 +- .../hadoop/hbase/master/package-tree.html |4 +- .../hbase/master/procedure/package-tree.html|4 +- .../org/apache/hadoop/hbase/package-tree.html | 16 +- .../hadoop/hbase/procedure2/package-tree.html |4 +- .../store/wal/ProcedureWALFormat.html | 22 +- .../wal/ProcedureWALFormatReader.Entry.html | 36 +- .../ProcedureWALFormatReader.EntryIterator.html | 18 +- ...rocedureWALFormatReader.WalProcedureMap.html | 68 +- .../store/wal/ProcedureWALFormatReader.html | 18 +- .../hadoop/hbase/quotas/package-tree.html |4 +- .../hadoop/hbase/regionserver/package-tree.html | 18 +- .../hbase/regionserver/wal/package-tree.html|2 +- .../hadoop/hbase/replication/package-tree.html |2 +- .../hadoop/hbase/rest/model/package-tree.html |2 +- .../hbase/security/access/package-tree.html |2 +- .../hadoop/hbase/security/package-tree.html |2 +- .../hadoop/hbase/util/BloomFilterUtil.html | 57 +- .../apache/hadoop/hbase/util/package-tree.html |8 +- .../org/apache/hadoop/hbase/Version.html|6 +- ...cedureWALFormat.InvalidWALDataException.html | 343 +- .../store/wal/ProcedureWALFormat.Loader.html| 343 +- .../store/wal/ProcedureWALFormat.html | 343 +- .../wal/ProcedureWALFormatReader.Entry.html | 1413 .../ProcedureWALFormatReader.EntryIterator.html | 1413 ...rocedureWALFormatReader.WalProcedureMap.html | 1413 .../store/wal/ProcedureWALFormatReader.html | 1413 .../hadoop/hbase/util/BloomFilterUtil.html | 511 +-- downloads.html |4 +- export_control.html |4 +- index.html |4 +- integration.html|4 +- issue-tracking.html |4 +- license.html|4 +- mail-lists.html |4 +- metrics.html|4 +- old_news.html |4 +- plugin-management.html |4 +- plugins.html|4 +- poweredbyhbase.html |4 +- project-info.html |4 +- project-reports.html|4 +- project-summary.html|4 +- pseudo-distributed.html |4 +- replication.html|4 +- resources.html |4 +- source-repository.html |4 +- sponsors.html |4 +- supportingprojects.html |4 +- team-list.html |4 +- testdevapidocs/allclasses-frame.html|4 + testdevapidocs/allclasses-noframe.html |4 + testdevapidocs/index-all.html | 24 + .../hadoop/hbase/RESTApiClusterManager.html |4 +- .../hbase/ResourceChecker.ResourceAnalyzer.html |2 +-
[03/19] hbase-site git commit: Published site at 5d14c1af65c02f4e87059337c35e4431505de91c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/836bc028/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.SystemLoadAverageResourceAnalyzer.html -- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.SystemLoadAverageResourceAnalyzer.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.SystemLoadAverageResourceAnalyzer.html index cb4c472..47aac2c 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.SystemLoadAverageResourceAnalyzer.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.SystemLoadAverageResourceAnalyzer.html @@ -26,179 +26,221 @@ 018 019package org.apache.hadoop.hbase; 020 -021import java.util.ArrayList; -022import java.util.HashSet; -023import java.util.List; -024import java.util.Map; -025import java.util.Set; -026import java.util.concurrent.ConcurrentHashMap; -027import org.apache.hadoop.hbase.ResourceChecker.Phase; -028import org.apache.hadoop.hbase.util.JVM; -029import org.junit.runner.notification.RunListener; -030 -031/** -032 * Listen to the test progress and check the usage of: -033 * ul -034 * lithreads/li -035 * liopen file descriptor/li -036 * limax open file descriptor/li -037 * /ul -038 * p -039 * When surefire forkMode=once/always/perthread, this code is executed on the forked process. -040 */ -041public class ResourceCheckerJUnitListener extends RunListener { -042 private MapString, ResourceChecker rcs = new ConcurrentHashMap(); -043 -044 static class ThreadResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { -045private static SetString initialThreadNames = new HashSet(); -046private static ListString stringsToLog = null; -047 -048@Override -049public int getVal(Phase phase) { -050 MapThread, StackTraceElement[] stackTraces = Thread.getAllStackTraces(); -051 if (phase == Phase.INITIAL) { -052stringsToLog = null; -053for (Thread t : stackTraces.keySet()) { -054 initialThreadNames.add(t.getName()); -055} -056 } else if (phase == Phase.END) { -057if (stackTraces.size() initialThreadNames.size()) { -058 stringsToLog = new ArrayList(); -059 for (Thread t : stackTraces.keySet()) { -060if (!initialThreadNames.contains(t.getName())) { -061 stringsToLog.add("\nPotentially hanging thread: " + t.getName() + "\n"); -062 StackTraceElement[] stackElements = stackTraces.get(t); -063 for (StackTraceElement ele : stackElements) { -064stringsToLog.add("\t" + ele + "\n"); -065 } -066} -067 } -068} -069 } -070 return stackTraces.size(); -071} -072 -073@Override -074public int getMax() { -075 return 500; -076} -077 -078@Override -079public ListString getStringsToLog() { -080 return stringsToLog; -081} -082 } -083 -084 -085 static class OpenFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { -086@Override -087public int getVal(Phase phase) { -088 if (!JVM.isUnix()) { -089return 0; -090 } -091 JVM jvm = new JVM(); -092 return (int) jvm.getOpenFileDescriptorCount(); -093} -094 -095@Override -096public int getMax() { -097 return 1024; -098} -099 } -100 -101 static class MaxFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { -102@Override -103public int getVal(Phase phase) { -104 if (!JVM.isUnix()) { -105return 0; -106 } -107 JVM jvm = new JVM(); -108 return (int) jvm.getMaxFileDescriptorCount(); -109} -110 } -111 -112 static class SystemLoadAverageResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { -113@Override -114public int getVal(Phase phase) { -115 if (!JVM.isUnix()) { -116return 0; -117 } -118 return (int) (new JVM().getSystemLoadAverage() * 100); -119} -120 } -121 -122 static class ProcessCountResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { -123@Override -124public int getVal(Phase phase) { -125 if (!JVM.isUnix()) { -126return 0; -127 } -128 return new JVM().getNumberOfRunningProcess(); -129} -130 } -131 -132 static class AvailableMemoryMBResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { -133@Override -134public int getVal(Phase phase) { -135 if (!JVM.isUnix()) { -136return 0; -137 } -138 return (int) (new JVM().getFreeMemory() / (1024L * 1024L)); -139} -140 } -141 -142 -143 /** -144 * To be implemented by sub classes if they want to add specific ResourceAnalyzer. -145 */ -146 protected void addResourceAnalyzer(ResourceChecker rc) { -147 } -148 -149 -150 private
hbase git commit: HBASE-21179 Fix the number of actions in responseTooSlow log
Repository: hbase Updated Branches: refs/heads/branch-1.2 6679643d6 -> b760dfeb7 HBASE-21179 Fix the number of actions in responseTooSlow log Signed-off-by: Andrew Purtell Signed-off-by: Yu Li Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b760dfeb Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b760dfeb Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b760dfeb Branch: refs/heads/branch-1.2 Commit: b760dfeb75a6e8c3116fb9f18c8d723b1b4d6552 Parents: 6679643 Author: Guangxu Cheng Authored: Wed Sep 12 12:55:47 2018 +0800 Committer: Guangxu Cheng Committed: Thu Sep 13 15:14:44 2018 +0800 -- .../org/apache/hadoop/hbase/protobuf/ProtobufUtil.java| 10 -- 1 file changed, 8 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/b760dfeb/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index edd395a..24937ca 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -2762,6 +2762,13 @@ public final class ProtobufUtil { ", row=" + getStringForByteString(r.getGet().getRow()); } else if (m instanceof ClientProtos.MultiRequest) { ClientProtos.MultiRequest r = (ClientProtos.MultiRequest) m; + + // Get the number of Actions + int actionsCount = 0; + for (ClientProtos.RegionAction regionAction : r.getRegionActionList()) { +actionsCount += regionAction.getActionCount(); + } + // Get first set of Actions. ClientProtos.RegionAction actions = r.getRegionActionList().get(0); String row = actions.getActionCount() <= 0? "": @@ -2769,8 +2776,7 @@ public final class ProtobufUtil { actions.getAction(0).getGet().getRow(): actions.getAction(0).getMutation().getRow()); return "region= " + getStringForByteString(actions.getRegion().getValue()) + - ", for " + r.getRegionActionCount() + - " actions and 1st row key=" + row; + ", for " + actionsCount + " action(s) and 1st row key=" + row; } else if (m instanceof ClientProtos.MutateRequest) { ClientProtos.MutateRequest r = (ClientProtos.MutateRequest) m; return "region= " + getStringForByteString(r.getRegion().getValue()) +
[2/3] hbase git commit: HBASE-21179 Fix the number of actions in responseTooSlow log
HBASE-21179 Fix the number of actions in responseTooSlow log Signed-off-by: Andrew Purtell Signed-off-by: Yu Li Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3679b403 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3679b403 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3679b403 Branch: refs/heads/branch-1.3 Commit: 3679b403d6bc4228cddde177a9234325bde96205 Parents: 2dec020 Author: Guangxu Cheng Authored: Wed Sep 12 12:55:47 2018 +0800 Committer: Guangxu Cheng Committed: Thu Sep 13 15:11:54 2018 +0800 -- .../org/apache/hadoop/hbase/protobuf/ProtobufUtil.java| 10 -- 1 file changed, 8 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/3679b403/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index ed13617..98ee13e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -2843,6 +2843,13 @@ public final class ProtobufUtil { ", row=" + getStringForByteString(r.getGet().getRow()); } else if (m instanceof ClientProtos.MultiRequest) { ClientProtos.MultiRequest r = (ClientProtos.MultiRequest) m; + + // Get the number of Actions + int actionsCount = 0; + for (ClientProtos.RegionAction regionAction : r.getRegionActionList()) { +actionsCount += regionAction.getActionCount(); + } + // Get first set of Actions. ClientProtos.RegionAction actions = r.getRegionActionList().get(0); String row = actions.getActionCount() <= 0? "": @@ -2850,8 +2857,7 @@ public final class ProtobufUtil { actions.getAction(0).getGet().getRow(): actions.getAction(0).getMutation().getRow()); return "region= " + getStringForByteString(actions.getRegion().getValue()) + - ", for " + r.getRegionActionCount() + - " actions and 1st row key=" + row; + ", for " + actionsCount + " action(s) and 1st row key=" + row; } else if (m instanceof ClientProtos.MutateRequest) { ClientProtos.MutateRequest r = (ClientProtos.MutateRequest) m; return "region= " + getStringForByteString(r.getRegion().getValue()) +
[3/3] hbase git commit: HBASE-21179 Fix the number of actions in responseTooSlow log
HBASE-21179 Fix the number of actions in responseTooSlow log Signed-off-by: Andrew Purtell Signed-off-by: Yu Li Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/54b00559 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/54b00559 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/54b00559 Branch: refs/heads/branch-1.4 Commit: 54b005591f3a2a98614136c0dfeacbda01189411 Parents: cdb79a5 Author: Guangxu Cheng Authored: Wed Sep 12 12:55:47 2018 +0800 Committer: Guangxu Cheng Committed: Thu Sep 13 15:13:55 2018 +0800 -- .../org/apache/hadoop/hbase/protobuf/ProtobufUtil.java| 10 -- 1 file changed, 8 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/54b00559/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index a76353b..e630b98 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -2976,6 +2976,13 @@ public final class ProtobufUtil { ", row=" + getStringForByteString(r.getGet().getRow()); } else if (m instanceof ClientProtos.MultiRequest) { ClientProtos.MultiRequest r = (ClientProtos.MultiRequest) m; + + // Get the number of Actions + int actionsCount = 0; + for (ClientProtos.RegionAction regionAction : r.getRegionActionList()) { +actionsCount += regionAction.getActionCount(); + } + // Get first set of Actions. ClientProtos.RegionAction actions = r.getRegionActionList().get(0); String row = actions.getActionCount() <= 0? "": @@ -2983,8 +2990,7 @@ public final class ProtobufUtil { actions.getAction(0).getGet().getRow(): actions.getAction(0).getMutation().getRow()); return "region= " + getStringForByteString(actions.getRegion().getValue()) + - ", for " + r.getRegionActionCount() + - " actions and 1st row key=" + row; + ", for " + actionsCount + " action(s) and 1st row key=" + row; } else if (m instanceof ClientProtos.MutateRequest) { ClientProtos.MutateRequest r = (ClientProtos.MutateRequest) m; return "region= " + getStringForByteString(r.getRegion().getValue()) +
[1/3] hbase git commit: HBASE-21179 Fix the number of actions in responseTooSlow log
Repository: hbase Updated Branches: refs/heads/branch-1 7f5de94ca -> c458757f1 refs/heads/branch-1.3 2dec02022 -> 3679b403d refs/heads/branch-1.4 cdb79a543 -> 54b005591 HBASE-21179 Fix the number of actions in responseTooSlow log Signed-off-by: Andrew Purtell Signed-off-by: Yu Li Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c458757f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c458757f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c458757f Branch: refs/heads/branch-1 Commit: c458757f102e084e5ec38a3b0a78bcc08714df58 Parents: 7f5de94 Author: Guangxu Cheng Authored: Wed Sep 12 12:55:47 2018 +0800 Committer: Guangxu Cheng Committed: Thu Sep 13 15:10:19 2018 +0800 -- .../org/apache/hadoop/hbase/protobuf/ProtobufUtil.java| 10 -- 1 file changed, 8 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/c458757f/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index a76353b..e630b98 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -2976,6 +2976,13 @@ public final class ProtobufUtil { ", row=" + getStringForByteString(r.getGet().getRow()); } else if (m instanceof ClientProtos.MultiRequest) { ClientProtos.MultiRequest r = (ClientProtos.MultiRequest) m; + + // Get the number of Actions + int actionsCount = 0; + for (ClientProtos.RegionAction regionAction : r.getRegionActionList()) { +actionsCount += regionAction.getActionCount(); + } + // Get first set of Actions. ClientProtos.RegionAction actions = r.getRegionActionList().get(0); String row = actions.getActionCount() <= 0? "": @@ -2983,8 +2990,7 @@ public final class ProtobufUtil { actions.getAction(0).getGet().getRow(): actions.getAction(0).getMutation().getRow()); return "region= " + getStringForByteString(actions.getRegion().getValue()) + - ", for " + r.getRegionActionCount() + - " actions and 1st row key=" + row; + ", for " + actionsCount + " action(s) and 1st row key=" + row; } else if (m instanceof ClientProtos.MutateRequest) { ClientProtos.MutateRequest r = (ClientProtos.MutateRequest) m; return "region= " + getStringForByteString(r.getRegion().getValue()) +