[4/4] hbase git commit: HBASE-14644 Region in transition metric is broken (Huaxiang Sun)
HBASE-14644 Region in transition metric is broken (Huaxiang Sun) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/11844dca Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/11844dca Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/11844dca Branch: refs/heads/branch-1.2 Commit: 11844dcaec01301dd3a3987d1375d40d0f83f439 Parents: d2d3dcd Author: Matteo Bertozzi Authored: Fri Jun 10 21:49:10 2016 -0700 Committer: Matteo Bertozzi Committed: Fri Jun 10 21:57:21 2016 -0700 -- .../hadoop/hbase/master/AssignmentManager.java | 4 + .../org/apache/hadoop/hbase/master/HMaster.java | 25 +++- .../hbase/master/MetricsAssignmentManager.java | 4 + .../hbase/regionserver/HRegionServer.java | 4 - .../master/TestAssignmentManagerMetrics.java| 136 +++ 5 files changed, 167 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/11844dca/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index 629d307..9308af6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -330,6 +330,10 @@ public class AssignmentManager extends ZooKeeperListener { useZKForAssignment = ConfigUtil.useZKForAssignment(conf); } + MetricsAssignmentManager getAssignmentManagerMetrics() { +return this.metricsAssignmentManager; + } + /** * Add the listener to the notification list. * @param listener The AssignmentListener to register http://git-wip-us.apache.org/repos/asf/hbase/blob/11844dca/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 5fff011..b373207 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hbase.NamespaceNotFoundException; import org.apache.hadoop.hbase.PleaseHoldException; import org.apache.hadoop.hbase.ProcedureInfo; import org.apache.hadoop.hbase.RegionStateListener; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; @@ -292,6 +293,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { private RegionNormalizerChore normalizerChore; private ClusterStatusChore clusterStatusChore; private ClusterStatusPublisher clusterStatusPublisherChore = null; + private PeriodicDoMetrics periodicDoMetricsChore = null; CatalogJanitor catalogJanitorChore; private LogCleaner logCleaner; @@ -343,6 +345,19 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } } + private static class PeriodicDoMetrics extends ScheduledChore { +private final HMaster server; +public PeriodicDoMetrics(int doMetricsInterval, final HMaster server) { + super(server.getServerName() + "-DoMetricsChore", server, doMetricsInterval); + this.server = server; +} + +@Override +protected void chore() { + server.doMetrics(); +} + } + /** * Initializes the HMaster. The steps are as follows: * @@ -410,6 +425,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } } +// Do Metrics periodically +periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this); +getChoreService().scheduleChore(periodicDoMetricsChore); + // Some unit tests don't need a cluster, so no zookeeper at all if (!conf.getBoolean("hbase.testing.nocluster", false)) { activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this); @@ -535,8 +554,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { * Emit the HMaster metrics, such as region in transition metrics. * Surrounding in a try block just to be sure metrics doesn't abort HMaster. */ - @Override - protected void doMetrics() { + private void doMetrics() { try { if (assignmentManager != null) { assignmentManager.updateRegionsInTransitionMetrics(); @@ -1183,6 +1201,9 @@ public class HMaster extends HRegionServer implements Master
[2/4] hbase git commit: HBASE-14644 Region in transition metric is broken (Huaxiang Sun)
HBASE-14644 Region in transition metric is broken (Huaxiang Sun) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d1bec9d0 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d1bec9d0 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d1bec9d0 Branch: refs/heads/branch-1 Commit: d1bec9d0b8f26543ff88299e7c456d678506332d Parents: 37807b3 Author: Matteo Bertozzi Authored: Fri Jun 10 21:49:10 2016 -0700 Committer: Matteo Bertozzi Committed: Fri Jun 10 21:53:14 2016 -0700 -- .../hadoop/hbase/master/AssignmentManager.java | 4 + .../org/apache/hadoop/hbase/master/HMaster.java | 25 +++- .../hbase/master/MetricsAssignmentManager.java | 4 + .../hbase/regionserver/HRegionServer.java | 4 - .../master/TestAssignmentManagerMetrics.java| 136 +++ 5 files changed, 167 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/d1bec9d0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index 7ae3e24..11e0798 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -331,6 +331,10 @@ public class AssignmentManager extends ZooKeeperListener { useZKForAssignment = ConfigUtil.useZKForAssignment(conf); } + MetricsAssignmentManager getAssignmentManagerMetrics() { +return this.metricsAssignmentManager; + } + /** * Add the listener to the notification list. * @param listener The AssignmentListener to register http://git-wip-us.apache.org/repos/asf/hbase/blob/d1bec9d0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 4b7e181..086329e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -66,6 +66,7 @@ import org.apache.hadoop.hbase.NamespaceNotFoundException; import org.apache.hadoop.hbase.PleaseHoldException; import org.apache.hadoop.hbase.ProcedureInfo; import org.apache.hadoop.hbase.RegionStateListener; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; @@ -304,6 +305,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { private RegionNormalizerChore normalizerChore; private ClusterStatusChore clusterStatusChore; private ClusterStatusPublisher clusterStatusPublisherChore = null; + private PeriodicDoMetrics periodicDoMetricsChore = null; CatalogJanitor catalogJanitorChore; private LogCleaner logCleaner; @@ -358,6 +360,19 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } } + private static class PeriodicDoMetrics extends ScheduledChore { +private final HMaster server; +public PeriodicDoMetrics(int doMetricsInterval, final HMaster server) { + super(server.getServerName() + "-DoMetricsChore", server, doMetricsInterval); + this.server = server; +} + +@Override +protected void chore() { + server.doMetrics(); +} + } + /** * Initializes the HMaster. The steps are as follows: * @@ -425,6 +440,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } } +// Do Metrics periodically +periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this); +getChoreService().scheduleChore(periodicDoMetricsChore); + // Some unit tests don't need a cluster, so no zookeeper at all if (!conf.getBoolean("hbase.testing.nocluster", false)) { activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this); @@ -550,8 +569,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { * Emit the HMaster metrics, such as region in transition metrics. * Surrounding in a try block just to be sure metrics doesn't abort HMaster. */ - @Override - protected void doMetrics() { + private void doMetrics() { try { if (assignmentManager != null) { assignmentManager.updateRegionsInTransitionMetrics(); @@ -1209,6 +1227,9 @@ public class HMaster extends HRegionServer implements MasterSe
[1/4] hbase git commit: HBASE-14644 Region in transition metric is broken (Huaxiang Sun)
Repository: hbase Updated Branches: refs/heads/branch-1 37807b38a -> d1bec9d0b refs/heads/branch-1.2 d2d3dcdae -> 11844dcae refs/heads/branch-1.3 d8d63d671 -> dac8c2bb1 refs/heads/master bd45cf347 -> 4c885f4fd HBASE-14644 Region in transition metric is broken (Huaxiang Sun) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4c885f4f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4c885f4f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4c885f4f Branch: refs/heads/master Commit: 4c885f4fdbef1efa7f4b6f43bc024db6280cde39 Parents: bd45cf3 Author: Matteo Bertozzi Authored: Fri Jun 10 21:49:10 2016 -0700 Committer: Matteo Bertozzi Committed: Fri Jun 10 21:52:24 2016 -0700 -- .../hadoop/hbase/master/AssignmentManager.java | 4 + .../org/apache/hadoop/hbase/master/HMaster.java | 26 +++- .../hbase/master/MetricsAssignmentManager.java | 4 + .../hbase/regionserver/HRegionServer.java | 4 - .../master/TestAssignmentManagerMetrics.java| 136 +++ 5 files changed, 168 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/4c885f4f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index f16463f..49026ac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -258,6 +258,10 @@ public class AssignmentManager { this.tableLockManager = tableLockManager; } + MetricsAssignmentManager getAssignmentManagerMetrics() { +return this.metricsAssignmentManager; + } + /** * Add the listener to the notification list. * @param listener The AssignmentListener to register http://git-wip-us.apache.org/repos/asf/hbase/blob/4c885f4f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index d368ffb..efb6b6e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -73,6 +73,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.PleaseHoldException; import org.apache.hadoop.hbase.ProcedureInfo; import org.apache.hadoop.hbase.RegionStateListener; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; @@ -305,6 +306,7 @@ public class HMaster extends HRegionServer implements MasterServices { private RegionNormalizerChore normalizerChore; private ClusterStatusChore clusterStatusChore; private ClusterStatusPublisher clusterStatusPublisherChore = null; + private PeriodicDoMetrics periodicDoMetricsChore = null; CatalogJanitor catalogJanitorChore; private LogCleaner logCleaner; @@ -370,6 +372,19 @@ public class HMaster extends HRegionServer implements MasterServices { } } + private static class PeriodicDoMetrics extends ScheduledChore { +private final HMaster server; +public PeriodicDoMetrics(int doMetricsInterval, final HMaster server) { + super(server.getServerName() + "-DoMetricsChore", server, doMetricsInterval); + this.server = server; +} + +@Override +protected void chore() { + server.doMetrics(); +} + } + /** * Initializes the HMaster. The steps are as follows: * @@ -433,6 +448,10 @@ public class HMaster extends HRegionServer implements MasterServices { } } +// Do Metrics periodically +periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this); +getChoreService().scheduleChore(periodicDoMetricsChore); + // Some unit tests don't need a cluster, so no zookeeper at all if (!conf.getBoolean("hbase.testing.nocluster", false)) { activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this); @@ -563,8 +582,7 @@ public class HMaster extends HRegionServer implements MasterServices { * Emit the HMaster metrics, such as region in transition metrics. * Surrounding in a try block just to be sure metrics doesn't abort HMaster. */ - @Override - protected void doMetrics() { + private void doMetrics() { try { if (assignmentManager != null) { as
[3/4] hbase git commit: HBASE-14644 Region in transition metric is broken (Huaxiang Sun)
HBASE-14644 Region in transition metric is broken (Huaxiang Sun) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dac8c2bb Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dac8c2bb Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dac8c2bb Branch: refs/heads/branch-1.3 Commit: dac8c2bb1017b77633afbef1430b08ac82d99ce4 Parents: d8d63d6 Author: Matteo Bertozzi Authored: Fri Jun 10 21:49:10 2016 -0700 Committer: Matteo Bertozzi Committed: Fri Jun 10 21:55:37 2016 -0700 -- .../hadoop/hbase/master/AssignmentManager.java | 4 + .../org/apache/hadoop/hbase/master/HMaster.java | 25 +++- .../hbase/master/MetricsAssignmentManager.java | 4 + .../hbase/regionserver/HRegionServer.java | 4 - .../master/TestAssignmentManagerMetrics.java| 136 +++ 5 files changed, 167 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/dac8c2bb/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index 6223c65..4ed162f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -331,6 +331,10 @@ public class AssignmentManager extends ZooKeeperListener { useZKForAssignment = ConfigUtil.useZKForAssignment(conf); } + MetricsAssignmentManager getAssignmentManagerMetrics() { +return this.metricsAssignmentManager; + } + /** * Add the listener to the notification list. * @param listener The AssignmentListener to register http://git-wip-us.apache.org/repos/asf/hbase/blob/dac8c2bb/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 9323c36..f9d68f4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -66,6 +66,7 @@ import org.apache.hadoop.hbase.NamespaceNotFoundException; import org.apache.hadoop.hbase.PleaseHoldException; import org.apache.hadoop.hbase.ProcedureInfo; import org.apache.hadoop.hbase.RegionStateListener; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; @@ -304,6 +305,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { private RegionNormalizerChore normalizerChore; private ClusterStatusChore clusterStatusChore; private ClusterStatusPublisher clusterStatusPublisherChore = null; + private PeriodicDoMetrics periodicDoMetricsChore = null; CatalogJanitor catalogJanitorChore; private LogCleaner logCleaner; @@ -358,6 +360,19 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } } + private static class PeriodicDoMetrics extends ScheduledChore { +private final HMaster server; +public PeriodicDoMetrics(int doMetricsInterval, final HMaster server) { + super(server.getServerName() + "-DoMetricsChore", server, doMetricsInterval); + this.server = server; +} + +@Override +protected void chore() { + server.doMetrics(); +} + } + /** * Initializes the HMaster. The steps are as follows: * @@ -425,6 +440,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } } +// Do Metrics periodically +periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this); +getChoreService().scheduleChore(periodicDoMetricsChore); + // Some unit tests don't need a cluster, so no zookeeper at all if (!conf.getBoolean("hbase.testing.nocluster", false)) { activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this); @@ -550,8 +569,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { * Emit the HMaster metrics, such as region in transition metrics. * Surrounding in a try block just to be sure metrics doesn't abort HMaster. */ - @Override - protected void doMetrics() { + private void doMetrics() { try { if (assignmentManager != null) { assignmentManager.updateRegionsInTransitionMetrics(); @@ -1209,6 +1227,9 @@ public class HMaster extends HRegionServer implements Master
[40/50] hbase git commit: HBASE-15959 Fix flaky test TestRegionServerMetrics.testMobMetrics (Huaxiang Sun and Jingcheng Du)
HBASE-15959 Fix flaky test TestRegionServerMetrics.testMobMetrics (Huaxiang Sun and Jingcheng Du) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a7172d56 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a7172d56 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a7172d56 Branch: refs/heads/hbase-12439 Commit: a7172d5611f33dde2566a4e140aeddc675a56f3a Parents: d9463bc Author: Jonathan M Hsieh Authored: Wed Jun 8 19:07:13 2016 -0700 Committer: Jonathan M Hsieh Committed: Wed Jun 8 19:07:13 2016 -0700 -- .../hadoop/hbase/regionserver/TestRegionServerMetrics.java | 4 1 file changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/a7172d56/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java index 06db468..89a82a7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -451,6 +451,10 @@ public class TestRegionServerMetrics { admin.flush(tableName); } region.getTableDesc().getFamily(cf).setMobThreshold(0); + + // closing the region forces the compaction.discharger to archive the compacted hfiles + ((HRegion) region).close(); + // metrics are reset by the region initialization ((HRegion) region).initialize(); region.compact(true);
[43/50] hbase git commit: HBASE-15994 Allow selection of RpcSchedulers Adds logging by the RpcExecutors of their run configs Adds a FifoRpcSchedulerFactory so you can try Fifo scheduler.
HBASE-15994 Allow selection of RpcSchedulers Adds logging by the RpcExecutors of their run configs Adds a FifoRpcSchedulerFactory so you can try Fifo scheduler. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/031b7450 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/031b7450 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/031b7450 Branch: refs/heads/hbase-12439 Commit: 031b745001c7d54ef13f3cd6d725d0eb78095785 Parents: 407aa4d Author: stack Authored: Wed Jun 8 20:23:11 2016 -0700 Committer: stack Committed: Wed Jun 8 20:23:11 2016 -0700 -- .../hbase/ipc/BalancedQueueRpcExecutor.java | 4 ++ .../hadoop/hbase/ipc/FifoRpcScheduler.java | 5 ++ .../hadoop/hbase/ipc/SimpleRpcScheduler.java| 54 --- .../regionserver/FifoRpcSchedulerFactory.java | 47 + .../hbase/regionserver/RpcSchedulerFactory.java | 4 +- .../regionserver/SimpleRpcSchedulerFactory.java | 6 +- .../regionserver/TestRpcSchedulerFactory.java | 71 7 files changed, 161 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/031b7450/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java index e4205eb..3505221 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java @@ -22,6 +22,8 @@ import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HBaseInterfaceAudience; @@ -36,6 +38,7 @@ import org.apache.hadoop.hbase.util.ReflectionUtils; @InterfaceAudience.LimitedPrivate({ HBaseInterfaceAudience.COPROC, HBaseInterfaceAudience.PHOENIX }) @InterfaceStability.Evolving public class BalancedQueueRpcExecutor extends RpcExecutor { + private static final Log LOG = LogFactory.getLog(BalancedQueueRpcExecutor.class); protected final List> queues; private final QueueBalancer balancer; @@ -62,6 +65,7 @@ public class BalancedQueueRpcExecutor extends RpcExecutor { queues = new ArrayList>(numQueues); this.balancer = getBalancer(numQueues); initializeQueues(numQueues, queueClass, initargs); +LOG.debug(name + " queues=" + numQueues + " handlerCount=" + handlerCount); } protected void initializeQueues(final int numQueues, http://git-wip-us.apache.org/repos/asf/hbase/blob/031b7450/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java index ee36f3f..70d903a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.ipc; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.DaemonThreadFactory; @@ -32,6 +34,7 @@ import java.util.concurrent.atomic.AtomicInteger; * This can be used for HMaster, where no prioritization is needed. */ public class FifoRpcScheduler extends RpcScheduler { + private static final Log LOG = LogFactory.getLog(FifoRpcScheduler.class); private final int handlerCount; private final int maxQueueLength; private final AtomicInteger queueSize = new AtomicInteger(0); @@ -41,6 +44,8 @@ public class FifoRpcScheduler extends RpcScheduler { this.handlerCount = handlerCount; this.maxQueueLength = conf.getInt(RpcScheduler.IPC_SERVER_MAX_CALLQUEUE_LENGTH, handlerCount * RpcServer.DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER); +LOG.info("Using " + this.getClass().getSimpleName() + " as user call queue; handlerCount=" + +handlerCount + "; maxQueueLength=" + maxQueueLength); } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/031b7450/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/SimpleRpcScheduler.java -- diff --git a/hbase-server/src/main/java/org/ap
[48/50] hbase git commit: HBASE-15958 Implement ClaimQueues on top of HBase
HBASE-15958 Implement ClaimQueues on top of HBase Building on HBase-15883. Now implementing the claim queues procedure within an HBase table. Also added UnitTests to test claimQueue. Peer tracking will still be performed by ZooKeeper though. Also modified the queueId tracking procedure so we no longer have to perform scans over the Replication Table. This does make our queue naming schema slightly different from ReplicationQueuesZKImpl though. Signed-off-by: Elliott Clark Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/babdedc1 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/babdedc1 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/babdedc1 Branch: refs/heads/hbase-12439 Commit: babdedc1b0f0159eb526fb5c9ee08525de7ce404 Parents: 108d39a Author: Joseph Hwang Authored: Thu May 19 17:14:33 2016 -0700 Committer: Elliott Clark Committed: Thu Jun 9 15:05:54 2016 -0700 -- .../hbase/replication/ReplicationQueues.java| 8 +- .../replication/ReplicationQueuesArguments.java | 4 +- .../replication/ReplicationQueuesHBaseImpl.java | 485 --- .../replication/ReplicationQueuesZKImpl.java| 26 +- .../regionserver/ReplicationSourceManager.java | 7 +- .../replication/TestReplicationStateBasic.java | 6 +- .../TestReplicationStateHBaseImpl.java | 302 +--- .../replication/TestReplicationStateZKImpl.java | 1 - .../TestReplicationSourceManager.java | 12 +- 9 files changed, 579 insertions(+), 272 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/babdedc1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java index 809b122..0de0cc8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java @@ -19,8 +19,8 @@ package org.apache.hadoop.hbase.replication; import java.util.List; -import java.util.SortedMap; -import java.util.SortedSet; +import java.util.Map; +import java.util.Set; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -96,10 +96,10 @@ public interface ReplicationQueues { /** * Take ownership for the set of queues belonging to a dead region server. * @param regionserver the id of the dead region server - * @return A SortedMap of the queues that have been claimed, including a SortedSet of WALs in + * @return A Map of the queues that have been claimed, including a Set of WALs in * each queue. Returns an empty map if no queues were failed-over. */ - SortedMap> claimQueues(String regionserver); + Map> claimQueues(String regionserver); /** * Get a list of all region servers that have outstanding replication queues. These servers could http://git-wip-us.apache.org/repos/asf/hbase/blob/babdedc1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java index 4907b73..4fdc4e7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java @@ -56,11 +56,11 @@ public class ReplicationQueuesArguments { this.conf = conf; } - public Abortable getAbort() { + public Abortable getAbortable() { return abort; } - public void setAbort(Abortable abort) { + public void setAbortable(Abortable abort) { this.abort = abort; } } http://git-wip-us.apache.org/repos/asf/hbase/blob/babdedc1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesHBaseImpl.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesHBaseImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesHBaseImpl.java index 29f0632..34a5289 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesHBaseImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesHBaseImpl.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hbase.replication; +import or
[47/50] hbase git commit: HBASE-15925 provide default values for hadoop compat module related properties that match default hadoop profile.
HBASE-15925 provide default values for hadoop compat module related properties that match default hadoop profile. Signed-off-by: Mikhail Antonov Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/108d39a7 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/108d39a7 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/108d39a7 Branch: refs/heads/hbase-12439 Commit: 108d39a7277bdcbc8b4bc9c50bab457a86a71971 Parents: 55a04b7 Author: Sean Busbey Authored: Thu Jun 9 13:30:45 2016 -0500 Committer: Sean Busbey Committed: Thu Jun 9 16:29:28 2016 -0500 -- pom.xml | 7 +++ 1 file changed, 7 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/108d39a7/pom.xml -- diff --git a/pom.xml b/pom.xml index 6652b78..0e33ae8 100644 --- a/pom.xml +++ b/pom.xml @@ -1212,6 +1212,13 @@ 2.7.1 3.0.0-SNAPSHOT + +${hadoop-two.version} +hbase-hadoop2-compat +src/main/assembly/hadoop-two-compat.xml + 1.2 1.9
[31/50] hbase git commit: HBASE-15948 Port "HADOOP-9956 RPC listener inefficiently assigns connections to readers" Adds HADOOP-9955 RPC idle connection closing is extremely inefficient Then removes qu
HBASE-15948 Port "HADOOP-9956 RPC listener inefficiently assigns connections to readers" Adds HADOOP-9955 RPC idle connection closing is extremely inefficient Then removes queue added by HADOOP-9956 at Enis suggestion Changes how we do accounting of Connections to match how it is done in Hadoop. Adds a ConnectionManager class. Adds new configurations for this new class. "hbase.ipc.client.idlethreshold" 4000 "hbase.ipc.client.connection.idle-scan-interval.ms" 1 "hbase.ipc.client.connection.maxidletime" 1 "hbase.ipc.client.kill.max", 10 "hbase.ipc.server.handler.queue.size", 100 The new scheme does away with synchronization that purportedly would freeze out reads while we were cleaning up stale connections (according to HADOOP-9955) Also adds in new mechanism for accepting Connections by pulling in as many as we can at a time adding them to a Queue instead of doing one at a time. Can help when bursty traffic according to HADOOP-9956. Removes a blocking while Reader is busy parsing a request. Adds configuration "hbase.ipc.server.read.connection-queue.size" with default of 100 for queue size. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e0b70c00 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e0b70c00 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e0b70c00 Branch: refs/heads/hbase-12439 Commit: e0b70c00e74aeaac33570508e3732a53daea839e Parents: da88b48 Author: stack Authored: Fri Jun 3 15:38:07 2016 -0700 Committer: stack Committed: Tue Jun 7 13:10:14 2016 -0700 -- .../hbase/ipc/MetricsHBaseServerSource.java | 10 +- .../ipc/MetricsHBaseServerWrapperImpl.java | 6 +- .../org/apache/hadoop/hbase/ipc/RpcServer.java | 408 +++ .../regionserver/SimpleRpcSchedulerFactory.java | 2 +- .../hadoop/hbase/ipc/AbstractTestIPC.java | 2 +- 5 files changed, 241 insertions(+), 187 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/e0b70c00/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java -- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java index bb89789..ce57e0f 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java @@ -52,14 +52,16 @@ public interface MetricsHBaseServerSource extends BaseSource { String TOTAL_CALL_TIME_NAME = "totalCallTime"; String TOTAL_CALL_TIME_DESC = "Total call time, including both queued and processing time."; String QUEUE_SIZE_NAME = "queueSize"; - String QUEUE_SIZE_DESC = "Number of bytes in the call queues."; + String QUEUE_SIZE_DESC = "Number of bytes in the call queues; request has been read and " + +"parsed and is waiting to run or is currently being executed."; String GENERAL_QUEUE_NAME = "numCallsInGeneralQueue"; - String GENERAL_QUEUE_DESC = "Number of calls in the general call queue."; + String GENERAL_QUEUE_DESC = "Number of calls in the general call queue; " + +"parsed requests waiting in scheduler to be executed"; String PRIORITY_QUEUE_NAME = "numCallsInPriorityQueue"; String REPLICATION_QUEUE_NAME = "numCallsInReplicationQueue"; String REPLICATION_QUEUE_DESC = - "Number of calls in the replication call queue."; - String PRIORITY_QUEUE_DESC = "Number of calls in the priority call queue."; + "Number of calls in the replication call queue waiting to be run"; + String PRIORITY_QUEUE_DESC = "Number of calls in the priority call queue waiting to be run"; String NUM_OPEN_CONNECTIONS_NAME = "numOpenConnections"; String NUM_OPEN_CONNECTIONS_DESC = "Number of open connections."; String NUM_ACTIVE_HANDLER_NAME = "numActiveHandler"; http://git-wip-us.apache.org/repos/asf/hbase/blob/e0b70c00/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java index 9979c75..4f53709 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java @@ -36,7 +36,7 @@ public class MetricsHBaseServerWrapperImpl implements MetricsHBaseServerWrapper if (!isServer
[22/50] hbase git commit: HBASE-15883 Adding WAL files and tracking offsets in HBase.
HBASE-15883 Adding WAL files and tracking offsets in HBase. Implemented ReplicationQueuesHBaseImpl that tracks WAL offsets and replication queues in an HBase table. Only wrote the basic tracking methods, have not implemented claimQueue() or HFileRef methods yet. Wrote a basic unit test for ReplicationQueueHBaseImpl that tests the implemented functions on a single Region Server Signed-off-by: Elliott Clark Signed-off-by: Elliott Clark Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/21e98271 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/21e98271 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/21e98271 Branch: refs/heads/hbase-12439 Commit: 21e98271c32f0d44106515a72b2c92d518c03668 Parents: 9a53d8b Author: Joseph Hwang Authored: Thu May 19 17:14:33 2016 -0700 Committer: Elliott Clark Committed: Fri Jun 3 15:23:10 2016 -0700 -- .../hbase/replication/ReplicationFactory.java | 11 +- .../hbase/replication/ReplicationQueues.java| 8 +- .../replication/ReplicationQueuesArguments.java | 66 +++ .../replication/ReplicationQueuesHBaseImpl.java | 491 +++ .../replication/ReplicationQueuesZKImpl.java| 13 +- .../replication/regionserver/Replication.java | 12 +- .../regionserver/ReplicationSourceManager.java | 5 +- .../replication/TestReplicationAdmin.java | 3 +- .../hbase/master/cleaner/TestLogsCleaner.java | 3 +- .../cleaner/TestReplicationHFileCleaner.java| 4 +- .../replication/TestReplicationStateBasic.java | 2 +- .../TestReplicationStateHBaseImpl.java | 243 + .../replication/TestReplicationStateZKImpl.java | 13 +- .../TestReplicationSourceManager.java | 36 +- .../hadoop/hbase/util/TestHBaseFsckOneRS.java | 4 +- 15 files changed, 871 insertions(+), 43 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/21e98271/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java index 91e77ca..e264a4d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java @@ -18,6 +18,7 @@ */ package org.apache.hadoop.hbase.replication; +import org.apache.commons.lang.reflect.ConstructorUtils; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Abortable; @@ -30,9 +31,11 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; @InterfaceAudience.Private public class ReplicationFactory { - public static ReplicationQueues getReplicationQueues(final ZooKeeperWatcher zk, - Configuration conf, Abortable abortable) { -return new ReplicationQueuesZKImpl(zk, conf, abortable); + public static ReplicationQueues getReplicationQueues(ReplicationQueuesArguments args) + throws Exception { +Class classToBuild = args.getConf().getClass("hbase.region.replica." + +"replication.ReplicationQueuesType", ReplicationQueuesZKImpl.class); +return (ReplicationQueues) ConstructorUtils.invokeConstructor(classToBuild, args); } public static ReplicationQueuesClient getReplicationQueuesClient(final ZooKeeperWatcher zk, @@ -44,7 +47,7 @@ public class ReplicationFactory { Abortable abortable) { return getReplicationPeers(zk, conf, null, abortable); } - + public static ReplicationPeers getReplicationPeers(final ZooKeeperWatcher zk, Configuration conf, final ReplicationQueuesClient queuesClient, Abortable abortable) { return new ReplicationPeersZKImpl(zk, conf, queuesClient, abortable); http://git-wip-us.apache.org/repos/asf/hbase/blob/21e98271/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java index 0d47a88..db6da91 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java @@ -83,13 +83,13 @@ public interface ReplicationQueues { /** * Get a list of all WALs in the given queue. * @param queueId a String that identifies the queue - * @return a list of WALs, null if this region server is dead
[44/50] hbase git commit: HBASE-15975 logic in TestHTableDescriptor#testAddCoprocessorWithSpecStr is wrong (Huaxiang Sun)
HBASE-15975 logic in TestHTableDescriptor#testAddCoprocessorWithSpecStr is wrong (Huaxiang Sun) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/41cc2155 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/41cc2155 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/41cc2155 Branch: refs/heads/hbase-12439 Commit: 41cc215544deb5f1f6cf522efd19229da0c99dd2 Parents: 031b745 Author: Matteo Bertozzi Authored: Wed Jun 8 22:42:07 2016 -0700 Committer: Matteo Bertozzi Committed: Wed Jun 8 22:42:50 2016 -0700 -- .../hadoop/hbase/TestHTableDescriptor.java | 19 ++- 1 file changed, 10 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/41cc2155/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java -- diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java index 680f2c1..70380e6 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java @@ -55,35 +55,36 @@ public class TestHTableDescriptor { public void testAddCoprocessorWithSpecStr() throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME); String cpName = "a.b.c.d"; -boolean expected = false; try { htd.addCoprocessorWithSpec(cpName); + fail(); } catch (IllegalArgumentException iae) { - expected = true; + // Expected as cpName is invalid } -if (!expected) fail(); + // Try minimal spec. try { htd.addCoprocessorWithSpec("file:///some/path" + "|" + cpName); + fail(); } catch (IllegalArgumentException iae) { - expected = false; + // Expected to be invalid } -if (expected) fail(); + // Try more spec. String spec = "hdfs:///foo.jar|com.foo.FooRegionObserver|1001|arg1=1,arg2=2"; try { htd.addCoprocessorWithSpec(spec); } catch (IllegalArgumentException iae) { - expected = false; + fail(); } -if (expected) fail(); + // Try double add of same coprocessor try { htd.addCoprocessorWithSpec(spec); + fail(); } catch (IOException ioe) { - expected = true; + // Expect that the coprocessor already exists } -if (!expected) fail(); } @Test
[41/50] hbase git commit: HBASE-15994 Allow selection of RpcSchedulers Adds logging by the RpcExecutors of their run configs Changes the default RpcSchedulerFactory from SimpleRpcSchedulerFactory.clas
HBASE-15994 Allow selection of RpcSchedulers Adds logging by the RpcExecutors of their run configs Changes the default RpcSchedulerFactory from SimpleRpcSchedulerFactory.class to RpcSchedulerFactoryImpl.class. RpcSchedulerFactoryImpl.class. defaults to using SimpleRpcSchedulerFactory.class and the SimpleRpcScheduler, as has been default up to this, unless you set "hbase.region.server.rpc.scheduler.class" to org.apache.hadoop.hbase.ipc.FifoRpcScheduler Signed-off-by: stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3ac4a57f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3ac4a57f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3ac4a57f Branch: refs/heads/hbase-12439 Commit: 3ac4a57fd205e1909c874cabd7fda9fb176f3f0f Parents: a7172d5 Author: stack Authored: Wed Jun 8 09:53:51 2016 -0700 Committer: stack Committed: Wed Jun 8 20:20:50 2016 -0700 -- .../hbase/ipc/BalancedQueueRpcExecutor.java | 4 ++ .../hadoop/hbase/ipc/FifoRpcScheduler.java | 5 ++ .../hadoop/hbase/ipc/SimpleRpcScheduler.java| 54 --- .../regionserver/FifoRpcSchedulerFactory.java | 47 + .../hbase/regionserver/RpcSchedulerFactory.java | 4 +- .../regionserver/SimpleRpcSchedulerFactory.java | 6 +- .../regionserver/TestRpcSchedulerFactory.java | 71 7 files changed, 161 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/3ac4a57f/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java index e4205eb..3505221 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java @@ -22,6 +22,8 @@ import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HBaseInterfaceAudience; @@ -36,6 +38,7 @@ import org.apache.hadoop.hbase.util.ReflectionUtils; @InterfaceAudience.LimitedPrivate({ HBaseInterfaceAudience.COPROC, HBaseInterfaceAudience.PHOENIX }) @InterfaceStability.Evolving public class BalancedQueueRpcExecutor extends RpcExecutor { + private static final Log LOG = LogFactory.getLog(BalancedQueueRpcExecutor.class); protected final List> queues; private final QueueBalancer balancer; @@ -62,6 +65,7 @@ public class BalancedQueueRpcExecutor extends RpcExecutor { queues = new ArrayList>(numQueues); this.balancer = getBalancer(numQueues); initializeQueues(numQueues, queueClass, initargs); +LOG.debug(name + " queues=" + numQueues + " handlerCount=" + handlerCount); } protected void initializeQueues(final int numQueues, http://git-wip-us.apache.org/repos/asf/hbase/blob/3ac4a57f/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java index ee36f3f..70d903a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.ipc; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.DaemonThreadFactory; @@ -32,6 +34,7 @@ import java.util.concurrent.atomic.AtomicInteger; * This can be used for HMaster, where no prioritization is needed. */ public class FifoRpcScheduler extends RpcScheduler { + private static final Log LOG = LogFactory.getLog(FifoRpcScheduler.class); private final int handlerCount; private final int maxQueueLength; private final AtomicInteger queueSize = new AtomicInteger(0); @@ -41,6 +44,8 @@ public class FifoRpcScheduler extends RpcScheduler { this.handlerCount = handlerCount; this.maxQueueLength = conf.getInt(RpcScheduler.IPC_SERVER_MAX_CALLQUEUE_LENGTH, handlerCount * RpcServer.DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER); +LOG.info("Using " + this.getClass().getSimpleName() + " as user call queue; handlerCount=" + +handler
[35/50] hbase git commit: Revert "HBASE-15948 Port "HADOOP-9956 RPC listener inefficiently assigns connections to readers"" Revert mistaken commit... This reverts commit e0b70c00e74aeaac33570508e3732a
Revert "HBASE-15948 Port "HADOOP-9956 RPC listener inefficiently assigns connections to readers"" Revert mistaken commit... This reverts commit e0b70c00e74aeaac33570508e3732a53daea839e. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e66ecd7d Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e66ecd7d Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e66ecd7d Branch: refs/heads/hbase-12439 Commit: e66ecd7db68d6ef57084543d08f7774c82f22f45 Parents: 6d5a259 Author: stack Authored: Tue Jun 7 16:41:30 2016 -0700 Committer: stack Committed: Tue Jun 7 16:41:30 2016 -0700 -- .../hbase/ipc/MetricsHBaseServerSource.java | 10 +- .../ipc/MetricsHBaseServerWrapperImpl.java | 6 +- .../org/apache/hadoop/hbase/ipc/RpcServer.java | 408 --- .../regionserver/SimpleRpcSchedulerFactory.java | 2 +- .../hadoop/hbase/ipc/AbstractTestIPC.java | 2 +- 5 files changed, 187 insertions(+), 241 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/e66ecd7d/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java -- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java index ce57e0f..bb89789 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java @@ -52,16 +52,14 @@ public interface MetricsHBaseServerSource extends BaseSource { String TOTAL_CALL_TIME_NAME = "totalCallTime"; String TOTAL_CALL_TIME_DESC = "Total call time, including both queued and processing time."; String QUEUE_SIZE_NAME = "queueSize"; - String QUEUE_SIZE_DESC = "Number of bytes in the call queues; request has been read and " + -"parsed and is waiting to run or is currently being executed."; + String QUEUE_SIZE_DESC = "Number of bytes in the call queues."; String GENERAL_QUEUE_NAME = "numCallsInGeneralQueue"; - String GENERAL_QUEUE_DESC = "Number of calls in the general call queue; " + -"parsed requests waiting in scheduler to be executed"; + String GENERAL_QUEUE_DESC = "Number of calls in the general call queue."; String PRIORITY_QUEUE_NAME = "numCallsInPriorityQueue"; String REPLICATION_QUEUE_NAME = "numCallsInReplicationQueue"; String REPLICATION_QUEUE_DESC = - "Number of calls in the replication call queue waiting to be run"; - String PRIORITY_QUEUE_DESC = "Number of calls in the priority call queue waiting to be run"; + "Number of calls in the replication call queue."; + String PRIORITY_QUEUE_DESC = "Number of calls in the priority call queue."; String NUM_OPEN_CONNECTIONS_NAME = "numOpenConnections"; String NUM_OPEN_CONNECTIONS_DESC = "Number of open connections."; String NUM_ACTIVE_HANDLER_NAME = "numActiveHandler"; http://git-wip-us.apache.org/repos/asf/hbase/blob/e66ecd7d/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java index 4f53709..9979c75 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java @@ -36,7 +36,7 @@ public class MetricsHBaseServerWrapperImpl implements MetricsHBaseServerWrapper if (!isServerStarted()) { return 0; } -return server.callQueueSizeInBytes.get(); +return server.callQueueSize.get(); } @Override @@ -65,10 +65,10 @@ public class MetricsHBaseServerWrapperImpl implements MetricsHBaseServerWrapper @Override public int getNumOpenConnections() { -if (!isServerStarted()) { +if (!isServerStarted() || this.server.connectionList == null) { return 0; } -return server.getNumOpenConnections(); +return server.connectionList.size(); } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/e66ecd7d/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java index aca3fdd..483ce86 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
[50/50] hbase git commit: HBASE-16004 Update to Netty 4.1.1
HBASE-16004 Update to Netty 4.1.1 Signed-off-by: stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bd45cf34 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bd45cf34 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bd45cf34 Branch: refs/heads/hbase-12439 Commit: bd45cf34762332a3a51f605798a3e050e7a1e62e Parents: 6da6bab Author: Jurriaan Mous Authored: Fri Jun 10 17:57:42 2016 +0200 Committer: stack Committed: Fri Jun 10 12:14:05 2016 -0700 -- .../main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java| 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/bd45cf34/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java index c1ed748..723a234 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java @@ -347,13 +347,13 @@ public class AsyncRpcClient extends AbstractRpcClient { } // do not close global EventLoopGroup. if (!useGlobalEventLoopGroup) { - bootstrap.group().shutdownGracefully(); + bootstrap.config().group().shutdownGracefully(); } } @Override public EventLoop getEventExecutor() { -return this.bootstrap.group().next(); +return this.bootstrap.config().group().next(); } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/bd45cf34/pom.xml -- diff --git a/pom.xml b/pom.xml index 0e33ae8..93b7a0e 100644 --- a/pom.xml +++ b/pom.xml @@ -1254,7 +1254,7 @@ 4.0.3 2.4.1 1.3.3 -4.0.30.Final +4.1.1.Final 3.6.2.Final 2.1.2 1.0.8
[33/50] hbase git commit: HBASE-15981 Remove references to disabling table in docs around stripe and date-tiered compactions
HBASE-15981 Remove references to disabling table in docs around stripe and date-tiered compactions Signed-off-by: stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2da090f9 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2da090f9 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2da090f9 Branch: refs/heads/hbase-12439 Commit: 2da090f9a3ca42b07f342035c952ce0465c67265 Parents: 1125215 Author: Bryan Beaudreault Authored: Tue Jun 7 15:31:54 2016 -0400 Committer: stack Committed: Tue Jun 7 13:10:35 2016 -0700 -- src/main/asciidoc/_chapters/architecture.adoc | 6 ++ 1 file changed, 2 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/2da090f9/src/main/asciidoc/_chapters/architecture.adoc -- diff --git a/src/main/asciidoc/_chapters/architecture.adoc b/src/main/asciidoc/_chapters/architecture.adoc index faa1230..9f59cd5 100644 --- a/src/main/asciidoc/_chapters/architecture.adoc +++ b/src/main/asciidoc/_chapters/architecture.adoc @@ -2119,7 +2119,7 @@ This is not necessary on new tables. [[ops.date.tiered.config]] == Configuring Date Tiered Compaction -Each of the settings for date tiered compaction should be configured at the table or column family, after disabling the table. +Each of the settings for date tiered compaction should be configured at the table or column family level. If you use HBase shell, the general command pattern is as follows: [source,sql] @@ -2199,7 +2199,6 @@ You can enable stripe compaction for a table or a column family, by setting its You also need to set the `hbase.hstore.blockingStoreFiles` to a high number, such as 100 (rather than the default value of 10). .Procedure: Enable Stripe Compaction -. If the table already exists, disable the table. . Run one of following commands in the HBase shell. Replace the table name `orders_table` with the name of your table. + @@ -2215,7 +2214,6 @@ create 'orders_table', 'blobs_cf', CONFIGURATION => {'hbase.hstore.engine.class' . Enable the table. .Procedure: Disable Stripe Compaction -. Disable the table. . Set the `hbase.hstore.engine.class` option to either nil or `org.apache.hadoop.hbase.regionserver.DefaultStoreEngine`. Either option has the same effect. + @@ -2232,7 +2230,7 @@ This is not necessary on new tables. [[ops.stripe.config]] == Configuring Stripe Compaction -Each of the settings for stripe compaction should be configured at the table or column family, after disabling the table. +Each of the settings for stripe compaction should be configured at the table or column family level. If you use HBase shell, the general command pattern is as follows: [source,sql]
[14/50] hbase git commit: HBASE-15727 Canary Tool for Zookeeper (churro morales)
HBASE-15727 Canary Tool for Zookeeper (churro morales) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7e5d5308 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7e5d5308 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7e5d5308 Branch: refs/heads/hbase-12439 Commit: 7e5d530870f146dfdee52e5a228ad84f0aefafd7 Parents: cd25880 Author: tedyu Authored: Thu Jun 2 10:15:08 2016 -0700 Committer: tedyu Committed: Thu Jun 2 10:15:08 2016 -0700 -- .../org/apache/hadoop/hbase/tool/Canary.java| 142 ++- .../hadoop/hbase/tool/TestCanaryTool.java | 24 +++- 2 files changed, 160 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/7e5d5308/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java index ab9971d..360b0f5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java @@ -19,8 +19,14 @@ package org.apache.hadoop.hbase.tool; +import static org.apache.hadoop.hbase.HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT; +import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_ZNODE_PARENT; + +import com.google.common.collect.Lists; + import java.io.Closeable; import java.io.IOException; +import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -32,12 +38,12 @@ import java.util.Map; import java.util.Random; import java.util.Set; import java.util.TreeSet; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -78,20 +84,29 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.hadoop.hbase.util.RegionSplitter; +import org.apache.hadoop.hbase.zookeeper.EmptyWatcher; +import org.apache.hadoop.hbase.zookeeper.ZKConfig; import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.client.ConnectStringParser; +import org.apache.zookeeper.data.Stat; /** * HBase Canary Tool, that that can be used to do * "canary monitoring" of a running HBase cluster. * - * Here are two modes + * Here are three modes * 1. region mode - Foreach region tries to get one row per column family * and outputs some information about failure or latency. * * 2. regionserver mode - Foreach regionserver tries to get one row from one table * selected randomly and outputs some information about failure or latency. + * + * 3. zookeeper mode - for each zookeeper instance, selects a zNode and + * outputs some information about failure or latency. */ public final class Canary implements Tool { // Sink interface used by the canary to outputs information @@ -188,6 +203,55 @@ public final class Canary implements Tool { } } + public static class ZookeeperStdOutSink extends StdOutSink implements ExtendedSink { +@Override public void publishReadFailure(String zNode, String server) { + incReadFailureCount(); + LOG.error(String.format("Read from zNode:%s on zookeeper instance:%s", zNode, server)); +} + +@Override public void publishReadTiming(String znode, String server, long msTime) { + LOG.info(String.format("Read from zNode:%s on zookeeper instance:%s in %dms", + znode, server, msTime)); +} + } + + static class ZookeeperTask implements Callable { +private final Connection connection; +private final String host; +private String znode; +private final int timeout; +private ZookeeperStdOutSink sink; + +public ZookeeperTask(Connection connection, String host, String znode, int timeout, +ZookeeperStdOutSink sink) { + this.connection = connection; + this.host = host; + this.znode = znode; + this.timeout = timeout; + this.sink = sink; +} + +@Override public Void call() throws Exception { + ZooKeeper zooKeeper = null; + try { +zooKeeper = new ZooKeeper(host, timeout, EmptyWatcher.instance); +
[46/50] hbase git commit: HBASE-15990 The priority value of subsequent coprocessors in the Coprocessor.Priority.SYSTEM list are not incremented by one (ChiaPing Tsai)
HBASE-15990 The priority value of subsequent coprocessors in the Coprocessor.Priority.SYSTEM list are not incremented by one (ChiaPing Tsai) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/55a04b78 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/55a04b78 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/55a04b78 Branch: refs/heads/hbase-12439 Commit: 55a04b78102a3a919c6e2e86fcdf98dd1d9a24e4 Parents: 9012a0b Author: tedyu Authored: Thu Jun 9 10:26:10 2016 -0700 Committer: Ted Committed: Thu Jun 9 10:26:10 2016 -0700 -- .../hbase/coprocessor/CoprocessorHost.java | 5 ++-- .../hbase/coprocessor/TestCoprocessorHost.java | 26 ++-- 2 files changed, 21 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/55a04b78/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java index da0e8b1..e937569 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java @@ -158,9 +158,10 @@ public abstract class CoprocessorHost { implClass = cl.loadClass(className); // Add coprocessors as we go to guard against case where a coprocessor is specified twice // in the configuration -this.coprocessors.add(loadInstance(implClass, Coprocessor.PRIORITY_SYSTEM, conf)); +this.coprocessors.add(loadInstance(implClass, priority, conf)); LOG.info("System coprocessor " + className + " was loaded " + -"successfully with priority (" + priority++ + ")."); +"successfully with priority (" + priority + ")."); +++priority; } catch (Throwable t) { // We always abort if system coprocessors cannot be loaded abortServer(className, t); http://git-wip-us.apache.org/repos/asf/hbase/blob/55a04b78/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorHost.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorHost.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorHost.java index 58cd0fb..66b5c60 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorHost.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorHost.java @@ -29,6 +29,8 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.Assert; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -51,9 +53,8 @@ public class TestCoprocessorHost { return this.aborted; } } - @Test - public void testDoubleLoading() { + public void testDoubleLoadingAndPriorityValue() { final Configuration conf = HBaseConfiguration.create(); CoprocessorHost host = new CoprocessorHost(new TestAbortable()) { @@ -61,7 +62,7 @@ public class TestCoprocessorHost { @Override public CoprocessorEnvironment createEnvironment(Class implClass, - final Coprocessor instance, int priority, int sequence, Configuration conf) { + final Coprocessor instance, final int priority, int sequence, Configuration conf) { return new CoprocessorEnvironment() { final Coprocessor envInstance = instance; @@ -82,7 +83,7 @@ public class TestCoprocessorHost { @Override public int getPriority() { -return 0; +return priority; } @Override @@ -114,10 +115,19 @@ public class TestCoprocessorHost { }; final String key = "KEY"; final String coprocessor = "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver"; -// Try and load coprocessor three times. -conf.setStrings(key, coprocessor, coprocessor, coprocessor); +// Try and load a coprocessor three times +conf.setStrings(key, coprocessor, coprocessor, coprocessor, SimpleRegionObserverV2.class.getName()); host.loadSystemCoprocessors(conf, key); -// Only one coprocessor loaded -Assert.assertEquals(1, host.coprocessors.size()); +// Two coprocessors(SimpleRegionObserver and SimpleRegionObserverV2) loaded +Assert.assertEquals(2, host.coproce
[42/50] hbase git commit: Revert "HBASE-15994 Allow selection of RpcSchedulers Adds logging by the RpcExecutors of their run configs Changes the default RpcSchedulerFactory from SimpleRpcSchedulerFact
Revert "HBASE-15994 Allow selection of RpcSchedulers Adds logging by the RpcExecutors of their run configs Changes the default RpcSchedulerFactory from SimpleRpcSchedulerFactory.class to RpcSchedulerFactoryImpl.class. RpcSchedulerFactoryImpl.class. defaults to using SimpleRpcSchedulerFactory.class and the SimpleRpcScheduler, as has been default up to this, unless you set "hbase.region.server.rpc.scheduler.class" to org.apache.hadoop.hbase.ipc.FifoRpcScheduler" This reverts commit 3ac4a57fd205e1909c874cabd7fda9fb176f3f0f. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/407aa4d4 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/407aa4d4 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/407aa4d4 Branch: refs/heads/hbase-12439 Commit: 407aa4d4963eb0d4aa3e0950babc8cec31367e11 Parents: 3ac4a57 Author: stack Authored: Wed Jun 8 20:22:50 2016 -0700 Committer: stack Committed: Wed Jun 8 20:22:50 2016 -0700 -- .../hbase/ipc/BalancedQueueRpcExecutor.java | 4 -- .../hadoop/hbase/ipc/FifoRpcScheduler.java | 5 -- .../hadoop/hbase/ipc/SimpleRpcScheduler.java| 54 +++ .../regionserver/FifoRpcSchedulerFactory.java | 47 - .../hbase/regionserver/RpcSchedulerFactory.java | 4 +- .../regionserver/SimpleRpcSchedulerFactory.java | 6 +- .../regionserver/TestRpcSchedulerFactory.java | 71 7 files changed, 30 insertions(+), 161 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/407aa4d4/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java index 3505221..e4205eb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java @@ -22,8 +22,6 @@ import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HBaseInterfaceAudience; @@ -38,7 +36,6 @@ import org.apache.hadoop.hbase.util.ReflectionUtils; @InterfaceAudience.LimitedPrivate({ HBaseInterfaceAudience.COPROC, HBaseInterfaceAudience.PHOENIX }) @InterfaceStability.Evolving public class BalancedQueueRpcExecutor extends RpcExecutor { - private static final Log LOG = LogFactory.getLog(BalancedQueueRpcExecutor.class); protected final List> queues; private final QueueBalancer balancer; @@ -65,7 +62,6 @@ public class BalancedQueueRpcExecutor extends RpcExecutor { queues = new ArrayList>(numQueues); this.balancer = getBalancer(numQueues); initializeQueues(numQueues, queueClass, initargs); -LOG.debug(name + " queues=" + numQueues + " handlerCount=" + handlerCount); } protected void initializeQueues(final int numQueues, http://git-wip-us.apache.org/repos/asf/hbase/blob/407aa4d4/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java index 70d903a..ee36f3f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hbase.ipc; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.DaemonThreadFactory; @@ -34,7 +32,6 @@ import java.util.concurrent.atomic.AtomicInteger; * This can be used for HMaster, where no prioritization is needed. */ public class FifoRpcScheduler extends RpcScheduler { - private static final Log LOG = LogFactory.getLog(FifoRpcScheduler.class); private final int handlerCount; private final int maxQueueLength; private final AtomicInteger queueSize = new AtomicInteger(0); @@ -44,8 +41,6 @@ public class FifoRpcScheduler extends RpcScheduler { this.handlerCount = handlerCount; this.maxQueueLength = conf.getInt(RpcScheduler.IPC_SERVER_MAX_CALLQUEUE_LENGTH, handlerCount * RpcServer.DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER); -LOG.info("Using " + this.getClass().getSimpleName() + " as us
[19/50] hbase git commit: HBASE-15845 Changes: - Renaming hbase.rb to hbase_constants.rb because there are two hbase.rb files right now which is confusing. - Remove omnipresence of formatter object si
HBASE-15845 Changes: - Renaming hbase.rb to hbase_constants.rb because there are two hbase.rb files right now which is confusing. - Remove omnipresence of formatter object since it is kind of a use-and-throw class. Commands should create an instance, use it to format the output and discard it. - Some refactoring Change-Id: If9ea9873904e0a39d199a6aa10e23864b86a2f09 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bdb46f01 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bdb46f01 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bdb46f01 Branch: refs/heads/hbase-12439 Commit: bdb46f01b9645a0aa4e9827a7e627de76cd7983b Parents: f0c159b Author: Apekshit Authored: Mon Dec 28 14:50:50 2015 -0800 Committer: Apekshit Sharma Committed: Fri Jun 3 13:37:38 2016 -0700 -- bin/hirb.rb | 8 +- hbase-shell/src/main/ruby/hbase.rb | 109 --- hbase-shell/src/main/ruby/hbase/admin.rb| 82 ++ hbase-shell/src/main/ruby/hbase/hbase.rb| 36 +++--- hbase-shell/src/main/ruby/hbase/quotas.rb | 3 +- .../src/main/ruby/hbase/replication_admin.rb| 3 +- .../src/main/ruby/hbase/rsgroup_admin.rb| 3 +- hbase-shell/src/main/ruby/hbase/security.rb | 3 +- hbase-shell/src/main/ruby/hbase/table.rb| 2 +- hbase-shell/src/main/ruby/hbase/taskmonitor.rb | 3 +- .../src/main/ruby/hbase/visibility_labels.rb| 5 +- hbase-shell/src/main/ruby/hbase_constants.rb| 109 +++ hbase-shell/src/main/ruby/shell.rb | 25 ++--- hbase-shell/src/main/ruby/shell/commands.rb | 9 +- hbase-shell/src/test/ruby/hbase/admin_test.rb | 3 +- hbase-shell/src/test/ruby/hbase/hbase_test.rb | 11 +- .../test/ruby/hbase/replication_admin_test.rb | 3 +- .../src/test/ruby/hbase/security_admin_test.rb | 3 +- hbase-shell/src/test/ruby/hbase/table_test.rb | 2 +- .../src/test/ruby/hbase/taskmonitor_test.rb | 2 +- .../ruby/hbase/visibility_labels_admin_test.rb | 3 +- .../src/test/ruby/shell/commands_test.rb| 2 +- .../src/test/ruby/shell/noninteractive_test.rb | 6 +- .../src/test/ruby/shell/rsgroup_shell_test.rb | 18 ++- hbase-shell/src/test/ruby/shell/shell_test.rb | 6 +- hbase-shell/src/test/ruby/test_helper.rb| 10 +- 26 files changed, 218 insertions(+), 251 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/bdb46f01/bin/hirb.rb -- diff --git a/bin/hirb.rb b/bin/hirb.rb index 94b5cdb..d0295d6 100644 --- a/bin/hirb.rb +++ b/bin/hirb.rb @@ -115,7 +115,7 @@ org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(log_level) org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(log_level) # Require HBase now after setting log levels -require 'hbase' +require 'hbase_constants' # Load hbase shell require 'shell' @@ -123,15 +123,11 @@ require 'shell' # Require formatter require 'shell/formatter' -# Presume console format. -# Formatter takes an :output_stream parameter, if you don't want STDOUT. -@formatter = Shell::Formatter::Console.new - # Setup the HBase module. Create a configuration. @hbase = Hbase::Hbase.new # Setup console -@shell = Shell::Shell.new(@hbase, @formatter, interactive) +@shell = Shell::Shell.new(@hbase, interactive) @shell.debug = @shell_debug # Add commands to this namespace http://git-wip-us.apache.org/repos/asf/hbase/blob/bdb46f01/hbase-shell/src/main/ruby/hbase.rb -- diff --git a/hbase-shell/src/main/ruby/hbase.rb b/hbase-shell/src/main/ruby/hbase.rb deleted file mode 100644 index bc6f37c..000 --- a/hbase-shell/src/main/ruby/hbase.rb +++ /dev/null @@ -1,109 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# HBase ruby classes. -# Has wrapper classes for org.apache.hadoop.hbase.client.Admin -# and for org.apache.hadoop.hbase.client.T
[20/50] hbase git commit: HBASE-15949 Cleanup TestRegionServerMetrics. @Before and @After to setup/teardown tables using @Rule to set table name based on testname. Refactor out copy-pasted code fragme
HBASE-15949 Cleanup TestRegionServerMetrics. @Before and @After to setup/teardown tables using @Rule to set table name based on testname. Refactor out copy-pasted code fragments to single function. (Apekshit) Change-Id: Ic22e5027cc3952bab5ec30070ed20e98017db65a Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b557f0be Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b557f0be Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b557f0be Branch: refs/heads/hbase-12439 Commit: b557f0bec62a48753e5d01d7a47f3c9e5a6b3ee8 Parents: bdb46f0 Author: Apekshit Authored: Wed Jun 1 21:52:29 2016 -0700 Committer: Apekshit Sharma Committed: Fri Jun 3 14:27:24 2016 -0700 -- .../regionserver/TestRegionServerMetrics.java | 682 +++ 1 file changed, 256 insertions(+), 426 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/b557f0be/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java index 18796bd..06db468 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.regionserver; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; @@ -27,11 +29,17 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import org.junit.After; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.junit.rules.TestRule; import static org.junit.Assert.*; @@ -39,23 +47,35 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; - @Category({RegionServerTests.class, MediumTests.class}) public class TestRegionServerMetrics { - private static MetricsAssertHelper metricsHelper; + private static final Log LOG = LogFactory.getLog(TestRegionServerMetrics.class); + + @Rule + public TestName testName = new TestName(); + + @ClassRule + public static TestRule timeout = CategoryBasedTimeout.forClass(TestRegionServerMetrics.class); static { Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG); } + private static MetricsAssertHelper metricsHelper; private static MiniHBaseCluster cluster; private static HRegionServer rs; private static Configuration conf; private static HBaseTestingUtility TEST_UTIL; + private static Connection connection; private static MetricsRegionServer metricsRegionServer; private static MetricsRegionServerSource serverSource; private static final int NUM_SCAN_NEXT = 30; private static int numScanNext = 0; + private static byte[] cf = Bytes.toBytes("cf"); + private static byte[] row = Bytes.toBytes("row"); + private static byte[] qualifier = Bytes.toBytes("qual"); + private static byte[] val = Bytes.toBytes("val"); + private static Admin admin; @BeforeClass public static void startCluster() throws Exception { @@ -65,12 +85,16 @@ public class TestRegionServerMetrics { conf.getLong("hbase.splitlog.max.resubmit", 0); // Make the failure test faster conf.setInt("zookeeper.recovery.retry", 0); +// testMobMetrics creates few hfiles and manages compaction manually. +conf.setInt("hbase.hstore.compactionThreshold", 100); +conf.setInt("hbase.hstore.compaction.max", 100); conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1); TEST_UTIL.startMiniCluster(1, 1); cluster = TEST_UTIL.getHBaseCluster(); - cluster.waitForActiveAndReadyMaster(); +admin = TEST_UTIL.getHBaseAdmin(); +connection = TEST_UTIL.getConnection(); while (cluster.getLiveRegionServerThreads().size() < 1) { Threads.sleep(100); @@ -88,551 +112,370 @@ public class TestRegionServerMetrics { } } - @Test(timeout = 30) + TableName tableName; + Table table; + + @Before + public void beforeTestMethod() throws Exception { +metricsRegionServer.getRegionServerWrapper().forceRecompute(); +tableName = TableName.valueOf(testName.getMethodName()); +tab
[39/50] hbase git commit: HBASE-15989 Remove hbase.online.schema.update.enable
HBASE-15989 Remove hbase.online.schema.update.enable Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d9463bcc Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d9463bcc Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d9463bcc Branch: refs/heads/hbase-12439 Commit: d9463bcce0e36bfb67c82acd1d7483f63b2764b7 Parents: d5d9b7d Author: Matteo Bertozzi Authored: Wed Jun 8 13:09:31 2016 -0700 Committer: Matteo Bertozzi Committed: Wed Jun 8 13:09:31 2016 -0700 -- .../client/replication/ReplicationAdmin.java| 13 +- .../src/main/resources/hbase-default.xml| 9 + .../procedure/MasterDDLOperationHelper.java | 15 --- .../master/procedure/ModifyTableProcedure.java | 5 --- .../apache/hadoop/hbase/client/TestAdmin1.java | 42 .../apache/hadoop/hbase/client/TestAdmin2.java | 1 - .../client/TestCloneSnapshotFromClient.java | 1 - .../hbase/client/TestFromClientSide3.java | 2 - .../client/TestRestoreSnapshotFromClient.java | 1 - .../hbase/io/encoding/TestChangingEncoding.java | 1 - .../hbase/master/TestTableLockManager.java | 1 - .../regionserver/TestEncryptionKeyRotation.java | 4 +- ...sibilityLabelReplicationWithExpAsString.java | 1 - .../TestVisibilityLabelsReplication.java| 1 - ...ibilityLabelsWithDefaultVisLabelService.java | 1 - .../TestRestoreFlushSnapshotFromClient.java | 1 - .../src/main/ruby/shell/commands/alter.rb | 16 .../hadoop/hbase/client/AbstractTestShell.java | 1 - .../hbase/client/rsgroup/TestShellRSGroups.java | 1 - src/main/asciidoc/_chapters/hbase-default.adoc | 10 - 20 files changed, 12 insertions(+), 115 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/d9463bcc/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java index a2ad2e7..d062448 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java @@ -516,9 +516,9 @@ public class ReplicationAdmin implements Closeable { if (repPeers == null || repPeers.size() <= 0) { throw new IllegalArgumentException("Found no peer cluster for replication."); } - + final TableName onlyTableNameQualifier = TableName.valueOf(tableName.getQualifierAsString()); - + for (ReplicationPeer repPeer : repPeers) { Map> tableCFMap = repPeer.getTableCFs(); // TODO Currently peer TableCFs will not include namespace so we need to check only for table @@ -595,20 +595,11 @@ public class ReplicationAdmin implements Closeable { admin = this.connection.getAdmin(); HTableDescriptor htd = admin.getTableDescriptor(tableName); if (isTableRepEnabled(htd) ^ isRepEnabled) { -boolean isOnlineSchemaUpdateEnabled = -this.connection.getConfiguration() -.getBoolean("hbase.online.schema.update.enable", true); -if (!isOnlineSchemaUpdateEnabled) { - admin.disableTable(tableName); -} for (HColumnDescriptor hcd : htd.getFamilies()) { hcd.setScope(isRepEnabled ? HConstants.REPLICATION_SCOPE_GLOBAL : HConstants.REPLICATION_SCOPE_LOCAL); } admin.modifyTable(tableName, htd); -if (!isOnlineSchemaUpdateEnabled) { - admin.enableTable(tableName); -} } } finally { if (admin != null) { http://git-wip-us.apache.org/repos/asf/hbase/blob/d9463bcc/hbase-common/src/main/resources/hbase-default.xml -- diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 62a6b62..55ac497 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -562,7 +562,7 @@ possible configurations would overwhelm and obscure the important. hbase.regions.slop 0.001 -Rebalance if any regionserver has average + (average * slop) regions. +Rebalance if any regionserver has average + (average * slop) regions. The default value of this parameter is 0.001 in StochasticLoadBalancer (the default load balancer), while the default is 0.2 in other load balancers (i.e., SimpleLoadBalancer). @@ -865,7 +865,7 @@ possible configurations would overwhelm and obscure the important.
[34/50] hbase git commit: Revert "HBASE-15967 Metric for active ipc Readers and make default fraction of cpu count" Revert mistaken commit This reverts commit 1125215aad3f5b149f3458ba7019c5920f6dca66.
Revert "HBASE-15967 Metric for active ipc Readers and make default fraction of cpu count" Revert mistaken commit This reverts commit 1125215aad3f5b149f3458ba7019c5920f6dca66. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6d5a2593 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6d5a2593 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6d5a2593 Branch: refs/heads/hbase-12439 Commit: 6d5a25935e5ce983e14eff576a699ed1948566d2 Parents: 2da090f Author: stack Authored: Tue Jun 7 16:41:01 2016 -0700 Committer: stack Committed: Tue Jun 7 16:41:01 2016 -0700 -- .../hbase/ipc/MetricsHBaseServerSource.java | 11 +++--- .../hbase/ipc/MetricsHBaseServerSourceImpl.java | 19 .../org/apache/hadoop/hbase/ipc/RpcServer.java | 23 +--- 3 files changed, 8 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5a2593/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java -- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java index 43515cd..ce57e0f 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java @@ -86,13 +86,6 @@ public interface MetricsHBaseServerSource extends BaseSource { String EXCEPTIONS_MULTI_TOO_LARGE_DESC = "A response to a multi request was too large and the " + "rest of the requests will have to be retried."; - String RUNNING_READERS = "runningReaders"; - String RUNNING_READERS_DESCRIPTION = - "Count of Reader threads currently busy parsing requests to hand off to the scheduler"; - - void incrRunningReaders(); - void decrRunningReaders(); - void authorizationSuccess(); void authorizationFailure(); @@ -129,4 +122,6 @@ public interface MetricsHBaseServerSource extends BaseSource { void processedCall(int processingTime); void queuedAndProcessedCall(int totalTime); -} \ No newline at end of file + + +} http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5a2593/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java -- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java index 24cc0fb..c72641d 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java @@ -57,12 +57,6 @@ public class MetricsHBaseServerSourceImpl extends BaseSourceImpl private MetricHistogram requestSize; private MetricHistogram responseSize; - /** - * The count of readers currently working parsing a request as opposed to being blocked on the - * selector waiting on requests to come in. - */ - private final MutableFastCounter runningReaders; - public MetricsHBaseServerSourceImpl(String metricsName, String metricsDescription, String metricsContext, @@ -92,9 +86,6 @@ public class MetricsHBaseServerSourceImpl extends BaseSourceImpl this.exceptionsMultiTooLarge = this.getMetricsRegistry() .newCounter(EXCEPTIONS_MULTI_TOO_LARGE_NAME, EXCEPTIONS_MULTI_TOO_LARGE_DESC, 0L); -this.runningReaders = this.getMetricsRegistry() -.newCounter(RUNNING_READERS, RUNNING_READERS_DESCRIPTION, 0L); - this.authenticationSuccesses = this.getMetricsRegistry().newCounter( AUTHENTICATION_SUCCESSES_NAME, AUTHENTICATION_SUCCESSES_DESC, 0L); this.authenticationFailures = this.getMetricsRegistry().newCounter(AUTHENTICATION_FAILURES_NAME, @@ -118,16 +109,6 @@ public class MetricsHBaseServerSourceImpl extends BaseSourceImpl } @Override - public void incrRunningReaders() { -this.runningReaders.incr(+1); - } - - @Override - public void decrRunningReaders() { -this.runningReaders.incr(-1); - } - - @Override public void authorizationSuccess() { authorizationSuccesses.incr(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5a2593/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.j
[15/50] hbase git commit: HBASE-15881 Allow BZIP2 compression.
HBASE-15881 Allow BZIP2 compression. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fc890a2e Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fc890a2e Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fc890a2e Branch: refs/heads/hbase-12439 Commit: fc890a2ecb12c3c664ad00f8b2f8788dba04c71d Parents: 7e5d530 Author: Lars Hofhansl Authored: Thu Jun 2 12:19:02 2016 -0700 Committer: Lars Hofhansl Committed: Thu Jun 2 12:19:02 2016 -0700 -- .../hadoop/hbase/io/compress/Compression.java | 29 +++- .../hadoop/hbase/util/TestCompressionTest.java | 2 ++ 2 files changed, 30 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/fc890a2e/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java -- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java index 821b21f..6dc4190 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java @@ -235,7 +235,34 @@ public final class Compression { throw new RuntimeException(e); } } - }; +}, +BZIP2("bzip2") { + // Use base type to avoid compile-time dependencies. + private volatile transient CompressionCodec bzipCodec; + private transient Object lock = new Object(); + + @Override + CompressionCodec getCodec(Configuration conf) { +if (bzipCodec == null) { + synchronized (lock) { +if (bzipCodec == null) { + bzipCodec = buildCodec(conf); +} + } +} +return bzipCodec; + } + + private CompressionCodec buildCodec(Configuration conf) { +try { + Class externalCodec = + getClassLoaderForCodec().loadClass("org.apache.hadoop.io.compress.BZip2Codec"); + return (CompressionCodec) ReflectionUtils.newInstance(externalCodec, conf); +} catch (ClassNotFoundException e) { + throw new RuntimeException(e); +} + } +}; private final Configuration conf; private final String compressName; http://git-wip-us.apache.org/repos/asf/hbase/blob/fc890a2e/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java index 43c7cfc..398f3f0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java @@ -75,12 +75,14 @@ public class TestCompressionTest { nativeCodecTest("LZO", "lzo2", "com.hadoop.compression.lzo.LzoCodec"); nativeCodecTest("LZ4", null, "org.apache.hadoop.io.compress.Lz4Codec"); nativeCodecTest("SNAPPY", "snappy", "org.apache.hadoop.io.compress.SnappyCodec"); + nativeCodecTest("BZIP2", "bzip2", "org.apache.hadoop.io.compress.BZip2Codec"); } else { // Hadoop nativelib is not available LOG.debug("Native code not loaded"); assertFalse(CompressionTest.testCompression("LZO")); assertFalse(CompressionTest.testCompression("LZ4")); assertFalse(CompressionTest.testCompression("SNAPPY")); + assertFalse(CompressionTest.testCompression("BZIP2")); } }
[23/50] hbase git commit: HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active cleanup scan to finish (Stephen Yuan Jiang)
HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active cleanup scan to finish (Stephen Yuan Jiang) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/de1b5ff7 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/de1b5ff7 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/de1b5ff7 Branch: refs/heads/hbase-12439 Commit: de1b5ff7761b899691085e147c885e15897a562c Parents: 21e9827 Author: Stephen Yuan Jiang Authored: Fri Jun 3 15:48:48 2016 -0700 Committer: Stephen Yuan Jiang Committed: Fri Jun 3 15:49:23 2016 -0700 -- .../org/apache/hadoop/hbase/master/CatalogJanitor.java | 13 - 1 file changed, 12 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/de1b5ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java index b9abc65..c93b307 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.PairOfSameType; +import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.Triple; /** @@ -86,7 +87,17 @@ public class CatalogJanitor extends ScheduledChore { * @param enabled */ public boolean setEnabled(final boolean enabled) { -return this.enabled.getAndSet(enabled); +boolean alreadyEnabled = this.enabled.getAndSet(enabled); +// If disabling is requested on an already enabled chore, we could have an active +// scan still going on, callers might not be aware of that and do further action thinkng +// that no action would be from this chore. In this case, the right action is to wait for +// the active scan to complete before exiting this function. +if (!enabled && alreadyEnabled) { + while (alreadyRunning.get()) { +Threads.sleepWithoutInterrupt(100); + } +} +return alreadyEnabled; } boolean getEnabled() {
[32/50] hbase git commit: HBASE-15967 Metric for active ipc Readers and make default fraction of cpu count Add new metric hbase.regionserver.ipc.runningReaders Also make it so Reader count is a factor
HBASE-15967 Metric for active ipc Readers and make default fraction of cpu count Add new metric hbase.regionserver.ipc.runningReaders Also make it so Reader count is a factor of processor count Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1125215a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1125215a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1125215a Branch: refs/heads/hbase-12439 Commit: 1125215aad3f5b149f3458ba7019c5920f6dca66 Parents: e0b70c0 Author: stack Authored: Sun Jun 5 11:12:05 2016 -0700 Committer: stack Committed: Tue Jun 7 13:10:14 2016 -0700 -- .../hbase/ipc/MetricsHBaseServerSource.java | 11 +++--- .../hbase/ipc/MetricsHBaseServerSourceImpl.java | 19 .../org/apache/hadoop/hbase/ipc/RpcServer.java | 23 +++- 3 files changed, 45 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/1125215a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java -- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java index ce57e0f..43515cd 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java @@ -86,6 +86,13 @@ public interface MetricsHBaseServerSource extends BaseSource { String EXCEPTIONS_MULTI_TOO_LARGE_DESC = "A response to a multi request was too large and the " + "rest of the requests will have to be retried."; + String RUNNING_READERS = "runningReaders"; + String RUNNING_READERS_DESCRIPTION = + "Count of Reader threads currently busy parsing requests to hand off to the scheduler"; + + void incrRunningReaders(); + void decrRunningReaders(); + void authorizationSuccess(); void authorizationFailure(); @@ -122,6 +129,4 @@ public interface MetricsHBaseServerSource extends BaseSource { void processedCall(int processingTime); void queuedAndProcessedCall(int totalTime); - - -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/1125215a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java -- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java index c72641d..24cc0fb 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java @@ -57,6 +57,12 @@ public class MetricsHBaseServerSourceImpl extends BaseSourceImpl private MetricHistogram requestSize; private MetricHistogram responseSize; + /** + * The count of readers currently working parsing a request as opposed to being blocked on the + * selector waiting on requests to come in. + */ + private final MutableFastCounter runningReaders; + public MetricsHBaseServerSourceImpl(String metricsName, String metricsDescription, String metricsContext, @@ -86,6 +92,9 @@ public class MetricsHBaseServerSourceImpl extends BaseSourceImpl this.exceptionsMultiTooLarge = this.getMetricsRegistry() .newCounter(EXCEPTIONS_MULTI_TOO_LARGE_NAME, EXCEPTIONS_MULTI_TOO_LARGE_DESC, 0L); +this.runningReaders = this.getMetricsRegistry() +.newCounter(RUNNING_READERS, RUNNING_READERS_DESCRIPTION, 0L); + this.authenticationSuccesses = this.getMetricsRegistry().newCounter( AUTHENTICATION_SUCCESSES_NAME, AUTHENTICATION_SUCCESSES_DESC, 0L); this.authenticationFailures = this.getMetricsRegistry().newCounter(AUTHENTICATION_FAILURES_NAME, @@ -109,6 +118,16 @@ public class MetricsHBaseServerSourceImpl extends BaseSourceImpl } @Override + public void incrRunningReaders() { +this.runningReaders.incr(+1); + } + + @Override + public void decrRunningReaders() { +this.runningReaders.incr(-1); + } + + @Override public void authorizationSuccess() { authorizationSuccesses.incr(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/1125215a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hba
[49/50] hbase git commit: HBASE-15946. Eliminate possible security concerns in Store File metrics.
HBASE-15946. Eliminate possible security concerns in Store File metrics. Invoking 'hbase hfile' inside a servlet raises several concerns. This patch avoids invoking a separate process, and also adds validation that the file being read is at least inside the HBase root directory. Signed-off-by: Mikhail Antonov Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6da6babe Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6da6babe Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6da6babe Branch: refs/heads/hbase-12439 Commit: 6da6babe4faa7b2b16775d3cd5c861e71ef4cf31 Parents: babdedc Author: Sean Mackrory Authored: Tue May 31 10:28:27 2016 -0600 Committer: Mikhail Antonov Committed: Thu Jun 9 16:08:19 2016 -0700 -- .../hbase/io/hfile/HFilePrettyPrinter.java | 108 --- .../hbase-webapps/regionserver/storeFile.jsp| 35 +++--- 2 files changed, 83 insertions(+), 60 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/6da6babe/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java index e9e21fe..36067e5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java @@ -1,4 +1,3 @@ - /* * * Licensed to the Apache Software Foundation (ASF) under one @@ -115,6 +114,8 @@ public class HFilePrettyPrinter extends Configured implements Tool { private Map> mobFileLocations; private static final int FOUND_MOB_FILES_CACHE_CAPACITY = 50; private static final int MISSING_MOB_FILES_CACHE_CAPACITY = 20; + private PrintStream out = System.out; + private PrintStream err = System.err; /** * The row which the user wants to specify and print all the KeyValues for. @@ -161,6 +162,11 @@ public class HFilePrettyPrinter extends Configured implements Tool { options.addOptionGroup(files); } + public void setPrintStreams(PrintStream out, PrintStream err) { +this.out = out; +this.err = err; + } + public boolean parseOptions(String args[]) throws ParseException, IOException { if (args.length == 0) { @@ -192,7 +198,7 @@ public class HFilePrettyPrinter extends Configured implements Tool { row = Bytes.toBytesBinary(key); isSeekToRow = true; } else { -System.err.println("Invalid row is specified."); +err.println("Invalid row is specified."); System.exit(-1); } } @@ -206,17 +212,17 @@ public class HFilePrettyPrinter extends Configured implements Tool { String enc = HRegionInfo.encodeRegionName(rn); Path regionDir = new Path(tableDir, enc); if (verbose) -System.out.println("region dir -> " + regionDir); +out.println("region dir -> " + regionDir); List regionFiles = HFile.getStoreFiles(FileSystem.get(getConf()), regionDir); if (verbose) -System.out.println("Number of region files found -> " +out.println("Number of region files found -> " + regionFiles.size()); if (verbose) { int i = 1; for (Path p : regionFiles) { if (verbose) -System.out.println("Found file[" + i++ + "] -> " + p); +out.println("Found file[" + i++ + "] -> " + p); } } files.addAll(regionFiles); @@ -255,27 +261,46 @@ public class HFilePrettyPrinter extends Configured implements Tool { // iterate over all files found for (Path fileName : files) { try { -processFile(fileName); +int exitCode = processFile(fileName); +if (exitCode != 0) { + return exitCode; +} } catch (IOException ex) { LOG.error("Error reading " + fileName, ex); -System.exit(-2); +return -2; } } if (verbose || printKey) { - System.out.println("Scanned kv count -> " + count); + out.println("Scanned kv count -> " + count); } return 0; } - private void processFile(Path file) throws IOException { + public int processFile(Path file) throws IOException { if (verbose) - System.out.println("Scanning -> " + file); + out.println("Scanning -> " + file); + +Path rootPath = FSUtils.getRootDir(getConf()); +String rootString = rootPath + rootPath.SEPARATOR; +if (!file.toString().startsWith(rootString)) { + // First we see if fully-qualified URI matches the root dir. It might + // also be an absolute path
[24/50] hbase git commit: HBASE-15849 Simplify the way we handle runtime of commands. Functions format_simple_command and format_and_return_simple_command are used to print runtimes right now. They ar
HBASE-15849 Simplify the way we handle runtime of commands. Functions format_simple_command and format_and_return_simple_command are used to print runtimes right now. They are called from within every single command and use Ruby's 'yield' magic. Instead, we can simplify it using 'command_safe' function. Since command_safe wraps all commands, we can simply time before and after we call individual command. If a command only wants to time a part of its logic, it can set instance variables start_time and end_time accordingly which is far more simpler to understand and work with than 'yield'. Change-Id: Ibfacf3593175af22fc4f7d80896dd2f6d7c5dde3 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/70762faa Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/70762faa Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/70762faa Branch: refs/heads/hbase-12439 Commit: 70762faa98e6ad26f10fa8bd9ebc1824fdcb507c Parents: de1b5ff Author: Apekshit Sharma Authored: Fri Jun 3 14:13:45 2016 -0700 Committer: Apekshit Sharma Committed: Fri Jun 3 15:50:03 2016 -0700 -- hbase-shell/src/main/ruby/shell.rb | 7 ++--- hbase-shell/src/main/ruby/shell/commands.rb | 27 +++- .../main/ruby/shell/commands/abort_procedure.rb | 6 + .../src/main/ruby/shell/commands/add_labels.rb | 4 +-- .../src/main/ruby/shell/commands/add_peer.rb| 4 +-- .../src/main/ruby/shell/commands/alter.rb | 4 +-- .../src/main/ruby/shell/commands/alter_async.rb | 4 +-- .../main/ruby/shell/commands/alter_namespace.rb | 4 +-- .../src/main/ruby/shell/commands/append.rb | 10 .../ruby/shell/commands/append_peer_tableCFs.rb | 4 +-- .../src/main/ruby/shell/commands/assign.rb | 4 +-- .../main/ruby/shell/commands/balance_switch.rb | 6 + .../src/main/ruby/shell/commands/balancer.rb| 14 +- .../ruby/shell/commands/balancer_enabled.rb | 6 + .../shell/commands/catalogjanitor_enabled.rb| 6 + .../ruby/shell/commands/catalogjanitor_run.rb | 4 +-- .../shell/commands/catalogjanitor_switch.rb | 6 + .../src/main/ruby/shell/commands/clear_auths.rb | 4 +-- .../main/ruby/shell/commands/clone_snapshot.rb | 4 +-- .../main/ruby/shell/commands/close_region.rb| 4 +-- .../src/main/ruby/shell/commands/compact.rb | 4 +-- .../src/main/ruby/shell/commands/compact_rs.rb | 4 +-- .../src/main/ruby/shell/commands/count.rb | 4 +-- .../src/main/ruby/shell/commands/create.rb | 7 +++-- .../ruby/shell/commands/create_namespace.rb | 4 +-- .../src/main/ruby/shell/commands/delete.rb | 13 +- .../ruby/shell/commands/delete_all_snapshot.rb | 6 ++--- .../main/ruby/shell/commands/delete_snapshot.rb | 4 +-- .../shell/commands/delete_table_snapshots.rb| 19 +++--- .../src/main/ruby/shell/commands/deleteall.rb | 5 ++-- .../src/main/ruby/shell/commands/describe.rb| 4 +-- .../ruby/shell/commands/describe_namespace.rb | 3 --- .../src/main/ruby/shell/commands/disable.rb | 4 +-- .../main/ruby/shell/commands/disable_peer.rb| 4 +-- .../shell/commands/disable_table_replication.rb | 6 ++--- .../src/main/ruby/shell/commands/drop.rb| 4 +-- .../main/ruby/shell/commands/drop_namespace.rb | 4 +-- .../src/main/ruby/shell/commands/enable.rb | 4 +-- .../src/main/ruby/shell/commands/enable_peer.rb | 4 +-- .../shell/commands/enable_table_replication.rb | 6 ++--- .../src/main/ruby/shell/commands/exists.rb | 4 +-- .../src/main/ruby/shell/commands/flush.rb | 4 +-- hbase-shell/src/main/ruby/shell/commands/get.rb | 4 +-- .../src/main/ruby/shell/commands/get_auths.rb | 8 +++--- .../main/ruby/shell/commands/get_peer_config.rb | 7 +++-- .../src/main/ruby/shell/commands/get_table.rb | 4 +-- .../src/main/ruby/shell/commands/grant.rb | 9 +++ .../src/main/ruby/shell/commands/incr.rb| 12 - .../src/main/ruby/shell/commands/is_disabled.rb | 8 ++ .../src/main/ruby/shell/commands/is_enabled.rb | 6 + .../src/main/ruby/shell/commands/list.rb| 3 +-- .../src/main/ruby/shell/commands/list_labels.rb | 8 +++--- .../main/ruby/shell/commands/list_namespace.rb | 3 +-- .../shell/commands/list_namespace_tables.rb | 3 +-- .../ruby/shell/commands/list_peer_configs.rb| 18 ++--- .../src/main/ruby/shell/commands/list_peers.rb | 3 +-- .../main/ruby/shell/commands/list_procedures.rb | 3 +-- .../src/main/ruby/shell/commands/list_quotas.rb | 3 +-- .../shell/commands/list_replicated_tables.rb| 4 +-- .../main/ruby/shell/commands/list_snapshots.rb | 3 +-- .../ruby/shell/commands/list_table_snapshots.rb | 3 +-- .../main/ruby/shell/commands/locate_region.rb | 4 +-- .../main/ruby/shell/commands/major_compact.rb | 4 +-- .../main/ruby/shell/com
[45/50] hbase git commit: HBASE-15952 Bulk load data replication is not working when RS user does not have permission on hfile-refs node
HBASE-15952 Bulk load data replication is not working when RS user does not have permission on hfile-refs node Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9012a0b1 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9012a0b1 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9012a0b1 Branch: refs/heads/hbase-12439 Commit: 9012a0b123b3eea8b08c8687cef812e83e9b491d Parents: 41cc215 Author: Ashish Singhi Authored: Thu Jun 9 18:44:29 2016 +0530 Committer: Ashish Singhi Committed: Thu Jun 9 18:44:29 2016 +0530 -- .../replication/ReplicationPeersZKImpl.java | 21 - .../hbase/replication/ReplicationQueues.java| 6 .../replication/ReplicationQueuesHBaseImpl.java | 6 .../replication/ReplicationQueuesZKImpl.java| 33 .../regionserver/ReplicationSourceManager.java | 11 +-- .../cleaner/TestReplicationHFileCleaner.java| 1 + .../replication/TestReplicationStateBasic.java | 5 +++ 7 files changed, 53 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/9012a0b1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java index 15265d9..5af97c2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java @@ -129,17 +129,6 @@ public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements Re ZKUtil.createWithParents(this.zookeeper, this.peersZNode); - // Irrespective of bulk load hfile replication is enabled or not we add peerId node to - // hfile-refs node -- HBASE-15397 - try { -String peerId = ZKUtil.joinZNode(this.hfileRefsZNode, id); -LOG.info("Adding peer " + peerId + " to hfile reference queue."); -ZKUtil.createWithParents(this.zookeeper, peerId); - } catch (KeeperException e) { -throw new ReplicationException("Failed to add peer with id=" + id -+ ", node under hfile references node.", e); - } - List listOfOps = new ArrayList(); ZKUtilOp op1 = ZKUtilOp.createAndFailSilent(getPeerNode(id), ReplicationSerDeHelper.toByteArray(peerConfig)); @@ -166,16 +155,6 @@ public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements Re + " because that id does not exist."); } ZKUtil.deleteNodeRecursively(this.zookeeper, ZKUtil.joinZNode(this.peersZNode, id)); - // Delete peerId node from hfile-refs node irrespective of whether bulk loaded hfile - // replication is enabled or not - - String peerId = ZKUtil.joinZNode(this.hfileRefsZNode, id); - try { -LOG.info("Removing peer " + peerId + " from hfile reference queue."); -ZKUtil.deleteNodeRecursively(this.zookeeper, peerId); - } catch (NoNodeException e) { -LOG.info("Did not find node " + peerId + " to delete.", e); - } } catch (KeeperException e) { throw new ReplicationException("Could not remove peer with id=" + id, e); } http://git-wip-us.apache.org/repos/asf/hbase/blob/9012a0b1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java index db6da91..809b122 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java @@ -123,6 +123,12 @@ public interface ReplicationQueues { void addPeerToHFileRefs(String peerId) throws ReplicationException; /** + * Remove a peer from hfile reference queue. + * @param peerId peer cluster id to be removed + */ + void removePeerFromHFileRefs(String peerId); + + /** * Add new hfile references to the queue. * @param peerId peer cluster id to which the hfiles need to be replicated * @param files list of hfile references to be added http://git-wip-us.apache.org/repos/asf/hbase/blob/9012a0b1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesHBaseImpl.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/re
[37/50] hbase git commit: HBASE-15600 Add provision for adding mutations to memstore or able to write to same region in batchMutate coprocessor hooks (Rajeshbabu and Enis)
HBASE-15600 Add provision for adding mutations to memstore or able to write to same region in batchMutate coprocessor hooks (Rajeshbabu and Enis) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d05a3722 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d05a3722 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d05a3722 Branch: refs/heads/hbase-12439 Commit: d05a3722c8347363eb04a3e5457d13ae5d0d6de6 Parents: 3a95552 Author: Enis Soztutar Authored: Tue Jun 7 20:12:51 2016 -0700 Committer: Enis Soztutar Committed: Tue Jun 7 20:12:51 2016 -0700 -- .../hadoop/hbase/regionserver/HRegion.java | 76 - .../MiniBatchOperationInProgress.java | 29 +- .../regionserver/MultiRowMutationProcessor.java | 2 +- ...erverForAddingMutationsFromCoprocessors.java | 282 +++ 4 files changed, 370 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/d05a3722/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 6522fde..9c966cd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -3134,6 +3134,35 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi batchOp.retCodeDetails, batchOp.walEditsFromCoprocessors, firstIndex, lastIndexExclusive); if (coprocessorHost.preBatchMutate(miniBatchOp)) { return 0L; +} else { + for (int i = firstIndex; i < lastIndexExclusive; i++) { +if (batchOp.retCodeDetails[i].getOperationStatusCode() != OperationStatusCode.NOT_RUN) { + // lastIndexExclusive was incremented above. + continue; +} +// we pass (i - firstIndex) below since the call expects a relative index +Mutation[] cpMutations = miniBatchOp.getOperationsFromCoprocessors(i - firstIndex); +if (cpMutations == null) { + continue; +} +// Else Coprocessor added more Mutations corresponding to the Mutation at this index. +for (int j = 0; j < cpMutations.length; j++) { + Mutation cpMutation = cpMutations[j]; + Map> cpFamilyMap = cpMutation.getFamilyCellMap(); + checkAndPrepareMutation(cpMutation, replay, cpFamilyMap, now); + + // Acquire row locks. If not, the whole batch will fail. + acquiredRowLocks.add(getRowLockInternal(cpMutation.getRow(), true)); + + if (cpMutation.getDurability() == Durability.SKIP_WAL) { +recordMutationWithoutWal(cpFamilyMap); + } + + // Returned mutations from coprocessor correspond to the Mutation at index i. We can + // directly add the cells from those mutations to the familyMaps of this mutation. + mergeFamilyMaps(familyMaps[i], cpFamilyMap); // will get added to the memstore later +} + } } } @@ -3310,9 +3339,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi // call the coprocessor hook to do any finalization steps // after the put is done MiniBatchOperationInProgress miniBatchOp = -new MiniBatchOperationInProgress(batchOp.getMutationsForCoprocs(), -batchOp.retCodeDetails, batchOp.walEditsFromCoprocessors, firstIndex, -lastIndexExclusive); + new MiniBatchOperationInProgress(batchOp.getMutationsForCoprocs(), + batchOp.retCodeDetails, batchOp.walEditsFromCoprocessors, firstIndex, lastIndexExclusive); coprocessorHost.postBatchMutateIndispensably(miniBatchOp, success); } @@ -3320,6 +3348,18 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi } } + private void mergeFamilyMaps(Map> familyMap, + Map> toBeMerged) { +for (Map.Entry> entry : toBeMerged.entrySet()) { + List cells = familyMap.get(entry.getKey()); + if (cells == null) { +familyMap.put(entry.getKey(), entry.getValue()); + } else { +cells.addAll(entry.getValue()); + } +} + } + private void appendCurrentNonces(final Mutation mutation, final boolean replay, final WALEdit walEdit, final long now, final long currentNonceGroup, final long currentNonce) throws IOException { @@ -3348,18 +3388,7 @@ public class HRegion implements HeapSize, Prop
[28/50] hbase git commit: HBASE-15803 ZooKeeperWatcher's constructor can leak a ZooKeeper instance with throwing ZooKeeperConnectionException when canCreateBaseZNode is true
HBASE-15803 ZooKeeperWatcher's constructor can leak a ZooKeeper instance with throwing ZooKeeperConnectionException when canCreateBaseZNode is true Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7fd3532d Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7fd3532d Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7fd3532d Branch: refs/heads/hbase-12439 Commit: 7fd3532de63d7b1885d6993c11a35c2f85e26631 Parents: 15c03fd Author: tedyu Authored: Mon Jun 6 18:35:15 2016 -0700 Committer: tedyu Committed: Mon Jun 6 18:35:15 2016 -0700 -- .../apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java | 12 +++- 1 file changed, 11 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/7fd3532d/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java index 93828eb..ff3d1c7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java @@ -171,7 +171,17 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable { this.recoverableZooKeeper = ZKUtil.connect(conf, quorum, pendingWatcher, identifier); pendingWatcher.prepare(this); if (canCreateBaseZNode) { - createBaseZNodes(); + try { +createBaseZNodes(); + } catch (ZooKeeperConnectionException zce) { +try { + this.recoverableZooKeeper.close(); +} catch (InterruptedException ie) { + LOG.debug("Encountered InterruptedException when closing " + this.recoverableZooKeeper); + Thread.currentThread().interrupt(); +} +throw zce; + } } }
[36/50] hbase git commit: HBASE-15948 Port "HADOOP-9956 RPC listener inefficiently assigns connections to readers" Adds HADOOP-9955 RPC idle connection closing is extremely inefficient
HBASE-15948 Port "HADOOP-9956 RPC listener inefficiently assigns connections to readers" Adds HADOOP-9955 RPC idle connection closing is extremely inefficient Changes how we do accounting of Connections to match how it is done in Hadoop. Adds a ConnectionManager class. Adds new configurations for this new class. "hbase.ipc.client.idlethreshold" 4000 "hbase.ipc.client.connection.idle-scan-interval.ms" 1 "hbase.ipc.client.connection.maxidletime" 1 "hbase.ipc.client.kill.max", 10 "hbase.ipc.server.handler.queue.size", 100 The new scheme does away with synchronization that purportedly would freeze out reads while we were cleaning up stale connections (according to HADOOP-9955) Also adds in new mechanism for accepting Connections by pulling in as many as we can at a time adding them to a Queue instead of doing one at a time. Can help when bursty traffic according to HADOOP-9956. Removes a blocking while Reader is busy parsing a request. Adds configuration "hbase.ipc.server.read.connection-queue.size" with default of 100 for queue size. Signed-off-by: stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3a95552c Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3a95552c Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3a95552c Branch: refs/heads/hbase-12439 Commit: 3a95552cfe6205ae845e1a7e1b5907da55b1a044 Parents: e66ecd7 Author: stack Authored: Thu Jun 2 16:59:52 2016 -0700 Committer: stack Committed: Tue Jun 7 16:42:21 2016 -0700 -- .../hbase/ipc/MetricsHBaseServerSource.java | 10 +- .../ipc/MetricsHBaseServerWrapperImpl.java | 6 +- .../org/apache/hadoop/hbase/ipc/RpcServer.java | 421 +++ .../regionserver/SimpleRpcSchedulerFactory.java | 2 +- .../hadoop/hbase/ipc/AbstractTestIPC.java | 2 +- 5 files changed, 255 insertions(+), 186 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/3a95552c/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java -- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java index bb89789..ce57e0f 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java @@ -52,14 +52,16 @@ public interface MetricsHBaseServerSource extends BaseSource { String TOTAL_CALL_TIME_NAME = "totalCallTime"; String TOTAL_CALL_TIME_DESC = "Total call time, including both queued and processing time."; String QUEUE_SIZE_NAME = "queueSize"; - String QUEUE_SIZE_DESC = "Number of bytes in the call queues."; + String QUEUE_SIZE_DESC = "Number of bytes in the call queues; request has been read and " + +"parsed and is waiting to run or is currently being executed."; String GENERAL_QUEUE_NAME = "numCallsInGeneralQueue"; - String GENERAL_QUEUE_DESC = "Number of calls in the general call queue."; + String GENERAL_QUEUE_DESC = "Number of calls in the general call queue; " + +"parsed requests waiting in scheduler to be executed"; String PRIORITY_QUEUE_NAME = "numCallsInPriorityQueue"; String REPLICATION_QUEUE_NAME = "numCallsInReplicationQueue"; String REPLICATION_QUEUE_DESC = - "Number of calls in the replication call queue."; - String PRIORITY_QUEUE_DESC = "Number of calls in the priority call queue."; + "Number of calls in the replication call queue waiting to be run"; + String PRIORITY_QUEUE_DESC = "Number of calls in the priority call queue waiting to be run"; String NUM_OPEN_CONNECTIONS_NAME = "numOpenConnections"; String NUM_OPEN_CONNECTIONS_DESC = "Number of open connections."; String NUM_ACTIVE_HANDLER_NAME = "numActiveHandler"; http://git-wip-us.apache.org/repos/asf/hbase/blob/3a95552c/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java index 9979c75..4f53709 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java @@ -36,7 +36,7 @@ public class MetricsHBaseServerWrapperImpl implements MetricsHBaseServerWrapper if (!isServerStarted()) { return 0; } -return server.callQueueSize.get(); +return server.ca
[38/50] hbase git commit: HBASE-15107 Procedure v2 - Procedure Queue with Region locks
HBASE-15107 Procedure v2 - Procedure Queue with Region locks Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d5d9b7d5 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d5d9b7d5 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d5d9b7d5 Branch: refs/heads/hbase-12439 Commit: d5d9b7d500c4e2bdf67abed462966f7bf7df Parents: d05a372 Author: Matteo Bertozzi Authored: Wed Jun 8 12:52:58 2016 -0700 Committer: Matteo Bertozzi Committed: Wed Jun 8 12:52:58 2016 -0700 -- .../hadoop/hbase/procedure2/Procedure.java | 31 +- .../hbase/procedure2/ProcedureExecutor.java | 19 +- .../procedure2/ProcedureSuspendedException.java | 39 +++ .../hbase/procedure2/SequentialProcedure.java | 2 +- .../procedure/MasterProcedureScheduler.java | 286 +-- .../procedure/TableProcedureInterface.java | 1 + .../procedure/TestMasterProcedureScheduler.java | 228 ++- 7 files changed, 580 insertions(+), 26 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/d5d9b7d5/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java -- diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java index 7e58420..ee61841 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java @@ -79,6 +79,9 @@ public abstract class Procedure implements Comparable { private int childrenLatch = 0; private long lastUpdate; + // TODO: it will be nice having pointers to allow the scheduler doing suspend/resume tricks + private boolean suspended = false; + private RemoteProcedureException exception = null; private byte[] result = null; @@ -94,7 +97,7 @@ public abstract class Procedure implements Comparable { * @throws InterruptedException the procedure will be added back to the queue and retried later */ protected abstract Procedure[] execute(TEnvironment env) -throws ProcedureYieldException, InterruptedException; +throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException; /** * The code to undo what done by the execute() code. @@ -276,6 +279,9 @@ public abstract class Procedure implements Comparable { */ protected void toStringState(StringBuilder builder) { builder.append(getState()); +if (isSuspended()) { + builder.append("|SUSPENDED"); +} } /** @@ -319,7 +325,7 @@ public abstract class Procedure implements Comparable { } public long getParentProcId() { -return parentProcId; +return parentProcId.longValue(); } public NonceKey getNonceKey() { @@ -371,6 +377,23 @@ public abstract class Procedure implements Comparable { return false; } + /** + * @return true if the procedure is in a suspended state, + * waiting for the resources required to execute the procedure will become available. + */ + public synchronized boolean isSuspended() { +return suspended; + } + + public synchronized void suspend() { +suspended = true; + } + + public synchronized void resume() { +assert isSuspended() : this + " expected suspended state, got " + state; +suspended = false; + } + public synchronized RemoteProcedureException getException() { return exception; } @@ -398,7 +421,7 @@ public abstract class Procedure implements Comparable { * @return the timeout in msec */ public int getTimeout() { -return timeout; +return timeout.intValue(); } /** @@ -494,7 +517,7 @@ public abstract class Procedure implements Comparable { */ @InterfaceAudience.Private protected Procedure[] doExecute(final TEnvironment env) - throws ProcedureYieldException, InterruptedException { + throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException { try { updateTimestamp(); return execute(env); http://git-wip-us.apache.org/repos/asf/hbase/blob/d5d9b7d5/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java -- diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java index f43b65f..9d71f65 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java @@ -505,15 +50
[18/50] hbase git commit: HBASE-15927 Remove HMaster.assignRegion()
HBASE-15927 Remove HMaster.assignRegion() Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f0c159b5 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f0c159b5 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f0c159b5 Branch: refs/heads/hbase-12439 Commit: f0c159b5fed732cdc08bd2fd924ae81677d66a30 Parents: 0cbce07 Author: Matteo Bertozzi Authored: Fri Jun 3 12:36:25 2016 -0700 Committer: Matteo Bertozzi Committed: Fri Jun 3 12:36:25 2016 -0700 -- .../org/apache/hadoop/hbase/master/HMaster.java | 4 -- .../hadoop/hbase/HBaseTestingUtility.java | 15 + .../hbase/client/TestMetaWithReplicas.java | 2 +- .../master/TestAssignmentManagerOnCluster.java | 61 .../hadoop/hbase/util/TestHBaseFsckOneRS.java | 20 +++ .../hadoop/hbase/util/TestHBaseFsckTwoRS.java | 23 +++- 6 files changed, 58 insertions(+), 67 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index f8d0003..d368ffb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -2378,10 +2378,6 @@ public class HMaster extends HRegionServer implements MasterServices { return this.initializationBeforeMetaAssignment; } - public void assignRegion(HRegionInfo hri) { -assignmentManager.assign(hri); - } - /** * Compute the average load across all region servers. * Currently, this uses a very naive computation - just uses the number of http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 7b35815..bfa14cb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -3200,6 +3200,21 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { } /** + * Uses directly the assignment manager to assign the region. + * and waits until the specified region has completed assignment. + * @param tableName the table name + * @throws IOException + * @throw InterruptedException + * @return true if the region is assigned false otherwise. + */ + public boolean assignRegion(final HRegionInfo regionInfo) + throws IOException, InterruptedException { +final AssignmentManager am = getHBaseCluster().getMaster().getAssignmentManager(); +am.assign(regionInfo); +return am.waitForAssignment(regionInfo); + } + + /** * Wait until all regions for a table in hbase:meta have a non-empty * info:server, up to a configuable timeout value (default is 60 seconds) * This means all regions have been deployed, http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java index 5e302d2..8e87ceb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java @@ -448,7 +448,7 @@ public class TestMetaWithReplicas { // create in-memory state otherwise master won't assign TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() .getRegionStates().createRegionState(h); -TEST_UTIL.getMiniHBaseCluster().getMaster().assignRegion(h); +TEST_UTIL.assignRegion(h); HBaseFsckRepair.waitUntilAssigned(TEST_UTIL.getHBaseAdmin(), h); // check that problem exists HBaseFsck hbck = doFsck(TEST_UTIL.getConfiguration(), false); http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java b/hbase-server/src/test/java/org/ap
[03/50] hbase git commit: Remove the hbasecon banner logo
Remove the hbasecon banner logo Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c80e2326 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c80e2326 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c80e2326 Branch: refs/heads/hbase-12439 Commit: c80e23264204c651ec7bd3e0fbf294ac728359e6 Parents: 5ea2f09 Author: stack Authored: Tue May 31 10:14:40 2016 -0700 Committer: stack Committed: Tue May 31 10:14:40 2016 -0700 -- src/main/site/site.xml | 5 - 1 file changed, 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/c80e2326/src/main/site/site.xml -- diff --git a/src/main/site/site.xml b/src/main/site/site.xml index 6d4de53..fb237bb 100644 --- a/src/main/site/site.xml +++ b/src/main/site/site.xml @@ -43,13 +43,8 @@ -hbasecon2016 - -images/hbasecon2016-stacked.png -http://hbasecon.com/ Apache HBase
[30/50] hbase git commit: HBASE-15957 RpcClientImpl.close never ends in some circumstances
HBASE-15957 RpcClientImpl.close never ends in some circumstances Signed-off-by: Enis Soztutar Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/da88b482 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/da88b482 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/da88b482 Branch: refs/heads/hbase-12439 Commit: da88b4824054f57fbcbc7795469ab2369a39b5ed Parents: 376ad0d Author: Sergey Soldatov Authored: Sun Jun 5 23:46:03 2016 -0700 Committer: Enis Soztutar Committed: Tue Jun 7 11:33:03 2016 -0700 -- .../apache/hadoop/hbase/ipc/RpcClientImpl.java | 5 ++- .../hbase/ipc/IntegrationTestRpcClient.java | 35 2 files changed, 31 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/da88b482/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java index d8c87e9..dc05af1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java @@ -1202,9 +1202,8 @@ public class RpcClientImpl extends AbstractRpcClient { } if (connsToClose != null) { for (Connection conn : connsToClose) { -if (conn.markClosed(new InterruptedIOException("RpcClient is closing"))) { - conn.close(); -} +conn.markClosed(new InterruptedIOException("RpcClient is closing")); +conn.close(); } } // wait until all connections are closed http://git-wip-us.apache.org/repos/asf/hbase/blob/da88b482/hbase-it/src/test/java/org/apache/hadoop/hbase/ipc/IntegrationTestRpcClient.java -- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/ipc/IntegrationTestRpcClient.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/ipc/IntegrationTestRpcClient.java index c28f3e6..6c0fbcc 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/ipc/IntegrationTestRpcClient.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/ipc/IntegrationTestRpcClient.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.ipc; +import static org.apache.hadoop.hbase.ipc.RpcClient.SPECIFIC_WRITE_THREAD; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -41,12 +42,6 @@ import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.codec.Codec; -import org.apache.hadoop.hbase.ipc.AbstractRpcClient; -import org.apache.hadoop.hbase.ipc.AsyncRpcClient; -import org.apache.hadoop.hbase.ipc.FifoRpcScheduler; -import org.apache.hadoop.hbase.ipc.RpcClientImpl; -import org.apache.hadoop.hbase.ipc.RpcScheduler; -import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.ipc.protobuf.generated.TestRpcServiceProtos; import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos.EchoRequestProto; import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos.EchoResponseProto; @@ -290,6 +285,7 @@ public class IntegrationTestRpcClient { static class SimpleClient extends Thread { AbstractRpcClient rpcClient; AtomicBoolean running = new AtomicBoolean(true); +AtomicBoolean sending = new AtomicBoolean(false); AtomicReference exception = new AtomicReference<>(null); Cluster cluster; String id; @@ -319,6 +315,7 @@ public class IntegrationTestRpcClient { if (address == null) { throw new IOException("Listener channel is closed"); } + sending.set(true); ret = (EchoResponseProto) rpcClient.callBlockingMethod(md, null, param, ret, user, address); } catch (Exception e) { @@ -340,6 +337,9 @@ public class IntegrationTestRpcClient { void stopRunning() { running.set(false); } +boolean isSending() { + return sending.get(); +} void rethrowException() throws Throwable { if (exception.get() != null) { @@ -348,6 +348,29 @@ public class IntegrationTestRpcClient { } } + /* + Test that not started connections are successfully removed from connection pool when + rpc client is closing. + */ + @Test (timeout = 3) + public void testRpcWithWriteThread() throws IOException, InterruptedException { +LOG.info("Starting test"); +Cluster cluster = new Cluster(1, 1); +cluster.startServer(); +conf.setBoolean(SPECIFIC_WRITE_THREAD, true); +
[02/50] hbase git commit: HBASE-15919 Modify docs to change from @Rule to @ClassRule. Also clarify that timeout limits are on test case level. (Apekshit)
HBASE-15919 Modify docs to change from @Rule to @ClassRule. Also clarify that timeout limits are on test case level. (Apekshit) Change-Id: Ifcd0264ea147bcb1100db74d92da95b643f4793f Signed-off-by: stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5ea2f092 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5ea2f092 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5ea2f092 Branch: refs/heads/hbase-12439 Commit: 5ea2f092332515eea48136d7d92f7b8ea72df15b Parents: 75c2360 Author: Apekshit Authored: Tue May 31 03:30:50 2016 -0700 Committer: stack Committed: Tue May 31 10:12:00 2016 -0700 -- src/main/asciidoc/_chapters/developer.adoc | 93 + 1 file changed, 33 insertions(+), 60 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/5ea2f092/src/main/asciidoc/_chapters/developer.adoc -- diff --git a/src/main/asciidoc/_chapters/developer.adoc b/src/main/asciidoc/_chapters/developer.adoc index a11a04e..74ce3df 100644 --- a/src/main/asciidoc/_chapters/developer.adoc +++ b/src/main/asciidoc/_chapters/developer.adoc @@ -864,7 +864,8 @@ Also, keep in mind that if you are running tests in the `hbase-server` module yo [[hbase.unittests]] === Unit Tests -Apache HBase unit tests are subdivided into four categories: small, medium, large, and integration with corresponding JUnit link:http://www.junit.org/node/581[categories]: `SmallTests`, `MediumTests`, `LargeTests`, `IntegrationTests`. +Apache HBase test cases are subdivided into four categories: small, medium, large, and +integration with corresponding JUnit link:http://www.junit.org/node/581[categories]: `SmallTests`, `MediumTests`, `LargeTests`, `IntegrationTests`. JUnit categories are denoted using java annotations and look like this in your unit test code. [source,java] @@ -879,10 +880,11 @@ public class TestHRegionInfo { } -The above example shows how to mark a unit test as belonging to the `small` category. -All unit tests in HBase have a categorization. +The above example shows how to mark a test case as belonging to the `small` category. +All test cases in HBase should have a categorization. -The first three categories, `small`, `medium`, and `large`, are for tests run when you type `$ mvn test`. +The first three categories, `small`, `medium`, and `large`, are for test cases which run when you +type `$ mvn test`. In other words, these three categorizations are for HBase unit tests. The `integration` category is not for unit tests, but for integration tests. These are run when you invoke `$ mvn verify`. @@ -890,22 +892,23 @@ Integration tests are described in <>. HBase uses a patched maven surefire plugin and maven profiles to implement its unit test characterizations. -Keep reading to figure which annotation of the set small, medium, and large to put on your new HBase unit test. +Keep reading to figure which annotation of the set small, medium, and large to put on your new +HBase test case. .Categorizing Tests Small Tests (((SmallTests))):: - _Small_ tests are executed in a shared JVM. - We put in this category all the tests that can be executed quickly in a shared JVM. - The maximum execution time for a small test is 15 seconds, and small tests should not use a (mini)cluster. + _Small_ test cases are executed in a shared JVM and individual test cases should run in 15 seconds + or less; i.e. a link:https://en.wikipedia.org/wiki/JUnit[junit test fixture], a java object made + up of test methods, should finish in under 15 seconds. These test cases can not use mini cluster. + These are run as part of patch pre-commit. Medium Tests (((MediumTests))):: - _Medium_ tests represent tests that must be executed before proposing a patch. - They are designed to run in less than 30 minutes altogether, and are quite stable in their results. - They are designed to last less than 50 seconds individually. - They can use a cluster, and each of them is executed in a separate JVM. + _Medium_ test cases are executed in separate JVM and individual test case should run in 50 seconds + or less. Together, they should take less than 30 minutes, and are quite stable in their results. + These test cases can use a mini cluster. These are run as part of patch pre-commit. Large Tests (((LargeTests))):: - _Large_ tests are everything else. + _Large_ test cases are everything else. They are typically large-scale tests, regression tests for specific bugs, timeout tests, performance tests. They are executed before a commit on the pre-integration machines. They can be run on the developer machine as well. @@ -1049,9 +1052,7 @@ ConnectionCount=1 (was 1) * All tests must be c
[17/50] hbase git commit: HBASE-15888 Extend HBASE-12769 for bulk load data replication
HBASE-15888 Extend HBASE-12769 for bulk load data replication Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0cbce076 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0cbce076 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0cbce076 Branch: refs/heads/hbase-12439 Commit: 0cbce07626b77d9aa75a16f5e52c19428865dce7 Parents: 72d3f2a Author: Ashish Singhi Authored: Fri Jun 3 18:42:00 2016 +0530 Committer: Ashish Singhi Committed: Fri Jun 3 18:42:00 2016 +0530 -- .../replication/ReplicationPeersZKImpl.java | 6 ++ .../hbase/util/hbck/ReplicationChecker.java | 59 ++-- 2 files changed, 61 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/0cbce076/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java index 09d2100..15265d9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java @@ -550,6 +550,12 @@ public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements Re } } } + // Check for hfile-refs queue + if (-1 != ZKUtil.checkExists(zookeeper, hfileRefsZNode) + && queuesClient.getAllPeersFromHFileRefsQueue().contains(peerId)) { +throw new ReplicationException("Undeleted queue for peerId: " + peerId ++ ", found in hfile-refs node path " + hfileRefsZNode); + } } catch (KeeperException e) { throw new ReplicationException("Could not check queues deleted with id=" + peerId, e); } http://git-wip-us.apache.org/repos/asf/hbase/blob/0cbce076/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java index 89f2557..e472558 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java @@ -51,16 +51,21 @@ import org.apache.zookeeper.KeeperException; @InterfaceAudience.Private public class ReplicationChecker { private static final Log LOG = LogFactory.getLog(ReplicationChecker.class); + private final ZooKeeperWatcher zkw; private final ErrorReporter errorReporter; private final ReplicationQueuesClient queuesClient; private final ReplicationPeers replicationPeers; private final ReplicationQueueDeletor queueDeletor; // replicator with its queueIds for removed peers private final Map> undeletedQueueIds = new HashMap<>(); - + // replicator with its undeleted queueIds for removed peers in hfile-refs queue + private Set undeletedHFileRefsQueueIds = new HashSet<>(); + private final String hfileRefsZNode; + public ReplicationChecker(Configuration conf, ZooKeeperWatcher zkw, ClusterConnection connection, ErrorReporter errorReporter) throws IOException { try { + this.zkw = zkw; this.errorReporter = errorReporter; this.queuesClient = ReplicationFactory.getReplicationQueuesClient(zkw, conf, connection); this.queuesClient.init(); @@ -71,6 +76,13 @@ public class ReplicationChecker { } catch (ReplicationException e) { throw new IOException("failed to construct ReplicationChecker", e); } + +String replicationZNodeName = conf.get("zookeeper.znode.replication", "replication"); +String replicationZNode = ZKUtil.joinZNode(this.zkw.baseZNode, replicationZNodeName); +String hfileRefsZNodeName = + conf.get(ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY, + ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT); +hfileRefsZNode = ZKUtil.joinZNode(replicationZNode, hfileRefsZNodeName); } public boolean hasUnDeletedQueues() { @@ -103,13 +115,37 @@ public class ReplicationChecker { } catch (KeeperException ke) { throw new IOException(ke); } + +checkUnDeletedHFileRefsQueues(peerIds); + } + + private void checkUnDeletedHFileRefsQueues(Set peerIds) throws IOException { +try { + if (-1 == ZKUtil.checkExists(zkw, hfileRefsZNode)) { +return; + } + List listOfPeers = this.queuesClient.getAllPeersFromHFileRe
[25/50] hbase git commit: HBASE-15174 Client Public API should not have PB objects in 2.0 (Ram)
HBASE-15174 Client Public API should not have PB objects in 2.0 (Ram) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b21c56e7 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b21c56e7 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b21c56e7 Branch: refs/heads/hbase-12439 Commit: b21c56e7958652ca6e6daf04642eb51abaf2b3d7 Parents: 70762fa Author: Ramkrishna Authored: Mon Jun 6 10:11:38 2016 +0530 Committer: Ramkrishna Committed: Mon Jun 6 10:11:38 2016 +0530 -- .../org/apache/hadoop/hbase/RegionLoad.java | 1 + .../org/apache/hadoop/hbase/ServerLoad.java | 2 + .../hadoop/hbase/client/RegionLoadStats.java| 4 +- .../hbase/TestInterfaceAudienceAnnotations.java | 152 +++ .../org/apache/hadoop/hbase/util/Triple.java| 4 + 5 files changed, 161 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/b21c56e7/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java index 5bf2ec7..b5852d4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java @@ -38,6 +38,7 @@ public class RegionLoad { protected ClusterStatusProtos.RegionLoad regionLoadPB; + @InterfaceAudience.Private public RegionLoad(ClusterStatusProtos.RegionLoad regionLoadPB) { this.regionLoadPB = regionLoadPB; } http://git-wip-us.apache.org/repos/asf/hbase/blob/b21c56e7/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java index 1ddcc20..3ea59db 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java @@ -57,6 +57,7 @@ public class ServerLoad { private long totalCompactingKVs = 0; private long currentCompactedKVs = 0; + @InterfaceAudience.Private public ServerLoad(ClusterStatusProtos.ServerLoad serverLoad) { this.serverLoad = serverLoad; for (ClusterStatusProtos.RegionLoad rl: serverLoad.getRegionLoadsList()) { @@ -81,6 +82,7 @@ public class ServerLoad { // NOTE: Function name cannot start with "get" because then an OpenDataException is thrown because // HBaseProtos.ServerLoad cannot be converted to an open data type(see HBASE-5967). /* @return the underlying ServerLoad protobuf object */ + @InterfaceAudience.Private public ClusterStatusProtos.ServerLoad obtainServerLoadPB() { return serverLoad; } http://git-wip-us.apache.org/repos/asf/hbase/blob/b21c56e7/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLoadStats.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLoadStats.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLoadStats.java index 443026f..bfdb216 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLoadStats.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLoadStats.java @@ -20,11 +20,11 @@ package org.apache.hadoop.hbase.client; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; -@InterfaceAudience.Public -@InterfaceStability.Evolving /** * POJO representing region server load */ +@InterfaceAudience.Public +@InterfaceStability.Evolving public class RegionLoadStats { int memstoreLoad; int heapOccupancy; http://git-wip-us.apache.org/repos/asf/hbase/blob/b21c56e7/hbase-client/src/test/java/org/apache/hadoop/hbase/TestInterfaceAudienceAnnotations.java -- diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestInterfaceAudienceAnnotations.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestInterfaceAudienceAnnotations.java index 0e0fbb0..426b6a7 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestInterfaceAudienceAnnotations.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestInterfaceAudienceAnnotations.java @@ -20,7 +20,11 @@ package org.apache.hadoop.hbase; import java.io.IOException; import java.lang.annotation.Annotation; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; import java.lang.reflect.Modifier; +i
[11/50] hbase git commit: HBASE-15944 Spark test flooding mvn output. Redirect test logs to file. This doesn't fix the problem fully as I still see few logs being dumped in stdout. But it cleans up ma
HBASE-15944 Spark test flooding mvn output. Redirect test logs to file. This doesn't fix the problem fully as I still see few logs being dumped in stdout. But it cleans up majority of the earlier dump. (Apekshit) Change-Id: I6893301d154078a7cfb6b9af2eedc744deafb8d7 Signed-off-by: stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4ffea771 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4ffea771 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4ffea771 Branch: refs/heads/hbase-12439 Commit: 4ffea7711acc33dc3c0a75479b510d0683c11523 Parents: cfe868d Author: Apekshit Authored: Thu Jun 2 01:43:35 2016 -0700 Committer: stack Committed: Thu Jun 2 08:44:08 2016 -0700 -- hbase-spark/src/test/resources/log4j.properties | 10 +- 1 file changed, 9 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/4ffea771/hbase-spark/src/test/resources/log4j.properties -- diff --git a/hbase-spark/src/test/resources/log4j.properties b/hbase-spark/src/test/resources/log4j.properties index 4eeeb2c..6d9415b 100644 --- a/hbase-spark/src/test/resources/log4j.properties +++ b/hbase-spark/src/test/resources/log4j.properties @@ -15,7 +15,7 @@ # limitations under the License. # Define some default values that can be overridden by system properties -hbase.root.logger=INFO,console +hbase.root.logger=INFO,FA hbase.log.dir=. hbase.log.file=hbase.log @@ -50,6 +50,14 @@ log4j.appender.console.target=System.err log4j.appender.console.layout=org.apache.log4j.PatternLayout log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n +#File Appender +log4j.appender.FA=org.apache.log4j.FileAppender +log4j.appender.FA.append=false +log4j.appender.FA.file=target/log-output.txt +log4j.appender.FA.layout=org.apache.log4j.PatternLayout +log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n +log4j.appender.FA.Threshold = INFO + # Custom Logging levels #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
[09/50] hbase git commit: HBASE-15858 Some region server group shell commands don't work
HBASE-15858 Some region server group shell commands don't work Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a0f49c98 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a0f49c98 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a0f49c98 Branch: refs/heads/hbase-12439 Commit: a0f49c988419d48f6c655f46ac78f8199c643b50 Parents: 53eb27b Author: tedyu Authored: Wed Jun 1 21:13:33 2016 -0700 Committer: tedyu Committed: Wed Jun 1 21:13:33 2016 -0700 -- hbase-shell/src/main/ruby/shell/commands/get_rsgroup.rb | 2 +- .../src/main/ruby/shell/commands/get_server_rsgroup.rb | 2 +- hbase-shell/src/test/ruby/shell/rsgroup_shell_test.rb| 11 +++ 3 files changed, 9 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/a0f49c98/hbase-shell/src/main/ruby/shell/commands/get_rsgroup.rb -- diff --git a/hbase-shell/src/main/ruby/shell/commands/get_rsgroup.rb b/hbase-shell/src/main/ruby/shell/commands/get_rsgroup.rb index 6772aa1..ce4be71 100644 --- a/hbase-shell/src/main/ruby/shell/commands/get_rsgroup.rb +++ b/hbase-shell/src/main/ruby/shell/commands/get_rsgroup.rb @@ -34,7 +34,7 @@ EOF def command(group_name) now = Time.now formatter.header(['GROUP INFORMATION']) -group_admin.get_rsgroup(group_name) do |s| +rsgroup_admin.get_rsgroup(group_name) do |s| formatter.row([s]) end formatter.footer(now) http://git-wip-us.apache.org/repos/asf/hbase/blob/a0f49c98/hbase-shell/src/main/ruby/shell/commands/get_server_rsgroup.rb -- diff --git a/hbase-shell/src/main/ruby/shell/commands/get_server_rsgroup.rb b/hbase-shell/src/main/ruby/shell/commands/get_server_rsgroup.rb index 322f6bb..a689a7c 100644 --- a/hbase-shell/src/main/ruby/shell/commands/get_server_rsgroup.rb +++ b/hbase-shell/src/main/ruby/shell/commands/get_server_rsgroup.rb @@ -31,7 +31,7 @@ EOF def command(server) now = Time.now -group_name = rsgroup_admin.getGroupOfServer(server).getName +group_name = rsgroup_admin.getRSGroupOfServer(server).getName formatter.row([group_name]) formatter.footer(now, 1) end http://git-wip-us.apache.org/repos/asf/hbase/blob/a0f49c98/hbase-shell/src/test/ruby/shell/rsgroup_shell_test.rb -- diff --git a/hbase-shell/src/test/ruby/shell/rsgroup_shell_test.rb b/hbase-shell/src/test/ruby/shell/rsgroup_shell_test.rb index d892775..1040ed8 100644 --- a/hbase-shell/src/test/ruby/shell/rsgroup_shell_test.rb +++ b/hbase-shell/src/test/ruby/shell/rsgroup_shell_test.rb @@ -49,12 +49,15 @@ module Hbase assert_not_nil(group) assert_equal(0, group.getServers.count) - hostport = - @rsgroup_admin.getRSGroupInfo('default').getServers.iterator.next.toString + hostport = @rsgroup_admin.getRSGroupInfo('default').getServers.iterator.next + @shell.command('get_rsgroup', 'default') + hostPortStr = hostport.toString + @shell.command('get_server_rsgroup', [hostPortStr]) @shell.command('move_rsgroup_servers', group_name, - [hostport]) + [hostPortStr]) assert_equal(1, @rsgroup_admin.getRSGroupInfo(group_name).getServers.count) + assert_equal(group_name, @rsgroup_admin.getRSGroupOfServer(hostport).getName) @shell.command('move_rsgroup_tables', group_name, @@ -65,7 +68,7 @@ module Hbase @hbase.rsgroup_admin(@formatter).get_rsgroup(group_name) do |line| case count when 1 - assert_equal(hostport, line) + assert_equal(hostPortStr, line) when 3 assert_equal(table_name, line) end
[27/50] hbase git commit: HBASE-15965 - Testing by executing a command will cover the exact path users will trigger, so its better then directly calling library functions in tests. Changing the tests
HBASE-15965 - Testing by executing a command will cover the exact path users will trigger, so its better then directly calling library functions in tests. Changing the tests to use @shell.command(:, args) to execute them like it's a command coming from shell. Norm change: Commands should print the output user would like to see, but in the end, should also return the relevant value. This way: - Tests can use returned value to check that functionality works - Tests can capture stdout to assert particular kind of output user should see. - We do not print the return value in interactive mode and keep the output clean. See Shell.command() function. Bugs found due to this change: - Uncovered bug in major_compact.rb with this approach. It was calling admin.majorCompact() which doesn't exist but our tests didn't catch it since they directly tested admin.major_compact() - Enabled TestReplicationShell. If it's bad, flaky infra will take care of it. Change-Id: I5d8af16bf477a79a2f526a5bf11c245b02b7d276 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/15c03fd1 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/15c03fd1 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/15c03fd1 Branch: refs/heads/hbase-12439 Commit: 15c03fd1c97c271aca6dc30feab35ec0c9f8edbe Parents: 3d7840a Author: Apekshit Sharma Authored: Mon Jun 6 13:35:06 2016 -0700 Committer: Apekshit Sharma Committed: Mon Jun 6 17:50:22 2016 -0700 -- .../replication/ReplicationPeerConfig.java | 6 +- hbase-shell/src/main/ruby/hbase/admin.rb| 23 +- hbase-shell/src/main/ruby/hbase/table.rb| 1 + hbase-shell/src/main/ruby/shell.rb | 12 +- hbase-shell/src/main/ruby/shell/commands.rb | 2 + .../main/ruby/shell/commands/balance_rsgroup.rb | 9 +- .../src/main/ruby/shell/commands/create.rb | 2 + .../src/main/ruby/shell/commands/exists.rb | 4 +- .../src/main/ruby/shell/commands/get_auths.rb | 1 + .../main/ruby/shell/commands/get_peer_config.rb | 1 + .../src/main/ruby/shell/commands/is_enabled.rb | 4 +- .../shell/commands/list_namespace_tables.rb | 1 + .../ruby/shell/commands/list_peer_configs.rb| 1 + .../src/main/ruby/shell/commands/list_peers.rb | 1 + .../main/ruby/shell/commands/locate_region.rb | 1 + .../main/ruby/shell/commands/major_compact.rb | 2 +- .../ruby/shell/commands/show_peer_tableCFs.rb | 4 +- .../src/main/ruby/shell/commands/truncate.rb| 3 +- .../ruby/shell/commands/truncate_preserve.rb| 3 +- .../hbase/client/TestReplicationShell.java | 2 +- hbase-shell/src/test/ruby/hbase/admin_test.rb | 229 +-- .../test/ruby/hbase/replication_admin_test.rb | 110 - .../ruby/hbase/visibility_labels_admin_test.rb | 20 +- hbase-shell/src/test/ruby/test_helper.rb| 23 +- 24 files changed, 254 insertions(+), 211 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/15c03fd1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java index 7799de6..1d2066c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java @@ -96,8 +96,10 @@ public class ReplicationPeerConfig { @Override public String toString() { StringBuilder builder = new StringBuilder("clusterKey=").append(clusterKey).append(","); - builder.append("replicationEndpointImpl=").append(replicationEndpointImpl).append(",") -.append("tableCFs=").append(tableCFsMap.toString()); + builder.append("replicationEndpointImpl=").append(replicationEndpointImpl).append(","); +if (tableCFsMap != null) { + builder.append("tableCFs=").append(tableCFsMap.toString()); +} return builder.toString(); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/15c03fd1/hbase-shell/src/main/ruby/hbase/admin.rb -- diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index f32376d..d66c1d6 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -458,16 +458,17 @@ module Hbase #-- # Truncates table (deletes all records by recreating the table) -def truncate(table_name_str, conf = @conf) +
[29/50] hbase git commit: HBASE-15698 Increment TimeRange not serialized to server (Ted Yu)
HBASE-15698 Increment TimeRange not serialized to server (Ted Yu) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/376ad0d9 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/376ad0d9 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/376ad0d9 Branch: refs/heads/hbase-12439 Commit: 376ad0d9868e563e4153f8725afbf00a0293f72c Parents: 7fd3532 Author: Andrew Purtell Authored: Mon Jun 6 16:59:43 2016 -0700 Committer: Andrew Purtell Committed: Mon Jun 6 22:17:00 2016 -0700 -- .../hadoop/hbase/protobuf/ProtobufUtil.java | 21 +- .../coprocessor/TestIncrementTimeRange.java | 196 +++ 2 files changed, 210 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/376ad0d9/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index 0c34a17..fecc3c2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -,6 +,16 @@ public final class ProtobufUtil { return builder.build(); } + static void setTimeRange(final MutationProto.Builder builder, final TimeRange timeRange) { +if (!timeRange.isAllTime()) { + HBaseProtos.TimeRange.Builder timeRangeBuilder = +HBaseProtos.TimeRange.newBuilder(); + timeRangeBuilder.setFrom(timeRange.getMin()); + timeRangeBuilder.setTo(timeRange.getMax()); + builder.setTimeRange(timeRangeBuilder.build()); +} + } + /** * Convert a client Increment to a protobuf Mutate. * @@ -1126,13 +1136,7 @@ public final class ProtobufUtil { builder.setNonce(nonce); } TimeRange timeRange = increment.getTimeRange(); -if (!timeRange.isAllTime()) { - HBaseProtos.TimeRange.Builder timeRangeBuilder = -HBaseProtos.TimeRange.newBuilder(); - timeRangeBuilder.setFrom(timeRange.getMin()); - timeRangeBuilder.setTo(timeRange.getMax()); - builder.setTimeRange(timeRangeBuilder.build()); -} +setTimeRange(builder, timeRange); ColumnValue.Builder columnBuilder = ColumnValue.newBuilder(); QualifierValue.Builder valueBuilder = QualifierValue.newBuilder(); for (Map.Entry> family: increment.getFamilyCellMap().entrySet()) { @@ -1253,6 +1257,9 @@ public final class ProtobufUtil { final MutationProto.Builder builder, long nonce) throws IOException { getMutationBuilderAndSetCommonFields(type, mutation, builder); builder.setAssociatedCellCount(mutation.size()); +if (mutation instanceof Increment) { + setTimeRange(builder, ((Increment)mutation).getTimeRange()); +} if (nonce != HConstants.NO_NONCE) { builder.setNonce(nonce); } http://git-wip-us.apache.org/repos/asf/hbase/blob/376ad0d9/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestIncrementTimeRange.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestIncrementTimeRange.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestIncrementTimeRange.java new file mode 100644 index 000..35ed531 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestIncrementTimeRange.java @@ -0,0 +1,196 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; + +import org.apache.hadoop.hbase.Cell; +import org.apache.had
[12/50] hbase git commit: HBASE-15938 submit-patch.py: Don't crash if there are tests with same name. Refactor: Split out flaky dashboard html template to separate file. (Apekshit)
HBASE-15938 submit-patch.py: Don't crash if there are tests with same name. Refactor: Split out flaky dashboard html template to separate file. (Apekshit) Change-Id: Ie5875bdefbf886984a57dfc85661be2ac9592a7b Signed-off-by: stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9593a9f3 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9593a9f3 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9593a9f3 Branch: refs/heads/hbase-12439 Commit: 9593a9f39663e454a24f16c907e00e19cb65b903 Parents: 4ffea77 Author: Apekshit Authored: Wed Jun 1 19:12:50 2016 -0700 Committer: stack Committed: Thu Jun 2 08:55:42 2016 -0700 -- dev-support/findHangingTests.py | 14 ++- dev-support/flaky-dashboard-template.html | 122 + dev-support/report-flakies.py | 112 +-- 3 files changed, 137 insertions(+), 111 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/9593a9f3/dev-support/findHangingTests.py -- diff --git a/dev-support/findHangingTests.py b/dev-support/findHangingTests.py index 9ef8708..28f4895 100755 --- a/dev-support/findHangingTests.py +++ b/dev-support/findHangingTests.py @@ -46,14 +46,22 @@ def get_bad_tests(console_url): result1 = re.match("^Running org.apache.hadoop.hbase.(\w*\.)*(\w*)", line) if result1: test_case = result1.group(2) -hanging_tests.add(test_case) -all_tests.add(test_case) +if test_case in all_tests: +print ("ERROR! Multiple tests with same name '{}'. Might get wrong results " + "for this test.".format(test_case)) +else: +hanging_tests.add(test_case) +all_tests.add(test_case) result2 = re.match("^Tests run:.*- in org.apache.hadoop.hbase.(\w*\.)*(\w*)", line) if result2: test_case = result2.group(2) -hanging_tests.remove(test_case) if "FAILURE!" in line: failed_tests.add(test_case) +if test_case not in hanging_tests: +print ("ERROR! No test '{}' found in hanging_tests. Might get wrong results " +"for this test.".format(test_case)) +else: +hanging_tests.remove(test_case) result3 = re.match("^\s+(\w*).*\sTestTimedOut", line) if result3: test_case = result3.group(1) http://git-wip-us.apache.org/repos/asf/hbase/blob/9593a9f3/dev-support/flaky-dashboard-template.html -- diff --git a/dev-support/flaky-dashboard-template.html b/dev-support/flaky-dashboard-template.html new file mode 100644 index 000..77dfc86 --- /dev/null +++ b/dev-support/flaky-dashboard-template.html @@ -0,0 +1,122 @@ + + + + +Apache HBase Flaky Dashboard + +table { +table-layout: fixed; +} +th { +font-size: 15px; +} +td { +font-size: 18px; +vertical-align: text-top; +overflow: hidden; +white-space: nowrap; +} +.show_hide_button { +font-size: 100%; +padding: .5em 1em; +border: 0 rgba(0,0,0,0); +border-radius: 10px; +} + + + + +https://hbase.apache.org/images/hbase_logo_with_orca_large.png";> + + + Apache HBase Flaky Tests Dashboard + + + +{% set counter = 0 %} +{% for url in results %} +{% set result = results[url] %} +{# Dedup ids since test names may duplicate across urls #} +{% set counter = counter + 1 %} +Job : {{ url |e }} +🔗 + + + +Test Name +Flakyness +Failed/Timeout/Hanging +Run Ids + +{% for test in result %} +{% set all = result[test]['all'] %} +{% set failed = result[test]['failed'] %} +{% set timeout = result[test]['timeout'] %} +{% set hanging = result[test]['hanging'] %} +{% set success = all.difference(failed).difference(hanging) %} + +{{ test |e }} +{% set flakyness = +(failed|length + hanging|length) * 100 / all|length %} +{% if flakyness == 100 %} + +{% else %} + +{% endif %} +{{ "{:.1f}% ({} / {})".format( +flakyness, failed|length + hanging|length, all|length) }} + + +{{ failed|length }} / {{ timeout|length }} / {{ hanging|length }} + + +{% set id = "details_" ~ test ~ "_" ~ counter %} + +show/hide + +
[04/50] hbase git commit: HBASE-15917 Addendum. Fix bug in report-flakies.py where hanging tests are not being added to flaky list. (Apekshit) ADDENDUM #2!
HBASE-15917 Addendum. Fix bug in report-flakies.py where hanging tests are not being added to flaky list. (Apekshit) ADDENDUM #2! Change-Id: I9c55932d0f9e65b72ec8d3ae714144536b2bfe0a Signed-off-by: stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eb64cd9d Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eb64cd9d Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eb64cd9d Branch: refs/heads/hbase-12439 Commit: eb64cd9dd13ba297539c409989c63e800cb378a1 Parents: c80e232 Author: Apekshit Authored: Tue May 31 02:29:40 2016 -0700 Committer: stack Committed: Tue May 31 10:16:40 2016 -0700 -- dev-support/report-flakies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/eb64cd9d/dev-support/report-flakies.py -- diff --git a/dev-support/report-flakies.py b/dev-support/report-flakies.py index bdc88dc..c0d16c7 100755 --- a/dev-support/report-flakies.py +++ b/dev-support/report-flakies.py @@ -161,7 +161,7 @@ for url_max_build in expanded_urls: print "" -all_bad_tests = all_timeout_tests.union(all_failed_tests) +all_bad_tests = all_hanging_tests.union(all_failed_tests) if args.mvn: includes = ",".join(all_bad_tests) with open("./includes", "w") as inc_file:
[06/50] hbase git commit: HBASE-15923 Shell rows counter test fails
HBASE-15923 Shell rows counter test fails Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/015f2ef6 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/015f2ef6 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/015f2ef6 Branch: refs/heads/hbase-12439 Commit: 015f2ef6292df52270df8845ccd244a97deb9c98 Parents: 73ec338 Author: tedyu Authored: Tue May 31 14:21:32 2016 -0700 Committer: tedyu Committed: Tue May 31 14:21:32 2016 -0700 -- hbase-shell/src/test/ruby/hbase/table_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/015f2ef6/hbase-shell/src/test/ruby/hbase/table_test.rb -- diff --git a/hbase-shell/src/test/ruby/hbase/table_test.rb b/hbase-shell/src/test/ruby/hbase/table_test.rb index d4547b7..a617bc5 100644 --- a/hbase-shell/src/test/ruby/hbase/table_test.rb +++ b/hbase-shell/src/test/ruby/hbase/table_test.rb @@ -561,7 +561,7 @@ module Hbase define_test "scan with a block should yield rows and return rows counter" do rows = {} res = @test_table._scan_internal { |row, cells| rows[row] = cells } - assert_equal(rows.keys.size, res) + assert_equal([rows.keys.size,false], res) end define_test "scan should support COLUMNS with value CONVERTER information" do
[21/50] hbase git commit: HBASE-15929 There are two tests named TestRegionServerMetrics. This causes slight issues when parsing console output for hanging tests and flaky tests analysis. Moving .../hb
HBASE-15929 There are two tests named TestRegionServerMetrics. This causes slight issues when parsing console output for hanging tests and flaky tests analysis. Moving .../hbase/TestRegionServerMetrics.java to .../hbase/regionserver/TestRegionServerReadRequestMetrics.java. (Apekshit) Change-Id: I379c15fe2c2c01bed53bddf7619d5f2a07c5640e Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9a53d8b3 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9a53d8b3 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9a53d8b3 Branch: refs/heads/hbase-12439 Commit: 9a53d8b3850f0521c3a4ad194a3b84d7f4145007 Parents: b557f0b Author: Apekshit Authored: Thu Jun 2 20:06:20 2016 -0700 Committer: Apekshit Sharma Committed: Fri Jun 3 15:13:49 2016 -0700 -- .../hadoop/hbase/TestRegionServerMetrics.java | 379 -- .../TestRegionServerReadRequestMetrics.java | 387 +++ 2 files changed, 387 insertions(+), 379 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/9a53d8b3/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java deleted file mode 100644 index 76e5842..000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java +++ /dev/null @@ -1,379 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase; - -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Append; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Increment; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.RowMutations; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.filter.BinaryComparator; -import org.apache.hadoop.hbase.filter.CompareFilter; -import org.apache.hadoop.hbase.filter.RowFilter; -import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -@Category(MediumTests.class) -public class TestRegionServerMetrics { - private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static final TableName TABLE_NAME = TableName.valueOf("test"); - private static final byte[] CF1 = "c1".getBytes(); - private static final byte[] CF2 = "c2".getBytes(); - - private static final byte[] ROW1 = "a".getBytes(); - private static final byte[] ROW2 = "b".getBytes(); - private static final byte[] ROW3 = "c".getBytes(); - private static final byte[] COL1 = "q1".getBytes(); - private static final byte[] COL2 = "q2".getBytes(); - private static final byte[] COL3 = "q3".getBytes(); - private static final byte[] VAL1 = "v1".getBytes(); - private static final byte[] VAL2 = "v2".getBytes(); - private static final byte[] VAL3 = Bytes.toBytes(0L); - - private static final int MAX_TRY = 20; - private static final int SLEEP_MS = 100; - private static final int TTL = 1; - - private static Admin admin; - private static Collection serverNames; - private static Table table; - private static List tableRegions; - - private static Map requestsMap = new HashMap<>(); - private static Map requestsMapPrev = new HashMap<>(); - - @BeforeCla
[26/50] hbase git commit: HBASE-15954 REST server should log requests with TRACE instead of DEBUG
HBASE-15954 REST server should log requests with TRACE instead of DEBUG Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3d7840a1 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3d7840a1 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3d7840a1 Branch: refs/heads/hbase-12439 Commit: 3d7840a173aab97fb72409fa8c0f161fd7ad0e8f Parents: b21c56e Author: Enis Soztutar Authored: Mon Jun 6 10:58:37 2016 -0700 Committer: Enis Soztutar Committed: Mon Jun 6 10:58:37 2016 -0700 -- .../hadoop/hbase/rest/MultiRowResource.java | 4 +- .../hbase/rest/NamespacesInstanceResource.java | 24 - .../hadoop/hbase/rest/NamespacesResource.java | 4 +- .../hbase/rest/ProtobufStreamingUtil.java | 10 ++-- .../apache/hadoop/hbase/rest/RESTServer.java| 12 +++-- .../apache/hadoop/hbase/rest/RESTServlet.java | 5 +- .../hadoop/hbase/rest/RegionsResource.java | 4 +- .../apache/hadoop/hbase/rest/RootResource.java | 4 +- .../apache/hadoop/hbase/rest/RowResource.java | 54 ++-- .../hbase/rest/ScannerInstanceResource.java | 32 +++- .../hadoop/hbase/rest/ScannerResource.java | 17 +++--- .../hadoop/hbase/rest/SchemaResource.java | 22 .../rest/StorageClusterStatusResource.java | 4 +- .../rest/StorageClusterVersionResource.java | 4 +- .../apache/hadoop/hbase/rest/TableResource.java | 26 ++ .../hadoop/hbase/rest/VersionResource.java | 10 ++-- .../apache/hadoop/hbase/rest/client/Client.java | 44 .../hadoop/hbase/rest/filter/AuthFilter.java| 4 +- .../rest/filter/RestCsrfPreventionFilter.java | 15 +++--- .../consumer/ProtobufMessageBodyConsumer.java | 6 +-- .../hadoop/hbase/util/ConnectionCache.java | 6 ++- 21 files changed, 169 insertions(+), 142 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/3d7840a1/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java -- diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java index c88ac91..8ff3ef6 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java @@ -86,7 +86,9 @@ public class MultiRowResource extends ResourceBase implements Constants { } model.addRow(rowModel); } else { - LOG.trace("The row : " + rk + " not found in the table."); + if (LOG.isTraceEnabled()) { +LOG.trace("The row : " + rk + " not found in the table."); + } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/3d7840a1/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java -- diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java index 8f64738..c832905 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java @@ -91,8 +91,8 @@ public class NamespacesInstanceResource extends ResourceBase { MIMETYPE_PROTOBUF_IETF}) public Response get(final @Context ServletContext context, final @Context UriInfo uriInfo) { -if (LOG.isDebugEnabled()) { - LOG.debug("GET " + uriInfo.getAbsolutePath()); +if (LOG.isTraceEnabled()) { + LOG.trace("GET " + uriInfo.getAbsolutePath()); } servlet.getMetrics().incrementRequests(1); @@ -135,8 +135,8 @@ public class NamespacesInstanceResource extends ResourceBase { @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, MIMETYPE_PROTOBUF_IETF}) public Response put(final NamespacesInstanceModel model, final @Context UriInfo uriInfo) { -if (LOG.isDebugEnabled()) { - LOG.debug("PUT " + uriInfo.getAbsolutePath()); +if (LOG.isTraceEnabled()) { + LOG.trace("PUT " + uriInfo.getAbsolutePath()); } servlet.getMetrics().incrementRequests(1); return processUpdate(model, true, uriInfo); @@ -151,8 +151,8 @@ public class NamespacesInstanceResource extends ResourceBase { @PUT public Response putNoBody(final byte[] message, final @Context UriInfo uriInfo, final @Context HttpHeaders headers) { -if (LOG.isDebugEnabled()) { - LOG.debug("PUT " + uriInfo.getAbsolutePath()); +if (LOG.isTraceEnabled()) { + LOG.trace("PUT " + uriInfo.getAbsolutePath());
[13/50] hbase git commit: HBASE-15933 Addendum - make merge decision when sizes of both regions are known
HBASE-15933 Addendum - make merge decision when sizes of both regions are known Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cd258800 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cd258800 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cd258800 Branch: refs/heads/hbase-12439 Commit: cd2588001cf31ad2fb2020f9e021c9b1be1b76fc Parents: 9593a9f Author: tedyu Authored: Thu Jun 2 09:30:36 2016 -0700 Committer: tedyu Committed: Thu Jun 2 09:30:36 2016 -0700 -- .../hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/cd258800/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java index d209eb7..7a54d87 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java @@ -182,7 +182,7 @@ public class SimpleRegionNormalizer implements RegionNormalizer { if (mergeEnabled) { HRegionInfo hri2 = tableRegions.get(candidateIdx+1); long regionSize2 = getRegionSize(hri2); - if (regionSize + regionSize2 < avgRegionSize) { + if (regionSize > 0 && regionSize2 > 0 && regionSize + regionSize2 < avgRegionSize) { LOG.info("Table " + table + ", small region size: " + regionSize + " plus its neighbor size: " + regionSize2 + ", less than the avg size " + avgRegionSize + ", merging them");
[07/50] hbase git commit: HBASE-15932 Shell test fails due to uninitialized constant
HBASE-15932 Shell test fails due to uninitialized constant Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cbb95cd3 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cbb95cd3 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cbb95cd3 Branch: refs/heads/hbase-12439 Commit: cbb95cd3a9bf9a9f8558560ae58f4061a73f15a8 Parents: 015f2ef Author: tedyu Authored: Wed Jun 1 10:35:00 2016 -0700 Committer: tedyu Committed: Wed Jun 1 10:35:00 2016 -0700 -- .../src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java | 2 +- hbase-shell/src/main/ruby/hbase.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/cbb95cd3/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java index 799bf0b..b75e8cd 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java @@ -64,7 +64,7 @@ public class HColumnDescriptor implements Comparable { // Version 11 -- add column family level configuration. private static final byte COLUMN_DESCRIPTOR_VERSION = (byte) 11; - private static final String IN_MEMORY_COMPACTION = "IN_MEMORY_COMPACTION"; + public static final String IN_MEMORY_COMPACTION = "IN_MEMORY_COMPACTION"; // These constants are used as FileInfo keys public static final String COMPRESSION = "COMPRESSION"; http://git-wip-us.apache.org/repos/asf/hbase/blob/cbb95cd3/hbase-shell/src/main/ruby/hbase.rb -- diff --git a/hbase-shell/src/main/ruby/hbase.rb b/hbase-shell/src/main/ruby/hbase.rb index 189fa3a..bc6f37c 100644 --- a/hbase-shell/src/main/ruby/hbase.rb +++ b/hbase-shell/src/main/ruby/hbase.rb @@ -39,7 +39,7 @@ module HBaseConstants NAME = org.apache.hadoop.hbase.HConstants::NAME VERSIONS = org.apache.hadoop.hbase.HConstants::VERSIONS IN_MEMORY = org.apache.hadoop.hbase.HConstants::IN_MEMORY - IN_MEMORY_COMPACTION = org.apache.hadoop.hbase.HConstants::IN_MEMORY_COMPACTION + IN_MEMORY_COMPACTION = org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY_COMPACTION METADATA = org.apache.hadoop.hbase.HConstants::METADATA STOPROW = "STOPROW" STARTROW = "STARTROW"
[05/50] hbase git commit: HBASE-15907 updates for HBase Shell pre-splitting docs
HBASE-15907 updates for HBase Shell pre-splitting docs (cherry picked from commit 01adec574d9ccbdd6183466cb8ee6b43935d69ca) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/73ec3385 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/73ec3385 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/73ec3385 Branch: refs/heads/hbase-12439 Commit: 73ec33856d0ee2ac1e058c6f7e1ccffa4476fbc0 Parents: eb64cd9 Author: Ronan Stokes Authored: Mon May 30 23:52:43 2016 -0700 Committer: Misty Stanley-Jones Committed: Tue May 31 13:52:46 2016 -0700 -- src/main/asciidoc/_chapters/performance.adoc | 19 ++- src/main/asciidoc/_chapters/shell.adoc | 62 +++ 2 files changed, 79 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/73ec3385/src/main/asciidoc/_chapters/performance.adoc -- diff --git a/src/main/asciidoc/_chapters/performance.adoc b/src/main/asciidoc/_chapters/performance.adoc index a0c00ae..5f27640 100644 --- a/src/main/asciidoc/_chapters/performance.adoc +++ b/src/main/asciidoc/_chapters/performance.adoc @@ -499,7 +499,7 @@ For bulk imports, this means that all clients will write to the same region unti A useful pattern to speed up the bulk import process is to pre-create empty regions. Be somewhat conservative in this, because too-many regions can actually degrade performance. -There are two different approaches to pre-creating splits. +There are two different approaches to pre-creating splits using the HBase API. The first approach is to rely on the default `Admin` strategy (which is implemented in `Bytes.split`)... [source,java] @@ -511,7 +511,7 @@ int numberOfRegions = ...; // # of regions to create admin.createTable(table, startKey, endKey, numberOfRegions); -And the other approach is to define the splits yourself... +And the other approach, using the HBase API, is to define the splits yourself... [source,java] @@ -519,8 +519,23 @@ byte[][] splits = ...; // create your own splits admin.createTable(table, splits); +You can achieve a similar effect using the HBase Shell to create tables by specifying split options. + +[source] + +# create table with specific split points +hbase>create 't1','f1',SPLITS => ['\x10\x00', '\x20\x00', '\x30\x00', '\x40\x00'] + +# create table with four regions based on random bytes keys +hbase>create 't2','f1', { NUMREGIONS => 4 , SPLITALGO => 'UniformSplit' } + +# create table with five regions based on hex keys +create 't3','f1', { NUMREGIONS => 5, SPLITALGO => 'HexStringSplit' } + + See <> for issues related to understanding your keyspace and pre-creating regions. See <> for discussion on manually pre-splitting regions. +See <> for more details of using the HBase Shell to pre-split tables. [[def.log.flush]] === Table Creation: Deferred Log Flush http://git-wip-us.apache.org/repos/asf/hbase/blob/73ec3385/src/main/asciidoc/_chapters/shell.adoc -- diff --git a/src/main/asciidoc/_chapters/shell.adoc b/src/main/asciidoc/_chapters/shell.adoc index a4237fd..8f1f59b 100644 --- a/src/main/asciidoc/_chapters/shell.adoc +++ b/src/main/asciidoc/_chapters/shell.adoc @@ -352,6 +352,68 @@ hbase(main):022:0> Date.new(1218920189000).toString() => "Sat Aug 16 20:56:29 UT To output in a format that is exactly like that of the HBase log format will take a little messing with link:http://download.oracle.com/javase/6/docs/api/java/text/SimpleDateFormat.html[SimpleDateFormat]. +[[tricks.pre-split]] +=== Pre-splitting tables with the HBase Shell +You can use a variety of options to pre-split tables when creating them via the HBase Shell `create` command. + +The simplest approach is to specify an array of split points when creating the table. Note that when specifying string literals as split points, these will create split points based on the underlying byte representation of the string. So when specifying a split point of '10', we are actually specifying the byte split point '\x31\30'. + +The split points will define `n+1` regions where `n` is the number of split points. The lowest region will contain all keys from the lowest possible key up to but not including the first split point key. +The next region will contain keys from the first split point up to, but not including the next split point key. +This will continue for all split points up to the last. The last region will be defined from the last split point up to the maximum possible key. + +[source] + +hbase>create 't1','f',SPLITS => ['10','20',30'] + + +In the above example, the table 't1' will be created with column family 'f', pre-split to fo
[16/50] hbase git commit: HBASE-15939 Two shell test failures on master (Talat and Ted)
HBASE-15939 Two shell test failures on master (Talat and Ted) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/72d3f2a8 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/72d3f2a8 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/72d3f2a8 Branch: refs/heads/hbase-12439 Commit: 72d3f2a86825808af879ed77a9fafe4c42daed6a Parents: fc890a2 Author: tedyu Authored: Thu Jun 2 15:44:18 2016 -0700 Committer: tedyu Committed: Thu Jun 2 15:44:18 2016 -0700 -- hbase-shell/src/main/ruby/hbase/hbase.rb | 2 +- .../java/org/apache/hadoop/hbase/client/AbstractTestShell.java | 4 2 files changed, 5 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/72d3f2a8/hbase-shell/src/main/ruby/hbase/hbase.rb -- diff --git a/hbase-shell/src/main/ruby/hbase/hbase.rb b/hbase-shell/src/main/ruby/hbase/hbase.rb index 797d784..852f349 100644 --- a/hbase-shell/src/main/ruby/hbase/hbase.rb +++ b/hbase-shell/src/main/ruby/hbase/hbase.rb @@ -58,7 +58,7 @@ module Hbase # Create new one each time def table(table, shell) - ::Hbase::Table.new(@connection.getTable(table), shell) + ::Hbase::Table.new(@connection.getTable(TableName.valueOf(table)), shell) end def replication_admin(formatter) http://git-wip-us.apache.org/repos/asf/hbase/blob/72d3f2a8/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java -- diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java index ddcc1ea..87d14dd 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java @@ -44,6 +44,10 @@ public abstract class AbstractTestShell { TEST_UTIL.getConfiguration().setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, false); TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3); +// Below settings are necessary for task monitor test. +TEST_UTIL.getConfiguration().setInt(HConstants.MASTER_INFO_PORT, 0); +TEST_UTIL.getConfiguration().setInt(HConstants.REGIONSERVER_INFO_PORT, 0); + TEST_UTIL.getConfiguration().setBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO, true); // Security setup configuration SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration()); VisibilityTestUtil.enableVisiblityLabels(TEST_UTIL.getConfiguration());
[08/50] hbase git commit: HBASE-15931 Add log for long-running tasks in AsyncProcess
HBASE-15931 Add log for long-running tasks in AsyncProcess Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/53eb27bb Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/53eb27bb Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/53eb27bb Branch: refs/heads/hbase-12439 Commit: 53eb27bb60bc736f0c56c7a2facfb6e6ccb91be5 Parents: cbb95cd Author: Yu Li Authored: Thu Jun 2 12:00:42 2016 +0800 Committer: Yu Li Committed: Thu Jun 2 12:00:42 2016 +0800 -- .../hadoop/hbase/client/AsyncProcess.java | 53 .../hbase/client/BufferedMutatorImpl.java | 3 +- .../hadoop/hbase/client/TestAsyncProcess.java | 5 +- 3 files changed, 49 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/53eb27bb/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java index 6f7ba59..812e4bf 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java @@ -120,6 +120,12 @@ class AsyncProcess { */ public static final String LOG_DETAILS_FOR_BATCH_ERROR = "hbase.client.log.batcherrors.details"; + private final int thresholdToLogUndoneTaskDetails; + private static final String THRESHOLD_TO_LOG_UNDONE_TASK_DETAILS = + "hbase.client.threshold.log.details"; + private static final int DEFAULT_THRESHOLD_TO_LOG_UNDONE_TASK_DETAILS = 10; + private final int THRESHOLD_TO_LOG_REGION_DETAILS = 2; + /** * The context used to wait for results from one submit call. * 1) If AsyncProcess is set to track errors globally, and not per call (for HTable puts), @@ -332,6 +338,10 @@ class AsyncProcess { this.rpcCallerFactory = rpcCaller; this.rpcFactory = rpcFactory; this.logBatchErrorDetails = conf.getBoolean(LOG_DETAILS_FOR_BATCH_ERROR, false); + +this.thresholdToLogUndoneTaskDetails = +conf.getInt(THRESHOLD_TO_LOG_UNDONE_TASK_DETAILS, + DEFAULT_THRESHOLD_TO_LOG_UNDONE_TASK_DETAILS); } /** @@ -389,7 +399,7 @@ class AsyncProcess { List locationErrorRows = null; do { // Wait until there is at least one slot for a new task. - waitForMaximumCurrentTasks(maxTotalConcurrentTasks - 1); + waitForMaximumCurrentTasks(maxTotalConcurrentTasks - 1, tableName.getNameAsString()); // Remember the previous decisions about regions or region servers we put in the // final multi. @@ -1765,18 +1775,19 @@ class AsyncProcess { @VisibleForTesting /** Waits until all outstanding tasks are done. Used in tests. */ void waitUntilDone() throws InterruptedIOException { -waitForMaximumCurrentTasks(0); +waitForMaximumCurrentTasks(0, null); } /** Wait until the async does not have more than max tasks in progress. */ - private void waitForMaximumCurrentTasks(int max) throws InterruptedIOException { -waitForMaximumCurrentTasks(max, tasksInProgress, id); + private void waitForMaximumCurrentTasks(int max, String tableName) + throws InterruptedIOException { +waitForMaximumCurrentTasks(max, tasksInProgress, id, tableName); } // Break out this method so testable @VisibleForTesting - static void waitForMaximumCurrentTasks(int max, final AtomicLong tasksInProgress, final long id) - throws InterruptedIOException { + void waitForMaximumCurrentTasks(int max, final AtomicLong tasksInProgress, final long id, + String tableName) throws InterruptedIOException { long lastLog = EnvironmentEdgeManager.currentTime(); long currentInProgress, oldInProgress = Long.MAX_VALUE; while ((currentInProgress = tasksInProgress.get()) > max) { @@ -1785,7 +1796,11 @@ class AsyncProcess { if (now > lastLog + 1) { lastLog = now; LOG.info("#" + id + ", waiting for some tasks to finish. Expected max=" - + max + ", tasksInProgress=" + currentInProgress); + + max + ", tasksInProgress=" + currentInProgress + + " hasError=" + hasError() + tableName == null ? "" : ", tableName=" + tableName); + if (currentInProgress <= thresholdToLogUndoneTaskDetails) { +logDetailsOfUndoneTasks(currentInProgress); + } } } oldInProgress = currentInProgress; @@ -1802,6 +1817,25 @@ class AsyncProcess { } } + private void logDetailsOfUndoneTasks(long taskInProgress) { +ArrayList servers = new ArrayList(); +for (Map.Entry entry : taskCounterPerServer.entrySet()) { + if
[01/50] hbase git commit: HBASE-15918 Cleanup excludes/includes file after use in hbase-personality.sh to avoid asf license error. (Apekshit)
Repository: hbase Updated Branches: refs/heads/hbase-12439 a11091c49 -> bd45cf347 HBASE-15918 Cleanup excludes/includes file after use in hbase-personality.sh to avoid asf license error. (Apekshit) Change-Id: I3a47113ef9f598f90357b7cb580cbdc56f3c46f4 Signed-off-by: stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/75c23605 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/75c23605 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/75c23605 Branch: refs/heads/hbase-12439 Commit: 75c23605430266da0f30eef04b97ebd4b30c60b8 Parents: f0a1e22 Author: Apekshit Authored: Mon May 30 20:22:29 2016 -0700 Committer: stack Committed: Mon May 30 21:30:19 2016 -0700 -- dev-support/hbase-personality.sh | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/75c23605/dev-support/hbase-personality.sh -- diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index 6797090..3b6ebad 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -104,6 +104,7 @@ function personality_modules if [[ -n "${excludes}" ]]; then extra="${extra} -Dtest.exclude.pattern=${excludes}" fi + rm excludes else echo "Wget error $? in fetching excludes file from url" \ "${EXCLUDE_TESTS_URL}. Ignoring and proceeding." @@ -115,6 +116,7 @@ function personality_modules if [[ -n "${includes}" ]]; then extra="${extra} -Dtest=${includes}" fi + rm includes else echo "Wget error $? in fetching includes file from url" \ "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
[10/50] hbase git commit: HBASE-15933 NullPointerException may be thrown from SimpleRegionNormalizer#getRegionSize()
HBASE-15933 NullPointerException may be thrown from SimpleRegionNormalizer#getRegionSize() Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cfe868d5 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cfe868d5 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cfe868d5 Branch: refs/heads/hbase-12439 Commit: cfe868d56eeb0367c2fcf4a18a1d06c57abb7e54 Parents: a0f49c9 Author: tedyu Authored: Thu Jun 2 01:55:28 2016 -0700 Committer: tedyu Committed: Thu Jun 2 01:55:28 2016 -0700 -- .../hbase/master/normalizer/SimpleRegionNormalizer.java | 8 +++- 1 file changed, 7 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/cfe868d5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java index 583f873..d209eb7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java @@ -140,7 +140,9 @@ public class SimpleRegionNormalizer implements RegionNormalizer { for (int i = 0; i < tableRegions.size(); i++) { HRegionInfo hri = tableRegions.get(i); long regionSize = getRegionSize(hri); - totalSizeMb += regionSize; + if (regionSize > 0) { +totalSizeMb += regionSize; + } } double avgRegionSize = totalSizeMb / (double) tableRegions.size(); @@ -204,6 +206,10 @@ public class SimpleRegionNormalizer implements RegionNormalizer { getRegionServerOfRegion(hri); RegionLoad regionLoad = masterServices.getServerManager().getLoad(sn). getRegionsLoad().get(hri.getRegionName()); +if (regionLoad == null) { + LOG.debug(hri.getRegionNameAsString() + " was not found in RegionsLoad"); + return -1; +} return regionLoad.getStorefileSizeMB(); } }
hbase git commit: HBASE-16006 FileSystem should be obtained from specified path in WALInputFormat#getSplits()
Repository: hbase Updated Branches: refs/heads/HBASE-7912 5901cbbe6 -> 403697cd6 HBASE-16006 FileSystem should be obtained from specified path in WALInputFormat#getSplits() Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/403697cd Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/403697cd Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/403697cd Branch: refs/heads/HBASE-7912 Commit: 403697cd6881f6cc6591d076399a05b2bd285e3e Parents: 5901cbb Author: tedyu Authored: Fri Jun 10 15:16:48 2016 -0700 Committer: Ted Committed: Fri Jun 10 15:16:48 2016 -0700 -- .../java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java| 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/403697cd/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java index 736b8a5..2fa7a70 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java @@ -239,10 +239,9 @@ public class WALInputFormat extends InputFormat { long startTime = conf.getLong(startKey, Long.MIN_VALUE); long endTime = conf.getLong(endKey, Long.MAX_VALUE); -FileSystem fs = FileSystem.get(conf); - List allFiles = new ArrayList(); for(Path inputPath: inputPaths){ + FileSystem fs = inputPath.getFileSystem(conf); List files = getFiles(fs, inputPath, startTime, endTime); allFiles.addAll(files); }
hbase git commit: HBASE-16004 Update to Netty 4.1.1
Repository: hbase Updated Branches: refs/heads/master 6da6babe4 -> bd45cf347 HBASE-16004 Update to Netty 4.1.1 Signed-off-by: stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bd45cf34 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bd45cf34 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bd45cf34 Branch: refs/heads/master Commit: bd45cf34762332a3a51f605798a3e050e7a1e62e Parents: 6da6bab Author: Jurriaan Mous Authored: Fri Jun 10 17:57:42 2016 +0200 Committer: stack Committed: Fri Jun 10 12:14:05 2016 -0700 -- .../main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java| 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/bd45cf34/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java index c1ed748..723a234 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java @@ -347,13 +347,13 @@ public class AsyncRpcClient extends AbstractRpcClient { } // do not close global EventLoopGroup. if (!useGlobalEventLoopGroup) { - bootstrap.group().shutdownGracefully(); + bootstrap.config().group().shutdownGracefully(); } } @Override public EventLoop getEventExecutor() { -return this.bootstrap.group().next(); +return this.bootstrap.config().group().next(); } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/bd45cf34/pom.xml -- diff --git a/pom.xml b/pom.xml index 0e33ae8..93b7a0e 100644 --- a/pom.xml +++ b/pom.xml @@ -1254,7 +1254,7 @@ 4.0.3 2.4.1 1.3.3 -4.0.30.Final +4.1.1.Final 3.6.2.Final 2.1.2 1.0.8
hbase git commit: HBASE-15946 Eliminate possible security concerns in RS web UI's store file metrics (Sean Mackrory)
Repository: hbase Updated Branches: refs/heads/branch-1.2 f245fe106 -> d2d3dcdae HBASE-15946 Eliminate possible security concerns in RS web UI's store file metrics (Sean Mackrory) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d2d3dcda Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d2d3dcda Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d2d3dcda Branch: refs/heads/branch-1.2 Commit: d2d3dcdaec0412614badf77f866b89256296d8f4 Parents: f245fe1 Author: Sean Mackrory Authored: Tue May 31 10:28:27 2016 -0600 Committer: Mikhail Antonov Committed: Fri Jun 10 10:35:37 2016 -0700 -- .../hbase/io/hfile/HFilePrettyPrinter.java | 111 --- .../hbase-webapps/regionserver/storeFile.jsp| 35 +++--- 2 files changed, 86 insertions(+), 60 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/d2d3dcda/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java index f083f8d..e7cfa7b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java @@ -1,4 +1,3 @@ - /* * * Licensed to the Apache Software Foundation (ASF) under one @@ -97,6 +96,9 @@ public class HFilePrettyPrinter extends Configured implements Tool { private boolean checkFamily; private boolean isSeekToRow = false; + private PrintStream out = System.out; + private PrintStream err = System.err; + /** * The row which the user wants to specify and print all the KeyValues for. */ @@ -140,6 +142,11 @@ public class HFilePrettyPrinter extends Configured implements Tool { options.addOptionGroup(files); } + public void setPrintStreams(PrintStream out, PrintStream err) { +this.out = out; +this.err = err; + } + public boolean parseOptions(String args[]) throws ParseException, IOException { if (args.length == 0) { @@ -170,7 +177,7 @@ public class HFilePrettyPrinter extends Configured implements Tool { row = Bytes.toBytesBinary(key); isSeekToRow = true; } else { -System.err.println("Invalid row is specified."); +err.println("Invalid row is specified."); System.exit(-1); } } @@ -184,17 +191,17 @@ public class HFilePrettyPrinter extends Configured implements Tool { String enc = HRegionInfo.encodeRegionName(rn); Path regionDir = new Path(tableDir, enc); if (verbose) -System.out.println("region dir -> " + regionDir); +out.println("region dir -> " + regionDir); List regionFiles = HFile.getStoreFiles(FileSystem.get(getConf()), regionDir); if (verbose) -System.out.println("Number of region files found -> " +out.println("Number of region files found -> " + regionFiles.size()); if (verbose) { int i = 1; for (Path p : regionFiles) { if (verbose) -System.out.println("Found file[" + i++ + "] -> " + p); +out.println("Found file[" + i++ + "] -> " + p); } } files.addAll(regionFiles); @@ -227,27 +234,46 @@ public class HFilePrettyPrinter extends Configured implements Tool { // iterate over all files found for (Path fileName : files) { try { -processFile(fileName); +int exitCode = processFile(fileName); +if (exitCode != 0) { + return exitCode; +} } catch (IOException ex) { LOG.error("Error reading " + fileName, ex); -System.exit(-2); +return -2; } } if (verbose || printKey) { - System.out.println("Scanned kv count -> " + count); + out.println("Scanned kv count -> " + count); } return 0; } - private void processFile(Path file) throws IOException { + public int processFile(Path file) throws IOException { if (verbose) - System.out.println("Scanning -> " + file); + out.println("Scanning -> " + file); + +Path rootPath = FSUtils.getRootDir(getConf()); +String rootString = rootPath + rootPath.SEPARATOR; +if (!file.toString().startsWith(rootString)) { + // First we see if fully-qualified URI matches the root dir. It might + // also be an absolute path in the same filesystem, so we prepend the FS + // of the root dir and see if that fully-qualified URI matches. + FileSystem rootFS = rootPath.getFileSystem(getConf()); + String qualifiedFile = rootFS.g