Repository: hbase Updated Branches: refs/heads/master c8e9a295c -> c6e9dabe6
http://git-wip-us.apache.org/repos/asf/hbase/blob/c6e9dabe/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestDelayedUtil.java ---------------------------------------------------------------------- diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestDelayedUtil.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestDelayedUtil.java new file mode 100644 index 0000000..1e2db4d --- /dev/null +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestDelayedUtil.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.procedure2.util; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; + +@Category({MasterTests.class, SmallTests.class}) +public class TestDelayedUtil { + private static final Log LOG = LogFactory.getLog(TestDelayedUtil.class); + + @Test + public void testDelayedContainerEquals() { + Object o1 = new Object(); + Object o2 = new Object(); + ZeroDelayContainer<Long> lnull = new ZeroDelayContainer(null); + ZeroDelayContainer<Long> l10a = new ZeroDelayContainer<Long>(10L); + ZeroDelayContainer<Long> l10b = new ZeroDelayContainer(10L); + ZeroDelayContainer<Long> l15 = new ZeroDelayContainer(15L); + ZeroDelayContainer<Object> onull = new ZeroDelayContainer<Object>(null); + ZeroDelayContainer<Object> o1ca = new ZeroDelayContainer<Object>(o1); + ZeroDelayContainer<Object> o1cb = new ZeroDelayContainer<Object>(o1); + ZeroDelayContainer<Object> o2c = new ZeroDelayContainer<Object>(o2); + + ZeroDelayContainer[] items = new ZeroDelayContainer[] { + lnull, l10a, l10b, l15, onull, o1ca, o1cb, o2c, + }; + + assertContainersEquals(lnull, items, lnull, onull); + assertContainersEquals(l10a, items, l10a, l10b); + assertContainersEquals(l10b, items, l10a, l10b); + assertContainersEquals(l15, items, l15); + assertContainersEquals(onull, items, lnull, onull); + assertContainersEquals(o1ca, items, o1ca, o1cb); + assertContainersEquals(o1cb, items, o1ca, o1cb); + assertContainersEquals(o2c, items, o2c); + } + + private void assertContainersEquals(final ZeroDelayContainer src, + final ZeroDelayContainer[] items, final ZeroDelayContainer... matches) { + for (int i = 0; i < items.length; ++i) { + boolean shouldMatch = false; + for (int j = 0; j < matches.length; ++j) { + if (items[i] == matches[j]) { + shouldMatch = true; + break; + } + } + boolean isMatching = src.equals(items[i]); + assertEquals(src.getObject() + " unexpectedly match " + items[i].getObject(), + shouldMatch, isMatching); + } + } + + private static class ZeroDelayContainer<T> extends DelayedUtil.DelayedContainer<T> { + public ZeroDelayContainer(final T object) { + super(object); + } + + @Override + public long getTimeoutTimestamp() { + return 0; + } + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/c6e9dabe/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index af5d03d..a8329e3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -1035,6 +1035,7 @@ public class HMaster extends HRegionServer implements MasterServices { procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this)); procedureExecutor = new ProcedureExecutor(conf, procEnv, procedureStore, procEnv.getProcedureQueue()); + configurationManager.registerObserver(procEnv); final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, Math.max(Runtime.getRuntime().availableProcessors(), @@ -1048,6 +1049,7 @@ public class HMaster extends HRegionServer implements MasterServices { private void stopProcedureExecutor() { if (procedureExecutor != null) { + configurationManager.deregisterObserver(procedureExecutor.getEnvironment()); procedureExecutor.stop(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/c6e9dabe/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java index 183b41d..9362f24 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.conf.ConfigurationObserver; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.ipc.RpcServer; @@ -42,7 +43,7 @@ import org.apache.hadoop.hbase.util.FSUtils; @InterfaceAudience.Private @InterfaceStability.Evolving -public class MasterProcedureEnv { +public class MasterProcedureEnv implements ConfigurationObserver { private static final Log LOG = LogFactory.getLog(MasterProcedureEnv.class); @InterfaceAudience.Private @@ -161,4 +162,9 @@ public class MasterProcedureEnv { procSched.suspendEvent(event); } } + + @Override + public void onConfigurationChange(Configuration conf) { + master.getMasterProcedureExecutor().refreshConfiguration(conf); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/c6e9dabe/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java index cc9efd2..a7e1fb3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java @@ -191,6 +191,10 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler { serverBuckets[i] = null; } + // Remove Namespaces + clear(namespaceMap, null, NAMESPACE_QUEUE_KEY_COMPARATOR); + namespaceMap = null; + // Remove Tables clear(tableMap, tableRunQueue, TABLE_QUEUE_KEY_COMPARATOR); tableMap = null; @@ -204,12 +208,12 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler { Queue<T> node = AvlTree.getFirst(treeMap); assert !node.isSuspended() : "can't clear suspended " + node.getKey(); treeMap = AvlTree.remove(treeMap, node.getKey(), comparator); - removeFromRunQueue(fairq, node); + if (fairq != null) removeFromRunQueue(fairq, node); } } @Override - public int queueSize() { + protected int queueSize() { int count = 0; // Server queues http://git-wip-us.apache.org/repos/asf/hbase/blob/c6e9dabe/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureSchedulerPerformanceEvaluation.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureSchedulerPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureSchedulerPerformanceEvaluation.java index 6e4f3cd..ee9b8fc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureSchedulerPerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureSchedulerPerformanceEvaluation.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.master.TableLockManager; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.TestProcedure; +import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.util.AbstractHBaseTool; import org.apache.hadoop.hbase.util.Bytes; @@ -244,24 +245,29 @@ public class MasterProcedureSchedulerPerformanceEvaluation extends AbstractHBase protected int doWork() throws Exception { procedureScheduler = new MasterProcedureScheduler( UTIL.getConfiguration(), new TableLockManager.NullTableLockManager()); + procedureScheduler.start(); setupOperations(); final Thread[] threads = new Thread[numThreads]; for (int i = 0; i < numThreads; ++i) { threads[i] = new AddProcsWorker(); } - final float addBackTime = runThreads(threads) / 1000.0f; + final long addBackTime = runThreads(threads); System.out.println("Added " + numOps + " procedures to scheduler."); for (int i = 0; i < numThreads; ++i) { threads[i] = new PollAndLockWorker(); } - final float pollTime = runThreads(threads) / 1000.0f; + final long pollTime = runThreads(threads); + procedureScheduler.stop(); + + final float pollTimeSec = pollTime / 1000.0f; + final float addBackTimeSec = addBackTime / 1000.0f; System.out.println("******************************************"); - System.out.println("Time - addBack : " + addBackTime + "sec"); - System.out.println("Ops/sec - addBack : " + ((float)numOps / addBackTime)); - System.out.println("Time - poll : " + pollTime + "sec"); - System.out.println("Ops/sec - poll : " + ((float)numOps / pollTime)); + System.out.println("Time - addBack : " + StringUtils.humanTimeDiff(addBackTime)); + System.out.println("Ops/sec - addBack : " + StringUtils.humanSize(numOps / addBackTimeSec)); + System.out.println("Time - poll : " + StringUtils.humanTimeDiff(pollTime)); + System.out.println("Ops/sec - poll : " + StringUtils.humanSize(numOps / pollTimeSec)); System.out.println("Num Operations : " + numOps); System.out.println(); System.out.println("Completed : " + completed.get()); http://git-wip-us.apache.org/repos/asf/hbase/blob/c6e9dabe/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java index bcacb48..73a87d6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java @@ -59,11 +59,13 @@ public class TestMasterProcedureScheduler { public void setUp() throws IOException { conf = HBaseConfiguration.create(); queue = new MasterProcedureScheduler(conf, new TableLockManager.NullTableLockManager()); + queue.start(); } @After public void tearDown() throws IOException { assertEquals("proc-queue expected to be empty", 0, queue.size()); + queue.stop(); queue.clear(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/c6e9dabe/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java index d6ddd13..d66ca85 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.master.TableLockManager; import org.apache.hadoop.hbase.master.procedure.TestMasterProcedureScheduler.TestTableProcedure; import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -55,11 +56,13 @@ public class TestMasterProcedureSchedulerConcurrency { public void setUp() throws IOException { conf = HBaseConfiguration.create(); queue = new MasterProcedureScheduler(conf, new TableLockManager.NullTableLockManager()); + queue.start(); } @After public void tearDown() throws IOException { assertEquals("proc-queue expected to be empty", 0, queue.size()); + queue.stop(); queue.clear(); } @@ -207,6 +210,14 @@ public class TestMasterProcedureSchedulerConcurrency { } } + @Test(timeout=60000) + public void testMasterProcedureSchedulerPerformanceEvaluation() throws Exception { + // Make sure the tool does not get stuck + MasterProcedureSchedulerPerformanceEvaluation.main(new String[] { + "-num_ops", "1000" + }); + } + public static class TestTableProcSet { private final MasterProcedureScheduler queue;
