This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new e4644c3 HBASE-16488 Starting namespace and quota services in master
startup asynchronously
e4644c3 is described below
commit e4644c35c1e0de420a548bc0dfbe4d8049aca915
Author: Xu Cang <[email protected]>
AuthorDate: Tue Apr 23 13:16:09 2019 -0700
HBASE-16488 Starting namespace and quota services in master startup
asynchronously
Signed-off-by: Andrew Purtell <[email protected]>
---
.../org/apache/hadoop/hbase/master/HMaster.java | 105 +++++++++++++++++----
.../hadoop/hbase/master/MasterRpcServices.java | 4 +-
.../hadoop/hbase/master/TableNamespaceManager.java | 21 ++++-
.../master/procedure/CreateNamespaceProcedure.java | 19 ++--
.../master/procedure/CreateTableProcedure.java | 2 +-
.../master/procedure/DeleteNamespaceProcedure.java | 4 +-
.../hbase/master/procedure/MasterProcedureEnv.java | 4 +
.../master/procedure/ModifyNamespaceProcedure.java | 4 +-
.../apache/hadoop/hbase/HBaseTestingUtility.java | 34 +++++++
.../hbase/client/TestRollbackFromClient.java | 2 +-
.../hbase/master/TestAssignmentListener.java | 22 ++++-
.../hadoop/hbase/master/TestMasterFailover.java | 3 +
.../hadoop/hbase/master/TestMasterMetrics.java | 3 +
.../hadoop/hbase/master/TestMasterNoCluster.java | 8 --
.../master/handler/TestCreateTableHandler.java | 2 +-
.../procedure/TestCreateNamespaceProcedure.java | 4 +
.../procedure/TestMasterProcedureEvents.java | 71 +++++++++++++-
.../procedure/TestModifyNamespaceProcedure.java | 4 +
.../hadoop/hbase/regionserver/TestRegionOpen.java | 7 ++
.../hbase/wal/TestWALOpenAfterDNRollingStart.java | 1 +
20 files changed, 273 insertions(+), 51 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index e8b9a81..fa9906c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -160,6 +160,7 @@ import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.CompressionTest;
import org.apache.hadoop.hbase.util.ConfigUtil;
import org.apache.hadoop.hbase.util.EncryptionTest;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.ModifyRegionUtils;
@@ -313,6 +314,10 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
// initialization may have not completed yet.
volatile boolean serviceStarted = false;
+ // flag set after we complete asynchronized services and master
initialization is done,
+ private final ProcedureEvent namespaceManagerInitialized =
+ new ProcedureEvent("master namespace manager initialized");
+
// flag set after we complete assignMeta.
private final ProcedureEvent serverCrashProcessingEnabled =
new ProcedureEvent("server crash processing");
@@ -880,8 +885,9 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this);
getChoreService().scheduleChore(periodicDoMetricsChore);
- status.setStatus("Starting namespace manager");
- initNamespace();
+
+ status.setStatus("Starting namespace manager and quota manager");
+ initNamespaceAndQuotaManager();
if (this.cpHost != null) {
try {
@@ -902,11 +908,6 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
// Set master as 'initialized'.
setInitialized(true);
- assignmentManager.checkIfShouldMoveSystemRegionAsync();
-
- status.setStatus("Starting quota manager");
- initQuotaManager();
-
// assign the meta replicas
Set<ServerName> EMPTY_SET = new HashSet<ServerName>();
int numReplicas = conf.getInt(HConstants.META_REPLICAS_NUM,
@@ -949,12 +950,6 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
}
}
- private void initQuotaManager() throws IOException {
- quotaManager = new MasterQuotaManager(this);
- this.assignmentManager.setRegionStateListener((RegionStateListener)
quotaManager);
- quotaManager.start();
- }
-
/**
* Create a {@link ServerManager} instance.
* @param master
@@ -1101,12 +1096,61 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
}
}
- void initNamespace() throws IOException {
+ /*
+ * The main purpose is to start namespace manager and quota manager async to
+ * unblock overall master initialization
+ *
+ * @throws IOException
+ */
+ private void initNamespaceAndQuotaManager() throws IOException {
//create namespace manager
tableNamespaceManager = new TableNamespaceManager(this);
- tableNamespaceManager.start();
+ //create quota manager
+ this.quotaManager = new MasterQuotaManager(this);
+
this.assignmentManager.setRegionStateListener((RegionStateListener)quotaManager);
+
+ if (conf.getBoolean("hbase.master.start.wait.for.namespacemanager",
false)) {
+ // If being asked not to async start namespace manager, then just block
+ // master service starting until namespace manager is ready.
+ //
+ // Note: Quota manager depends on namespace manager. Therefore, its
starting
+ // method has to be in-sync with namespace manager.
+ LOG.info("Starting namespace manager and quota manager synchronously");
+
+ tableNamespaceManager.start();
+ setNamespaceManagerInitializedEvent(true);
+
+ quotaManager.start();
+ } else { // else asynchronously start namespace manager and quota manager
+ LOG.info("Starting namespace manager and quota manager asynchronously");
+ Threads.setDaemonThreadRunning(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ // Start namespace manager and wait to it to be fully started.
+ try {
+ tableNamespaceManager.start();
+ setNamespaceManagerInitializedEvent(true);
+ LOG.info("Namespace manager started.");
+ } catch (IOException e) {
+ LOG.error("Namespace manager failed to start. ", e);
+ abort("Shutdown Master due to namespace manager failed to start.
", e);
+ }
+ // Quota Manager depends on Namespace manager to be fully
initialized.
+ try {
+ quotaManager.start();
+ LOG.info("Quota manager started.");
+ } catch (IOException ie) {
+ LOG.error("Quota Manager failed to start. ", ie);
+ abort("Shutdown Master due to Quota Manager failure to start. ",
ie);
+ }
+ }
+ }, "Init Namespace Manager and Quota Manager Async"));
+ }
+ assignmentManager.checkIfShouldMoveSystemRegionAsync();
}
+
+
boolean isCatalogJanitorEnabled() {
return catalogJanitorChore != null ?
catalogJanitorChore.getEnabled() : false;
@@ -2665,11 +2709,28 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
void checkNamespaceManagerReady() throws IOException {
checkInitialized();
- if (tableNamespaceManager == null ||
- !tableNamespaceManager.isTableAvailableAndInitialized(true)) {
+
+ if (tableNamespaceManager == null) {
throw new IOException("Table Namespace Manager not ready yet, try again
later");
+ } else if (!tableNamespaceManager.isTableAvailableAndInitialized(true)) {
+ try {
+ // Wait some time.
+ long startTime = EnvironmentEdgeManager.currentTime();
+ int timeout = conf.getInt("hbase.master.namespace.wait.for.ready",
30000);
+ while (!tableNamespaceManager.isTableNamespaceManagerStarted() &&
+ EnvironmentEdgeManager.currentTime() - startTime < timeout) {
+ Thread.sleep(100);
+ }
+ } catch (InterruptedException e) {
+ throw (InterruptedIOException) new
InterruptedIOException().initCause(e);
+ }
+ if (!tableNamespaceManager.isTableNamespaceManagerStarted()) {
+ throw new IOException("Table Namespace Manager not fully initialized,
try again later");
+ }
}
}
+
+
/**
* Report whether this master is currently the active master or not.
* If not active master, we are parked on ZK waiting to become active.
@@ -2718,6 +2779,16 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
return initialized;
}
+ public void setNamespaceManagerInitializedEvent(boolean
isNamespaceManagerInitialized) {
+ procedureExecutor.getEnvironment().setEventReady(
+ namespaceManagerInitialized,
+ isNamespaceManagerInitialized);
+ }
+
+ public ProcedureEvent getNamespaceManagerInitializedEvent() {
+ return namespaceManagerInitialized;
+ }
+
/**
* ServerCrashProcessingEnabled is set false before completing assignMeta to
prevent processing
* of crashed servers.
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 6b67473..ee6f6f3 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -1323,7 +1323,7 @@ public class MasterRpcServices extends RSRpcServices
public RestoreSnapshotResponse restoreSnapshot(RpcController controller,
RestoreSnapshotRequest request) throws ServiceException {
try {
- master.checkInitialized();
+ master.checkNamespaceManagerReady();
master.snapshotManager.checkSnapshotSupport();
// ensure namespace exists
@@ -1631,7 +1631,7 @@ public class MasterRpcServices extends RSRpcServices
@Override
public SetQuotaResponse setQuota(RpcController c, SetQuotaRequest req)
throws ServiceException {
try {
- master.checkInitialized();
+ master.checkNamespaceManagerReady();
return master.getMasterQuotaManager().setQuota(req);
} catch (Exception e) {
throw new ServiceException(e);
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java
index 6bcfb77..33a2fcf 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.hbase.master;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
+
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.NavigableSet;
@@ -48,8 +51,6 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import com.google.common.collect.Sets;
-
/**
* This is a helper class used to manage the namespace
* metadata that is stored in TableName.NAMESPACE_TABLE_NAME
@@ -72,7 +73,8 @@ public class TableNamespaceManager {
public static final String KEY_MAX_REGIONS =
"hbase.namespace.quota.maxregions";
public static final String KEY_MAX_TABLES =
"hbase.namespace.quota.maxtables";
static final String NS_INIT_TIMEOUT = "hbase.master.namespace.init.timeout";
- static final int DEFAULT_NS_INIT_TIMEOUT = 300000;
+ static final int DEFAULT_NS_INIT_TIMEOUT = 1800000; // default is 30 minutes
+ private static final int WAIT_MESSAGE_TO_PRINTOUT = 300000; // print out
message every 5 minutes
public TableNamespaceManager(MasterServices masterServices) {
this.masterServices = masterServices;
@@ -91,12 +93,18 @@ public class TableNamespaceManager {
// If timed out, we will move ahead without initializing it.
// So that it should be initialized later on lazily.
long startTime = EnvironmentEdgeManager.currentTime();
+ long msgCount = 0;
+ long waitTime;
int timeout = conf.getInt(NS_INIT_TIMEOUT, DEFAULT_NS_INIT_TIMEOUT);
while (!isTableAvailableAndInitialized(false)) {
- if (EnvironmentEdgeManager.currentTime() - startTime + 100 > timeout) {
+ waitTime = EnvironmentEdgeManager.currentTime() - startTime;
+ if (waitTime > timeout) {
// We can't do anything if ns is not online.
throw new IOException("Timedout " + timeout + "ms waiting for
namespace table to " +
"be assigned");
+ } else if (waitTime > msgCount * WAIT_MESSAGE_TO_PRINTOUT) {
+ LOG.info("Waiting for namespace table to be online. Time waited = "
+ waitTime + " ms.");
+ msgCount++;
}
Thread.sleep(100);
}
@@ -105,6 +113,11 @@ public class TableNamespaceManager {
}
}
+ @VisibleForTesting
+ public boolean isTableNamespaceManagerStarted() {
+ return initialized;
+ }
+
private synchronized Table getNamespaceTable() throws IOException {
if (!isTableNamespaceManagerInitialized()) {
throw new IOException(this.getClass().getName() + " isn't ready to
serve");
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
index e873156..d19635c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
@@ -193,21 +193,20 @@ public class CreateNamespaceProcedure
sb.append(")");
}
- private boolean isBootstrapNamespace() {
+ private static boolean isBootstrapNamespace(final NamespaceDescriptor
nsDescriptor) {
return nsDescriptor.equals(NamespaceDescriptor.DEFAULT_NAMESPACE) ||
nsDescriptor.equals(NamespaceDescriptor.SYSTEM_NAMESPACE);
}
@Override
protected boolean acquireLock(final MasterProcedureEnv env) {
- if (!env.getMasterServices().isInitialized()) {
- // Namespace manager might not be ready if master is not fully
initialized,
- // return false to reject user namespace creation; return true for
default
- // and system namespace creation (this is part of master initialization).
- if (!isBootstrapNamespace() && env.waitInitialized(this)) {
- return false;
- }
+ // Namespace manager might not be ready if master is not fully initialized,
+ // return false to reject user namespace creation; return true for default
+ // and system namespace creation (this is part of master initialization).
+ if (!isBootstrapNamespace(nsDescriptor) &&
env.waitNamespaceManagerInitialized(this)) {
+ return false;
}
+
return env.getProcedureQueue().tryAcquireNamespaceExclusiveLock(this,
getNamespaceName());
}
@@ -331,7 +330,7 @@ public class CreateNamespaceProcedure
protected static void setNamespaceQuota(
final MasterProcedureEnv env,
final NamespaceDescriptor nsDescriptor) throws IOException {
- if (env.getMasterServices().isInitialized()) {
+ if (env.getMasterServices().isInitialized() &&
!isBootstrapNamespace(nsDescriptor)) {
env.getMasterServices().getMasterQuotaManager().setNamespaceQuota(nsDescriptor);
}
}
@@ -370,6 +369,6 @@ public class CreateNamespaceProcedure
protected boolean shouldWaitClientAck(MasterProcedureEnv env) {
// hbase and default namespaces are created on bootstrap internally by the
system
// the client does not know about this procedures.
- return !isBootstrapNamespace();
+ return !isBootstrapNamespace(nsDescriptor);
}
}
\ No newline at end of file
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
index 152af45..6de55fa 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
@@ -275,7 +275,7 @@ public class CreateTableProcedure
@Override
protected boolean acquireLock(final MasterProcedureEnv env) {
- if (!getTableName().isSystemTable() && env.waitInitialized(this)) {
+ if (!getTableName().isSystemTable() &&
env.waitNamespaceManagerInitialized(this)) {
return false;
}
return env.getProcedureQueue().tryAcquireTableExclusiveLock(this,
getTableName());
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
index a7ebc30..e011f29 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
@@ -210,7 +210,9 @@ public class DeleteNamespaceProcedure
@Override
protected boolean acquireLock(final MasterProcedureEnv env) {
- if (env.waitInitialized(this)) return false;
+ if (env.waitNamespaceManagerInitialized(this)) {
+ return false;
+ }
return env.getProcedureQueue().tryAcquireNamespaceExclusiveLock(this,
getNamespaceName());
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
index 3911f54..75822c8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
@@ -132,6 +132,10 @@ public class MasterProcedureEnv {
return procSched.waitEvent(((HMaster)master).getInitializedEvent(), proc);
}
+ public boolean waitNamespaceManagerInitialized(Procedure proc) {
+ return
procSched.waitEvent(((HMaster)master).getNamespaceManagerInitializedEvent(),
proc);
+ }
+
public boolean waitServerCrashProcessingEnabled(Procedure proc) {
return
procSched.waitEvent(((HMaster)master).getServerCrashProcessingEnabledEvent(),
proc);
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
index 5d9548b..5953032 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
@@ -191,7 +191,9 @@ public class ModifyNamespaceProcedure
@Override
protected boolean acquireLock(final MasterProcedureEnv env) {
- if (env.waitInitialized(this)) return false;
+ if (env.waitNamespaceManagerInitialized(this)) {
+ return false;
+ }
return env.getProcedureQueue().tryAcquireNamespaceExclusiveLock(this,
getNamespaceName());
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index cdebddb..c0092be 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -106,6 +106,7 @@ import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.visibility.VisibilityLabelsCache;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.JVMClusterUtil;
@@ -1483,6 +1484,7 @@ public class HBaseTestingUtility extends
HBaseCommonTestingUtility {
.setMaxVersions(numVersions);
desc.addFamily(hcd);
}
+ waitUntilTableNamespaceManagerStarted();
getHBaseAdmin().createTable(desc, startKey, endKey, numRegions);
// HBaseAdmin only waits for regions to appear in hbase:meta we should
wait until they are assigned
waitUntilAllRegionsAssigned(tableName);
@@ -1542,6 +1544,7 @@ public class HBaseTestingUtility extends
HBaseCommonTestingUtility {
hcd.setBlocksize(blockSize);
htd.addFamily(hcd);
}
+ waitUntilTableNamespaceManagerStarted();
getHBaseAdmin().createTable(htd, splitKeys);
// HBaseAdmin only waits for regions to appear in hbase:meta we should
wait until they are
// assigned
@@ -1558,6 +1561,7 @@ public class HBaseTestingUtility extends
HBaseCommonTestingUtility {
*/
public HTable createTable(HTableDescriptor htd, byte[][] splitRows)
throws IOException {
+ waitUntilTableNamespaceManagerStarted();
getHBaseAdmin().createTable(htd, splitRows);
// HBaseAdmin only waits for regions to appear in hbase:meta we should
wait until they are assigned
waitUntilAllRegionsAssigned(htd.getTableName());
@@ -1612,6 +1616,7 @@ public class HBaseTestingUtility extends
HBaseCommonTestingUtility {
hcd.setBloomFilterType(BloomType.NONE);
desc.addFamily(hcd);
}
+ waitUntilTableNamespaceManagerStarted();
getHBaseAdmin().createTable(desc);
return new HTable(c, desc.getTableName());
}
@@ -1634,6 +1639,7 @@ public class HBaseTestingUtility extends
HBaseCommonTestingUtility {
.setMaxVersions(numVersions);
desc.addFamily(hcd);
}
+ waitUntilTableNamespaceManagerStarted();
getHBaseAdmin().createTable(desc);
// HBaseAdmin only waits for regions to appear in hbase:meta we should
wait until they are assigned
waitUntilAllRegionsAssigned(tableName);
@@ -1658,6 +1664,7 @@ public class HBaseTestingUtility extends
HBaseCommonTestingUtility {
.setMaxVersions(numVersions);
desc.addFamily(hcd);
}
+ waitUntilTableNamespaceManagerStarted();
getHBaseAdmin().createTable(desc);
return new HTable(c, desc.getTableName());
}
@@ -1732,6 +1739,7 @@ public class HBaseTestingUtility extends
HBaseCommonTestingUtility {
HColumnDescriptor hcd = new
HColumnDescriptor(family).setMaxVersions(numVersions);
desc.addFamily(hcd);
}
+ waitUntilTableNamespaceManagerStarted();
getHBaseAdmin().createTable(desc, splitKeys);
// HBaseAdmin only waits for regions to appear in hbase:meta we should
wait until they are assigned
waitUntilAllRegionsAssigned(tableName);
@@ -1784,6 +1792,7 @@ public class HBaseTestingUtility extends
HBaseCommonTestingUtility {
.setBlocksize(blockSize);
desc.addFamily(hcd);
}
+ waitUntilTableNamespaceManagerStarted();
getHBaseAdmin().createTable(desc);
// HBaseAdmin only waits for regions to appear in hbase:meta we should
wait until they are assigned
waitUntilAllRegionsAssigned(tableName);
@@ -1823,6 +1832,7 @@ public class HBaseTestingUtility extends
HBaseCommonTestingUtility {
desc.addFamily(hcd);
i++;
}
+ waitUntilTableNamespaceManagerStarted();
getHBaseAdmin().createTable(desc);
// HBaseAdmin only waits for regions to appear in hbase:meta we should
wait until they are assigned
waitUntilAllRegionsAssigned(tableName);
@@ -1855,6 +1865,7 @@ public class HBaseTestingUtility extends
HBaseCommonTestingUtility {
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor hcd = new HColumnDescriptor(family);
desc.addFamily(hcd);
+ waitUntilTableNamespaceManagerStarted();
getHBaseAdmin().createTable(desc, splitRows);
// HBaseAdmin only waits for regions to appear in hbase:meta we should
wait until they are assigned
waitUntilAllRegionsAssigned(tableName);
@@ -1887,12 +1898,35 @@ public class HBaseTestingUtility extends
HBaseCommonTestingUtility {
HColumnDescriptor hcd = new HColumnDescriptor(family);
desc.addFamily(hcd);
}
+ waitUntilTableNamespaceManagerStarted();
getHBaseAdmin().createTable(desc, splitRows);
// HBaseAdmin only waits for regions to appear in hbase:meta we should
wait until they are assigned
waitUntilAllRegionsAssigned(desc.getTableName());
return new HTable(getConfiguration(), desc.getTableName());
}
+ public void waitUntilTableNamespaceManagerStarted() throws IOException {
+ final int maxwait = 60000;
+ final long startTime = EnvironmentEdgeManager.currentTime();
+ final HMaster master = getMiniHBaseCluster().getMaster();
+ do {
+ if(master.getTableNamespaceManager().isTableNamespaceManagerStarted()) {
+ break;
+ }
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupt while waiting for master namespace
manager starting.");
+ }
+ } while (EnvironmentEdgeManager.currentTime() - startTime < maxwait);
+ if(!master.getTableNamespaceManager().isTableNamespaceManagerStarted()) {
+ throw new IOException(
+ "Cannot continue testing due to master namespace manager not started
after waiting " +
+ (EnvironmentEdgeManager.currentTime() - startTime) + "
milliseconds");
+ }
+ }
+
+
/**
* Create an unmanaged WAL. Be sure to close it when you're through.
*/
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java
index f91adf4..fc20d70 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java
@@ -222,7 +222,7 @@ public class TestRollbackFromClient {
HColumnDescriptor col = new HColumnDescriptor(FAMILY);
col.setMaxVersions(versions);
desc.addFamily(col);
- TEST_UTIL.getHBaseAdmin().createTable(desc);
+ TEST_UTIL.createTable(tableName, FAMILY, versions);
int expected;
List<Cell> cells;
try (Connection conn =
ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
index 12d04ff..bd5e0fd 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -84,6 +85,18 @@ public class TestAssignmentListener {
Thread.sleep(100);
}
}
+
+ //wait on multiple possible values.
+ public int awaitModifications(List<Integer> values) throws
InterruptedException {
+ while (true) {
+ for (int v : values) {
+ if (modified.compareAndSet(v, 0)) {
+ return v;
+ }
+ Thread.sleep(100);
+ }
+ }
+ }
}
static class DummyAssignmentListener extends DummyListener implements
AssignmentListener {
@@ -226,10 +239,13 @@ public class TestAssignmentListener {
final byte[] FAMILY = Bytes.toBytes("cf");
// Create a new table, with a single region
- LOG.info("Create Table");
+ LOG.info("Create Table testtb");
TEST_UTIL.createTable(TABLE_NAME, FAMILY);
- listener.awaitModifications(1);
- assertEquals(1, listener.getLoadCount());
+ // Wait for 1 or 2 since if namespace table is created before testtb,
modified value will be 2
+ // Waiting on 1 will fail this test indefinitely.
+ List<Integer> modificationValues = Arrays.asList(1,2);
+ int modification = listener.awaitModifications(modificationValues);
+ assertEquals(modification, listener.getLoadCount());
assertEquals(0, listener.getCloseCount());
// Add some data
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
index 2228188..47577c4 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
@@ -508,6 +508,9 @@ public class TestMasterFailover {
HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
Configuration conf = TEST_UTIL.getConfiguration();
conf.setBoolean("hbase.assignment.usezk", true);
+ // The test depends on namespace region is online, therefore, we have to
+ // wait for namespace manager starting.
+ conf.setBoolean("hbase.master.start.wait.for.namespacemanager", true);
conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2);
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
index f6f9b49..c273afa 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
@@ -66,6 +66,9 @@ public class TestMasterMetrics {
public static void startCluster() throws Exception {
LOG.info("Starting cluster");
TEST_UTIL = new HBaseTestingUtility();
+ // The metrics depends on namespace region is online, therefore, we have to
+ // wait for namespace manager starting.
+
TEST_UTIL.getConfiguration().setBoolean("hbase.master.start.wait.for.namespacemanager",
true);
TEST_UTIL.startMiniCluster(1, 1, 1, null, MyMaster.class, null);
cluster = TEST_UTIL.getHBaseCluster();
LOG.info("Waiting for active/ready master");
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
index 4483fce..3b1daf2 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
@@ -215,10 +215,6 @@ public class TestMasterNoCluster {
public ClusterConnection getConnection() {
return mockedConnection;
}
-
- @Override
- void initNamespace() {
- }
};
master.start();
@@ -296,10 +292,6 @@ public class TestMasterNoCluster {
return null;
}
}
-
- @Override
- void initNamespace() {
- }
};
master.start();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
index f25e45c..eed85c1 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
@@ -54,10 +54,10 @@ public class TestCreateTableHandler {
private static final Log LOG =
LogFactory.getLog(TestCreateTableHandler.class);
private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
private static boolean throwException = false;
-
@Before
public void setUp() throws Exception {
+
TEST_UTIL.getConfiguration().setBoolean("hbase.master.start.wait.for.namespacemanager",
true);
TEST_UTIL.startMiniCluster(1);
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java
index e7ddff6..c59b27d 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java
@@ -52,6 +52,10 @@ public class TestCreateNamespaceProcedure {
private static void setupConf(Configuration conf) {
conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
+ // We have to disable the async namespace manager startup due to the
recovery and rollback
+ // tests would halt the execution of procedure. If at that time the
namespace table is
+ // not created, it would be forced to stop/wait and cause the test to fail.
+ conf.setBoolean("hbase.master.start.wait.for.namespacemanager", true);
}
@BeforeClass
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureEvents.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureEvents.java
index b7a6434..d23ab8a 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureEvents.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureEvents.java
@@ -35,6 +35,7 @@ import
org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -43,6 +44,8 @@ import org.junit.experimental.categories.Category;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
+
@Category({MasterTests.class, MediumTests.class})
public class TestMasterProcedureEvents {
private static final Log LOG =
LogFactory.getLog(TestCreateTableProcedure.class);
@@ -76,13 +79,77 @@ public class TestMasterProcedureEvents {
ProcedureExecutor<MasterProcedureEnv> procExec =
master.getMasterProcedureExecutor();
MasterProcedureScheduler procSched =
procExec.getEnvironment().getProcedureQueue();
+ HTableDescriptor htd = new HTableDescriptor(tableName);
+ HColumnDescriptor hcd = new HColumnDescriptor("f");
+ htd.addFamily(hcd);
+
+ while (!master.isInitialized()) {
+ Thread.sleep(250);
+ }
+ UTIL.createTable(htd, null);
+
+ // Modify the table descriptor
+ HTableDescriptor newHTD =
+ new
HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
+ long newMaxFileSize = newHTD.getMaxFileSize() * 2;
+ newHTD.setMaxFileSize(newMaxFileSize);
+ newHTD.setRegionReplication(3);
+
+ master.setInitialized(false); // fake it, set back later
+
+ ModifyTableProcedure proc = new
ModifyTableProcedure(procExec.getEnvironment(), newHTD);
+
+ long pollCalls = procSched.getPollCalls();
+ long nullPollCalls = procSched.getNullPollCalls();
+
+ long procId = procExec.submitProcedure(proc);
+ for (int i = 0; i < 10; ++i) {
+ Thread.sleep(100);
+ assertEquals(pollCalls + 1, procSched.getPollCalls());
+ assertEquals(nullPollCalls, procSched.getNullPollCalls());
+ }
+
+ master.setInitialized(true);
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+
+ assertEquals(pollCalls + 2, procSched.getPollCalls());
+ assertEquals(nullPollCalls, procSched.getNullPollCalls());
+ }
+
+ @Test
+ public void testNamespaceManagerInitializedEvent() throws Exception {
+ TableName tableName =
TableName.valueOf("testNamespaceManagerInitializedEvent");
+ HMaster master = UTIL.getMiniHBaseCluster().getMaster();
+ ProcedureExecutor<MasterProcedureEnv> procExec =
master.getMasterProcedureExecutor();
+ MasterProcedureScheduler procSched =
procExec.getEnvironment().getProcedureQueue();
+
HRegionInfo hri = new HRegionInfo(tableName);
HTableDescriptor htd = new HTableDescriptor(tableName);
HColumnDescriptor hcd = new HColumnDescriptor("f");
htd.addFamily(hcd);
while (!master.isInitialized()) Thread.sleep(250);
- master.setInitialized(false); // fake it, set back later
+
+ final int maxwait = 60000;
+ final long startTime = EnvironmentEdgeManager.currentTime();
+ do {
+ if(master.getTableNamespaceManager().isTableNamespaceManagerStarted()) {
+ break;
+ }
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupt while waiting for master namespace
manager starting.");
+ }
+ } while (EnvironmentEdgeManager.currentTime() - startTime < maxwait);
+
+ if(!master.getTableNamespaceManager().isTableNamespaceManagerStarted()) {
+ throw new IOException(
+ "Cannot continue testing due to master namespace manager not started
after waiting " +
+ (EnvironmentEdgeManager.currentTime() - startTime) + " milliseconds");
+ }
+
+ master.setNamespaceManagerInitializedEvent(false); // fake it, set back
later
CreateTableProcedure proc = new CreateTableProcedure(
procExec.getEnvironment(), htd, new HRegionInfo[] { hri });
@@ -97,7 +164,7 @@ public class TestMasterProcedureEvents {
assertEquals(nullPollCalls, procSched.getNullPollCalls());
}
- master.setInitialized(true);
+ master.setNamespaceManagerInitializedEvent(true);
ProcedureTestingUtility.waitProcedure(procExec, procId);
assertEquals(pollCalls + 2, procSched.getPollCalls());
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java
index 327afc4..394b114 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java
@@ -48,6 +48,10 @@ public class TestModifyNamespaceProcedure {
private static void setupConf(Configuration conf) {
conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
+ // We have to disable the async namespace manager startup due to the
recovery and rollback
+ // tests would halt the execution of procedure. If at that time the
namespace table is
+ // not created, it would be forced to stop/wait and cause the test to fail.
+ conf.setBoolean("hbase.master.start.wait.for.namespacemanager", true);
}
@BeforeClass
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
index f45df18..e69d717 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
@@ -66,6 +66,9 @@ public class TestRegionOpen {
@BeforeClass
public static void before() throws Exception {
+ // This test depends on namespace region open; therefore, we have to wait
for namespace
+ // manager start before continue.
+
HTU.getConfiguration().setBoolean("hbase.master.start.wait.for.namespacemanager",
true);
HTU.startMiniCluster(NB_SERVERS);
}
@@ -84,6 +87,10 @@ public class TestRegionOpen {
ThreadPoolExecutor exec = getRS().getExecutorService()
.getExecutorThreadPool(ExecutorType.RS_OPEN_PRIORITY_REGION);
+ // This test depends on namespace region open; therefore, we have to wait
for namespace
+ // manager start before continue.
+ HTU.waitUntilTableNamespaceManagerStarted();
+
assertEquals(1, exec.getCompletedTaskCount()); // namespace region
HTableDescriptor htd = new HTableDescriptor(tableName);
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALOpenAfterDNRollingStart.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALOpenAfterDNRollingStart.java
index ee1692e..4eb949f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALOpenAfterDNRollingStart.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALOpenAfterDNRollingStart.java
@@ -48,6 +48,7 @@ public class TestWALOpenAfterDNRollingStart {
false);
TEST_UTIL.getConfiguration().setLong("hbase.regionserver.hlog.check.lowreplication.interval",
checkLowReplicationInterval);
+
TEST_UTIL.getConfiguration().setBoolean("hbase.master.start.wait.for.namespacemanager",
true);
TEST_UTIL.startMiniDFSCluster(3);
TEST_UTIL.startMiniCluster(1);
