This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-11288.splittable-meta
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 21504fd8c99b09fd68c5cb540ddb2fde21f8c77c
Author: Duo Zhang <zhang...@apache.org>
AuthorDate: Wed May 27 11:30:37 2020 +0800
HBASE-24388 Store the locations of meta regions in master local store
(#1746)
Signed-off-by: Guanghao Zhang <zg...@apache.org>
Signed-off-by: stack <st...@apache.org>
---
.../org/apache/hadoop/hbase/MetaTableAccessor.java | 19 ++-
.../org/apache/hadoop/hbase/master/HMaster.java | 69 ++++++++--
.../hadoop/hbase/master/MasterMetaBootstrap.java | 144 +++++++++++++--------
.../hbase/master/assignment/AssignmentManager.java | 78 +++++++----
.../hbase/master/assignment/RegionStateStore.java | 72 ++++++-----
.../hadoop/hbase/master/store/LocalStore.java | 7 +
.../store/region/RegionProcedureStore.java | 5 +-
.../hbase/master/TestCloseAnOpeningRegion.java | 8 +-
.../hbase/master/TestClusterRestartFailover.java | 10 +-
.../master/assignment/MockMasterServices.java | 31 +++--
.../assignment/TestOpenRegionProcedureBackoff.java | 10 +-
.../assignment/TestOpenRegionProcedureHang.java | 10 +-
.../assignment/TestRaceBetweenSCPAndDTP.java | 10 +-
.../assignment/TestRaceBetweenSCPAndTRSP.java | 10 +-
.../TestRegionAssignedToMultipleRegionServers.java | 10 +-
.../assignment/TestReportOnlineRegionsRace.java | 10 +-
...tReportRegionStateTransitionFromDeadServer.java | 10 +-
.../TestReportRegionStateTransitionRetry.java | 10 +-
.../master/assignment/TestSCPGetRegionsRace.java | 10 +-
.../assignment/TestWakeUpUnexpectedProcedure.java | 10 +-
.../region/TestRegionProcedureStoreMigration.java | 2 +-
21 files changed, 361 insertions(+), 184 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index f0124d3..ed8de18 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -384,8 +384,8 @@ public class MetaTableAccessor {
}
/**
- * @return Deserialized values of <qualifier,regioninfo> pairs taken from
column values that match
- * the regex 'info:merge.*' in array of <code>cells</code>.
+ * @return Deserialized values of <qualifier,regioninfo> pairs taken
from column values that
+ * match the regex 'info:merge.*' in array of <code>cells</code>.
*/
@Nullable
public static Map<String, RegionInfo> getMergeRegionsWithName(Cell [] cells)
{
@@ -1403,6 +1403,21 @@ public class MetaTableAccessor {
}
}
+ public static Delete removeRegionReplica(byte[] metaRow, int
replicaIndexToDeleteFrom,
+ int numReplicasToRemove) {
+ int absoluteIndex = replicaIndexToDeleteFrom + numReplicasToRemove;
+ long now = EnvironmentEdgeManager.currentTime();
+ Delete deleteReplicaLocations = new Delete(metaRow);
+ for (int i = replicaIndexToDeleteFrom; i < absoluteIndex; i++) {
+ deleteReplicaLocations.addColumns(getCatalogFamily(),
getServerColumn(i), now);
+ deleteReplicaLocations.addColumns(getCatalogFamily(),
getSeqNumColumn(i), now);
+ deleteReplicaLocations.addColumns(getCatalogFamily(),
getStartCodeColumn(i), now);
+ deleteReplicaLocations.addColumns(getCatalogFamily(),
getServerNameColumn(i), now);
+ deleteReplicaLocations.addColumns(getCatalogFamily(),
getRegionStateColumn(i), now);
+ }
+ return deleteReplicaLocations;
+ }
+
/**
* Deletes some replica columns corresponding to replicas for the passed rows
* @param metaRows rows in hbase:meta
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 4b9cf73..4be605a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -58,6 +58,9 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellBuilderFactory;
+import org.apache.hadoop.hbase.CellBuilderType;
import org.apache.hadoop.hbase.ChoreService;
import org.apache.hadoop.hbase.ClusterId;
import org.apache.hadoop.hbase.ClusterMetrics;
@@ -81,9 +84,11 @@ import org.apache.hadoop.hbase.UnknownRegionException;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.MasterSwitchType;
+import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.RegionStatesCount;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.client.TableState;
@@ -100,6 +105,7 @@ import
org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.master.assignment.MergeTableRegionsProcedure;
import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
+import org.apache.hadoop.hbase.master.assignment.RegionStateStore;
import org.apache.hadoop.hbase.master.assignment.RegionStates;
import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
import org.apache.hadoop.hbase.master.balancer.BalancerChore;
@@ -176,6 +182,7 @@ import
org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshotNotifierFactory;
import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -210,6 +217,7 @@ import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
+import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker;
import org.apache.hadoop.hbase.zookeeper.SnapshotCleanupTracker;
import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
@@ -450,7 +458,7 @@ public class HMaster extends HRegionServer implements
MasterServices {
private ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
private ProcedureStore procedureStore;
- // the master local storage to store procedure data, etc.
+ // the master local storage to store procedure data, root table, etc.
private LocalStore localStore;
// handle table states
@@ -866,8 +874,50 @@ public class HMaster extends HRegionServer implements
MasterServices {
// Will be overriden in test to inject customized AssignmentManager
@VisibleForTesting
- protected AssignmentManager createAssignmentManager(MasterServices master) {
- return new AssignmentManager(master);
+ protected AssignmentManager createAssignmentManager(MasterServices master,
+ LocalStore localStore) {
+ return new AssignmentManager(master, localStore);
+ }
+
+ private void tryMigrateRootTableFromZooKeeper() throws IOException,
KeeperException {
+ // try migrate data from zookeeper
+ try (RegionScanner scanner =
+ localStore.getScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
+ List<Cell> cells = new ArrayList<>();
+ boolean moreRows = scanner.next(cells);
+ if (!cells.isEmpty() || moreRows) {
+ // notice that all replicas for a region are in the same row, so the
migration can be
+ // done with in a one row put, which means if we have data in root
table then we can make
+ // sure that the migration is done.
+ LOG.info("Root table already has data in it, skip migrating...");
+ return;
+ }
+ }
+ // start migrating
+ byte[] row =
MetaTableAccessor.getMetaKeyForRegion(RegionInfoBuilder.FIRST_META_REGIONINFO);
+ Put put = new Put(row);
+ List<String> metaReplicaNodes = zooKeeper.getMetaReplicaNodes();
+ StringBuilder info = new StringBuilder("Migrating meta location:");
+ for (String metaReplicaNode : metaReplicaNodes) {
+ int replicaId =
zooKeeper.getZNodePaths().getMetaReplicaIdFromZnode(metaReplicaNode);
+ RegionState state = MetaTableLocator.getMetaRegionState(zooKeeper,
replicaId);
+ info.append(" ").append(state);
+ put.setTimestamp(state.getStamp());
+ MetaTableAccessor.addRegionInfo(put, state.getRegion());
+ if (state.getServerName() != null) {
+ MetaTableAccessor.addLocation(put, state.getServerName(),
HConstants.NO_SEQNUM, replicaId);
+ }
+
put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(put.getRow())
+ .setFamily(HConstants.CATALOG_FAMILY)
+
.setQualifier(RegionStateStore.getStateColumn(replicaId)).setTimestamp(put.getTimestamp())
+
.setType(Cell.Type.Put).setValue(Bytes.toBytes(state.getState().name())).build());
+ }
+ if (!put.isEmpty()) {
+ LOG.info(info.toString());
+ localStore.update(r -> r.put(put));
+ } else {
+ LOG.info("No meta location avaiable on zookeeper, skip migrating...");
+ }
}
/**
@@ -883,6 +933,7 @@ public class HMaster extends HRegionServer implements
MasterServices {
* region server tracker
* <ol type='i'>
* <li>Create server manager</li>
+ * <li>Create root table</li>
* <li>Create procedure executor, load the procedures, but do not start
workers. We will start it
* later after we finish scheduling SCPs to avoid scheduling duplicated SCPs
for the same
* server</li>
@@ -967,13 +1018,16 @@ public class HMaster extends HRegionServer implements
MasterServices {
// initialize local store
localStore = LocalStore.create(this);
+
+ tryMigrateRootTableFromZooKeeper();
+
createProcedureExecutor();
Map<Class<?>, List<Procedure<MasterProcedureEnv>>> procsByType =
procedureExecutor.getActiveProceduresNoCopy().stream()
.collect(Collectors.groupingBy(p -> p.getClass()));
// Create Assignment Manager
- this.assignmentManager = createAssignmentManager(this);
+ this.assignmentManager = createAssignmentManager(this, localStore);
this.assignmentManager.start();
// TODO: TRSP can perform as the sub procedure for other procedures, so
even if it is marked as
// completed, it could still be in the procedure list. This is a bit
strange but is another
@@ -1318,18 +1372,15 @@ public class HMaster extends HRegionServer implements
MasterServices {
}
/**
- * <p>
* Create a {@link MasterMetaBootstrap} instance.
- * </p>
- * <p>
+ * <p/>
* Will be overridden in tests.
- * </p>
*/
@VisibleForTesting
protected MasterMetaBootstrap createMetaBootstrap() {
// We put this out here in a method so can do a Mockito.spy and stub it out
// w/ a mocked up MasterMetaBootstrap.
- return new MasterMetaBootstrap(this);
+ return new MasterMetaBootstrap(this, localStore);
}
/**
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMetaBootstrap.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMetaBootstrap.java
index 6e38bdd..5745581 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMetaBootstrap.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMetaBootstrap.java
@@ -19,14 +19,20 @@
package org.apache.hadoop.hbase.master;
import java.io.IOException;
-import java.util.List;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.RegionInfo;
-import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
+import org.apache.hadoop.hbase.master.assignment.RegionStates;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
@@ -43,73 +49,103 @@ class MasterMetaBootstrap {
private final HMaster master;
- public MasterMetaBootstrap(HMaster master) {
+ private final LocalStore localStore;
+
+ public MasterMetaBootstrap(HMaster master, LocalStore localStore) {
this.master = master;
+ this.localStore = localStore;
}
/**
* For assigning hbase:meta replicas only.
- * TODO: The way this assign runs, nothing but chance to stop all replicas
showing up on same
- * server as the hbase:meta region.
*/
- void assignMetaReplicas()
- throws IOException, InterruptedException, KeeperException {
+ void assignMetaReplicas() throws IOException, InterruptedException,
KeeperException {
int numReplicas =
master.getConfiguration().getInt(HConstants.META_REPLICAS_NUM,
- HConstants.DEFAULT_META_REPLICA_NUM);
- if (numReplicas <= 1) {
- // No replicaas to assign. Return.
- return;
- }
- final AssignmentManager assignmentManager = master.getAssignmentManager();
- if (!assignmentManager.isMetaLoaded()) {
- throw new IllegalStateException("hbase:meta must be initialized first
before we can " +
- "assign out its replicas");
- }
- ServerName metaServername =
MetaTableLocator.getMetaRegionLocation(this.master.getZooKeeper());
- for (int i = 1; i < numReplicas; i++) {
- // Get current meta state for replica from zk.
- RegionState metaState =
MetaTableLocator.getMetaRegionState(master.getZooKeeper(), i);
- RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
- RegionInfoBuilder.FIRST_META_REGIONINFO, i);
- LOG.debug(hri.getRegionNameAsString() + " replica region state from
zookeeper=" + metaState);
- if (metaServername.equals(metaState.getServerName())) {
- metaState = null;
- LOG.info(hri.getRegionNameAsString() +
- " old location is same as current hbase:meta location; setting
location as null...");
+ HConstants.DEFAULT_META_REPLICA_NUM);
+ // only try to assign meta replicas when there are more than 1 replicas
+ if (numReplicas > 1) {
+ final AssignmentManager am = master.getAssignmentManager();
+ if (!am.isMetaLoaded()) {
+ throw new IllegalStateException(
+ "hbase:meta must be initialized first before we can " + "assign out
its replicas");
}
- // These assigns run inline. All is blocked till they complete. Only
interrupt is shutting
- // down hosting server which calls AM#stop.
- if (metaState != null && metaState.getServerName() != null) {
- // Try to retain old assignment.
- assignmentManager.assign(hri, metaState.getServerName());
- } else {
- assignmentManager.assign(hri);
+ RegionStates regionStates = am.getRegionStates();
+ for (RegionInfo regionInfo :
regionStates.getRegionsOfTable(TableName.META_TABLE_NAME)) {
+ if (!RegionReplicaUtil.isDefaultReplica(regionInfo)) {
+ continue;
+ }
+ RegionState regionState = regionStates.getRegionState(regionInfo);
+ Set<ServerName> metaServerNames = new HashSet<ServerName>();
+ if (regionState.getServerName() != null) {
+ metaServerNames.add(regionState.getServerName());
+ }
+ for (int i = 1; i < numReplicas; i++) {
+ RegionInfo secondaryRegionInfo =
RegionReplicaUtil.getRegionInfoForReplica(regionInfo, i);
+ RegionState secondaryRegionState =
regionStates.getRegionState(secondaryRegionInfo);
+ ServerName sn = null;
+ if (secondaryRegionState != null) {
+ sn = secondaryRegionState.getServerName();
+ if (sn != null && !metaServerNames.add(sn)) {
+ LOG.info("{} old location {} is same with other hbase:meta
replica location;" +
+ " setting location as null...",
secondaryRegionInfo.getRegionNameAsString(), sn);
+ sn = null;
+ }
+ }
+ // These assigns run inline. All is blocked till they complete. Only
interrupt is shutting
+ // down hosting server which calls AM#stop.
+ if (sn != null) {
+ am.assign(secondaryRegionInfo, sn);
+ } else {
+ am.assign(secondaryRegionInfo);
+ }
+ }
}
}
+ // always try to remomve excess meta replicas
unassignExcessMetaReplica(numReplicas);
}
private void unassignExcessMetaReplica(int numMetaReplicasConfigured) {
- final ZKWatcher zooKeeper = master.getZooKeeper();
- // unassign the unneeded replicas (for e.g., if the previous master was
configured
- // with a replication of 3 and now it is 2, we need to unassign the 1
unneeded replica)
- try {
- List<String> metaReplicaZnodes = zooKeeper.getMetaReplicaNodes();
- for (String metaReplicaZnode : metaReplicaZnodes) {
- int replicaId =
zooKeeper.getZNodePaths().getMetaReplicaIdFromZnode(metaReplicaZnode);
- if (replicaId >= numMetaReplicasConfigured) {
- RegionState r = MetaTableLocator.getMetaRegionState(zooKeeper,
replicaId);
- LOG.info("Closing excess replica of meta region " + r.getRegion());
- // send a close and wait for a max of 30 seconds
-
ServerManager.closeRegionSilentlyAndWait(master.getAsyncClusterConnection(),
- r.getServerName(), r.getRegion(), 30000);
- ZKUtil.deleteNode(zooKeeper,
zooKeeper.getZNodePaths().getZNodeForReplica(replicaId));
+ ZKWatcher zooKeeper = master.getZooKeeper();
+ AssignmentManager am = master.getAssignmentManager();
+ RegionStates regionStates = am.getRegionStates();
+ Map<RegionInfo, Integer> region2MaxReplicaId = new HashMap<>();
+ for (RegionInfo regionInfo :
regionStates.getRegionsOfTable(TableName.META_TABLE_NAME)) {
+ RegionInfo primaryRegionInfo =
RegionReplicaUtil.getRegionInfoForDefaultReplica(regionInfo);
+ region2MaxReplicaId.compute(primaryRegionInfo,
+ (k, v) -> v == null ? regionInfo.getReplicaId() : Math.max(v,
regionInfo.getReplicaId()));
+ if (regionInfo.getReplicaId() < numMetaReplicasConfigured) {
+ continue;
+ }
+ RegionState regionState = regionStates.getRegionState(regionInfo);
+ try {
+
ServerManager.closeRegionSilentlyAndWait(master.getAsyncClusterConnection(),
+ regionState.getServerName(), regionInfo, 30000);
+ if (regionInfo.isFirst()) {
+ // for compatibility, also try to remove the replicas on zk.
+ ZKUtil.deleteNode(zooKeeper,
+
zooKeeper.getZNodePaths().getZNodeForReplica(regionInfo.getReplicaId()));
}
+ } catch (Exception e) {
+ // ignore the exception since we don't want the master to be wedged
due to potential
+ // issues in the cleanup of the extra regions. We can do that cleanup
via hbck or manually
+ LOG.warn("Ignoring exception " + e);
}
- } catch (Exception ex) {
- // ignore the exception since we don't want the master to be wedged due
to potential
- // issues in the cleanup of the extra regions. We can do that cleanup
via hbck or manually
- LOG.warn("Ignoring exception " + ex);
+ regionStates.deleteRegion(regionInfo);
}
+ region2MaxReplicaId.forEach((regionInfo, maxReplicaId) -> {
+ if (maxReplicaId >= numMetaReplicasConfigured) {
+ byte[] metaRow = MetaTableAccessor.getMetaKeyForRegion(regionInfo);
+ Delete delete = MetaTableAccessor.removeRegionReplica(metaRow,
numMetaReplicasConfigured,
+ maxReplicaId - numMetaReplicasConfigured + 1);
+ try {
+ localStore.update(r -> r.delete(delete));
+ } catch (IOException e) {
+ // ignore the exception since we don't want the master to be wedged
due to potential
+ // issues in the cleanup of the extra regions. We can do that
cleanup via hbck or manually
+ LOG.warn("Ignoring exception " + e);
+ }
+ }
+ });
}
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
index ab473b7..41da457 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
@@ -32,8 +32,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
-
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HConstants;
@@ -46,8 +46,10 @@ import
org.apache.hadoop.hbase.client.DoNotRetryRegionException;
import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.RegionStatesCount;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.exceptions.UnexpectedStateException;
import org.apache.hadoop.hbase.favored.FavoredNodesManager;
@@ -66,11 +68,13 @@ import
org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler;
import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureInMemoryChore;
import org.apache.hadoop.hbase.procedure2.util.StringUtils;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.SequenceId;
import org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer;
import org.apache.hadoop.hbase.util.Bytes;
@@ -79,8 +83,6 @@ import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.VersionInfo;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -172,19 +174,22 @@ public class AssignmentManager {
private final int assignMaxAttempts;
private final int assignRetryImmediatelyMaxAttempts;
+ private final LocalStore localStore;
+
private final Object checkIfShouldMoveSystemRegionLock = new Object();
private Thread assignThread;
- public AssignmentManager(final MasterServices master) {
- this(master, new RegionStateStore(master));
+ public AssignmentManager(MasterServices master, LocalStore localStore) {
+ this(master, localStore, new RegionStateStore(master, localStore));
}
@VisibleForTesting
- AssignmentManager(final MasterServices master, final RegionStateStore
stateStore) {
+ AssignmentManager(MasterServices master, LocalStore localStore,
RegionStateStore stateStore) {
this.master = master;
this.regionStateStore = stateStore;
this.metrics = new MetricsAssignmentManager();
+ this.localStore = localStore;
final Configuration conf = master.getConfiguration();
@@ -225,23 +230,52 @@ public class AssignmentManager {
// Start the Assignment Thread
startAssignmentThread();
- // load meta region state
- ZKWatcher zkw = master.getZooKeeper();
- // it could be null in some tests
- if (zkw != null) {
- RegionState regionState = MetaTableLocator.getMetaRegionState(zkw);
- RegionStateNode regionNode =
-
regionStates.getOrCreateRegionStateNode(RegionInfoBuilder.FIRST_META_REGIONINFO);
- regionNode.lock();
- try {
- regionNode.setRegionLocation(regionState.getServerName());
- regionNode.setState(regionState.getState());
- if (regionNode.getProcedure() != null) {
- regionNode.getProcedure().stateLoaded(this, regionNode);
+ // load meta region states.
+ // notice that, here we will load all replicas, and in MasterMetaBootstrap
we may assign new
+ // replicas, or remove excess replicas.
+ try (RegionScanner scanner =
+ localStore.getScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
+ List<Cell> cells = new ArrayList<>();
+ boolean moreRows;
+ do {
+ moreRows = scanner.next(cells);
+ if (cells.isEmpty()) {
+ continue;
}
- setMetaAssigned(regionState.getRegion(), regionState.getState() ==
State.OPEN);
- } finally {
- regionNode.unlock();
+ Result result = Result.create(cells);
+ cells.clear();
+ RegionStateStore
+ .visitMetaEntry((r, regionInfo, state, regionLocation, lastHost,
openSeqNum) -> {
+ RegionStateNode regionNode =
regionStates.getOrCreateRegionStateNode(regionInfo);
+ regionNode.lock();
+ try {
+ regionNode.setState(state);
+ regionNode.setLastHost(lastHost);
+ regionNode.setRegionLocation(regionLocation);
+ regionNode.setOpenSeqNum(openSeqNum);
+ if (regionNode.getProcedure() != null) {
+ regionNode.getProcedure().stateLoaded(this, regionNode);
+ }
+ if (RegionReplicaUtil.isDefaultReplica(regionInfo)) {
+ setMetaAssigned(regionInfo, state == State.OPEN);
+ }
+ } finally {
+ regionNode.unlock();
+ }
+ if (regionInfo.isFirst()) {
+ // for compatibility, mirror the meta region state to zookeeper
+ try {
+ regionStateStore.mirrorMetaLocation(regionInfo,
regionLocation, state);
+ } catch (IOException e) {
+ LOG.warn("Failed to mirror region location for {} to zk",
+ regionNode.toShortString());
+ }
+ }
+ }, result);
+ } while (moreRows);
+ if (!regionStates.hasTableRegionStates(TableName.META_TABLE_NAME)) {
+ // no meta regions yet, create the region node for the first meta
region
+
regionStates.createRegionStateNode(RegionInfoBuilder.FIRST_META_REGIONINFO);
}
}
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
index ffce38c..53eb57c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
@@ -22,7 +22,6 @@ import java.util.Collections;
import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;
-
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellBuilderFactory;
import org.apache.hadoop.hbase.CellBuilderType;
@@ -40,6 +39,7 @@ import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionState.State;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.util.StringUtils;
import org.apache.hadoop.hbase.util.Bytes;
@@ -65,10 +65,14 @@ public class RegionStateStore {
private final MasterServices master;
- public RegionStateStore(final MasterServices master) {
+ private final LocalStore localStore;
+
+ public RegionStateStore(MasterServices master, LocalStore localStore) {
this.master = master;
+ this.localStore = localStore;
}
+ @FunctionalInterface
public interface RegionStateVisitor {
void visitRegionState(Result result, RegionInfo regionInfo, State state,
ServerName regionLocation, ServerName lastHost, long openSeqNum);
@@ -115,7 +119,7 @@ public class RegionStateStore {
}
}
- private void visitMetaEntry(final RegionStateVisitor visitor, final Result
result)
+ public static void visitMetaEntry(final RegionStateVisitor visitor, final
Result result)
throws IOException {
final RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
if (rl == null) return;
@@ -147,34 +151,14 @@ public class RegionStateStore {
}
void updateRegionLocation(RegionStateNode regionStateNode) throws
IOException {
- if (regionStateNode.getRegionInfo().isMetaRegion()) {
- updateMetaLocation(regionStateNode.getRegionInfo(),
regionStateNode.getRegionLocation(),
- regionStateNode.getState());
- } else {
- long openSeqNum = regionStateNode.getState() == State.OPEN ?
regionStateNode.getOpenSeqNum()
- : HConstants.NO_SEQNUM;
- updateUserRegionLocation(regionStateNode.getRegionInfo(),
regionStateNode.getState(),
- regionStateNode.getRegionLocation(), openSeqNum,
- // The regionStateNode may have no procedure in a test scenario; allow
for this.
- regionStateNode.getProcedure() != null ?
regionStateNode.getProcedure().getProcId()
- : Procedure.NO_PROC_ID);
- }
- }
-
- private void updateMetaLocation(RegionInfo regionInfo, ServerName
serverName, State state)
- throws IOException {
- try {
- MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName,
regionInfo.getReplicaId(),
- state);
- } catch (KeeperException e) {
- throw new IOException(e);
- }
- }
-
- private void updateUserRegionLocation(RegionInfo regionInfo, State state,
- ServerName regionLocation, long openSeqNum,
- long pid) throws IOException {
long time = EnvironmentEdgeManager.currentTime();
+ long openSeqNum = regionStateNode.getState() == State.OPEN ?
regionStateNode.getOpenSeqNum() :
+ HConstants.NO_SEQNUM;
+ RegionInfo regionInfo = regionStateNode.getRegionInfo();
+ State state = regionStateNode.getState();
+ ServerName regionLocation = regionStateNode.getRegionLocation();
+ TransitRegionStateProcedure rit = regionStateNode.getProcedure();
+ long pid = rit != null ? rit.getProcId() : Procedure.NO_PROC_ID;
final int replicaId = regionInfo.getReplicaId();
final Put put = new Put(MetaTableAccessor.getMetaKeyForRegion(regionInfo),
time);
MetaTableAccessor.addRegionInfo(put, regionInfo);
@@ -216,12 +200,32 @@ public class RegionStateStore {
.build());
LOG.info(info.toString());
updateRegionLocation(regionInfo, state, put);
+ if (regionInfo.isMetaRegion() && regionInfo.isFirst()) {
+ // mirror the meta location to zookeeper
+ mirrorMetaLocation(regionInfo, regionLocation, state);
+ }
}
- private void updateRegionLocation(RegionInfo regionInfo, State state, Put
put)
+ public void mirrorMetaLocation(RegionInfo regionInfo, ServerName serverName,
State state)
throws IOException {
- try (Table table =
master.getConnection().getTable(TableName.META_TABLE_NAME)) {
- table.put(put);
+ try {
+ MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName,
regionInfo.getReplicaId(),
+ state);
+ } catch (KeeperException e) {
+ throw new IOException(e);
+ }
+ }
+
+ private void updateRegionLocation(RegionInfo regionInfo, State state, Put
put)
+ throws IOException {
+ try {
+ if (regionInfo.isMetaRegion()) {
+ localStore.update(r -> r.put(put));
+ } else {
+ try (Table table =
master.getConnection().getTable(TableName.META_TABLE_NAME)) {
+ table.put(put);
+ }
+ }
} catch (IOException e) {
// TODO: Revist!!!! Means that if a server is loaded, then we will abort
our host!
// In tests we abort the Master!
@@ -326,7 +330,7 @@ public class RegionStateStore {
}
}
- private static byte[] getStateColumn(int replicaId) {
+ public static byte[] getStateColumn(int replicaId) {
return replicaId == 0
? HConstants.STATE_QUALIFIER
: Bytes.toBytes(HConstants.STATE_QUALIFIER_STR +
META_REPLICA_ID_DELIMITER
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/store/LocalStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/store/LocalStore.java
index d2c6884..589a562 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/store/LocalStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/store/LocalStore.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.master.store;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
@@ -28,6 +29,8 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HRegion.FlushResult;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
@@ -87,6 +90,10 @@ public final class LocalStore {
public static final byte[] PROC_FAMILY = Bytes.toBytes("proc");
private static final TableDescriptor TABLE_DESC =
TableDescriptorBuilder.newBuilder(TABLE_NAME)
+
.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(HConstants.CATALOG_FAMILY)
+ .setMaxVersions(HConstants.DEFAULT_HBASE_META_VERSIONS).setInMemory(true)
+
.setBlocksize(HConstants.DEFAULT_HBASE_META_BLOCK_SIZE).setBloomFilterType(BloomType.ROWCOL)
+ .setDataBlockEncoding(DataBlockEncoding.ROW_INDEX_V1).build())
.setColumnFamily(ColumnFamilyDescriptorBuilder.of(PROC_FAMILY)).build();
private final LocalRegion region;
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStore.java
index 399fc81..e0ca3ea 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStore.java
@@ -444,8 +444,9 @@ public class RegionProcedureStore extends
ProcedureStoreBase {
Cell cell = cells.get(0);
cells.clear();
if (cell.getValueLength() == 0) {
- localStore.update(r -> r
- .delete(new Delete(cell.getRowArray(), cell.getRowOffset(),
cell.getRowLength())));
+ localStore.update(
+ r -> r.delete(new Delete(cell.getRowArray(), cell.getRowOffset(),
cell.getRowLength())
+ .addFamily(PROC_FAMILY)));
}
}
} catch (IOException e) {
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCloseAnOpeningRegion.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCloseAnOpeningRegion.java
index 4922224..0f60b8e 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCloseAnOpeningRegion.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCloseAnOpeningRegion.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -69,12 +70,13 @@ public class TestCloseAnOpeningRegion {
}
@Override
- protected AssignmentManager createAssignmentManager(MasterServices master)
{
- return new AssignmentManager(master) {
+ protected AssignmentManager createAssignmentManager(MasterServices master,
+ LocalStore localStore) {
+ return new AssignmentManager(master, localStore) {
@Override
public ReportRegionStateTransitionResponse reportRegionStateTransition(
- ReportRegionStateTransitionRequest req) throws PleaseHoldException
{
+ ReportRegionStateTransitionRequest req) throws PleaseHoldException {
ReportRegionStateTransitionResponse resp =
super.reportRegionStateTransition(req);
TransitionCode code = req.getTransition(0).getTransitionCode();
if (code == TransitionCode.OPENED && ARRIVE != null) {
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java
index fb5a778..ef7ec57 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java
@@ -40,6 +40,7 @@ import
org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.master.assignment.ServerState;
import org.apache.hadoop.hbase.master.assignment.ServerStateNode;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.test.MetricsAssertHelper;
@@ -178,15 +179,16 @@ public class TestClusterRestartFailover extends
AbstractTestRestartCluster {
}
@Override
- protected AssignmentManager createAssignmentManager(MasterServices master)
{
- return new AssignmentManagerForTest(master);
+ protected AssignmentManager createAssignmentManager(MasterServices master,
+ LocalStore localStore) {
+ return new AssignmentManagerForTest(master, localStore);
}
}
private static final class AssignmentManagerForTest extends
AssignmentManager {
- public AssignmentManagerForTest(MasterServices master) {
- super(master);
+ public AssignmentManagerForTest(MasterServices master, LocalStore
localStore) {
+ super(master, localStore);
}
@Override
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
index 2d01565..d56c0dc 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
@@ -48,6 +48,7 @@ import
org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
@@ -81,6 +82,7 @@ public class MockMasterServices extends
MockNoopMasterServices {
private final MasterWalManager walManager;
private final AssignmentManager assignmentManager;
private final TableStateManager tableStateManager;
+ private final LocalStore localStore;
private MasterProcedureEnv procedureEnv;
private ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
@@ -100,18 +102,21 @@ public class MockMasterServices extends
MockNoopMasterServices {
Superusers.initialize(conf);
this.fileSystemManager = new MasterFileSystem(conf);
this.walManager = new MasterWalManager(this);
+ this.localStore = LocalStore.create(this);
// Mock an AM.
- this.assignmentManager = new AssignmentManager(this, new
MockRegionStateStore(this)) {
- @Override
- public boolean isTableEnabled(final TableName tableName) {
- return true;
- }
-
- @Override
- public boolean isTableDisabled(final TableName tableName) {
- return false;
- }
- };
+ this.assignmentManager =
+ new AssignmentManager(this, localStore, new MockRegionStateStore(this,
localStore)) {
+
+ @Override
+ public boolean isTableEnabled(final TableName tableName) {
+ return true;
+ }
+
+ @Override
+ public boolean isTableDisabled(final TableName tableName) {
+ return false;
+ }
+ };
this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
this.serverManager = new ServerManager(this);
this.tableStateManager = Mockito.mock(TableStateManager.class);
@@ -290,8 +295,8 @@ public class MockMasterServices extends
MockNoopMasterServices {
}
private static class MockRegionStateStore extends RegionStateStore {
- public MockRegionStateStore(final MasterServices master) {
- super(master);
+ public MockRegionStateStore(MasterServices master, LocalStore localStore) {
+ super(master, localStore);
}
@Override
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureBackoff.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureBackoff.java
index 4112da7..555c38a 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureBackoff.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureBackoff.java
@@ -32,6 +32,7 @@ import
org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -55,8 +56,8 @@ public class TestOpenRegionProcedureBackoff {
private static final class AssignmentManagerForTest extends
AssignmentManager {
- public AssignmentManagerForTest(MasterServices master) {
- super(master);
+ public AssignmentManagerForTest(MasterServices master, LocalStore
localStore) {
+ super(master, localStore);
}
@Override
@@ -75,8 +76,9 @@ public class TestOpenRegionProcedureBackoff {
}
@Override
- protected AssignmentManager createAssignmentManager(MasterServices master)
{
- return new AssignmentManagerForTest(master);
+ protected AssignmentManager createAssignmentManager(MasterServices master,
+ LocalStore localStore) {
+ return new AssignmentManagerForTest(master, localStore);
}
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureHang.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureHang.java
index a25368f..ab1d26d 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureHang.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureHang.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -74,8 +75,8 @@ public class TestOpenRegionProcedureHang {
private static final class AssignmentManagerForTest extends
AssignmentManager {
- public AssignmentManagerForTest(MasterServices master) {
- super(master);
+ public AssignmentManagerForTest(MasterServices master, LocalStore
localStore) {
+ super(master, localStore);
}
@Override
@@ -110,8 +111,9 @@ public class TestOpenRegionProcedureHang {
}
@Override
- protected AssignmentManager createAssignmentManager(MasterServices master)
{
- return new AssignmentManagerForTest(master);
+ protected AssignmentManager createAssignmentManager(MasterServices master,
+ LocalStore localStore) {
+ return new AssignmentManagerForTest(master, localStore);
}
@Override
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndDTP.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndDTP.java
index a268af9..37a952f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndDTP.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndDTP.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -69,8 +70,8 @@ public class TestRaceBetweenSCPAndDTP {
private static final class AssignmentManagerForTest extends
AssignmentManager {
- public AssignmentManagerForTest(MasterServices master) {
- super(master);
+ public AssignmentManagerForTest(MasterServices master, LocalStore
localStore) {
+ super(master,localStore);
}
@Override
@@ -95,8 +96,9 @@ public class TestRaceBetweenSCPAndDTP {
}
@Override
- protected AssignmentManager createAssignmentManager(MasterServices master)
{
- return new AssignmentManagerForTest(master);
+ protected AssignmentManager createAssignmentManager(MasterServices master,
+ LocalStore localStore) {
+ return new AssignmentManagerForTest(master, localStore);
}
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndTRSP.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndTRSP.java
index a1b25e0..ad41f46 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndTRSP.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndTRSP.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -69,8 +70,8 @@ public class TestRaceBetweenSCPAndTRSP {
private static final class AssignmentManagerForTest extends
AssignmentManager {
- public AssignmentManagerForTest(MasterServices master) {
- super(master);
+ public AssignmentManagerForTest(MasterServices master, LocalStore
localStore) {
+ super(master, localStore);
}
@Override
@@ -108,8 +109,9 @@ public class TestRaceBetweenSCPAndTRSP {
}
@Override
- protected AssignmentManager createAssignmentManager(MasterServices master)
{
- return new AssignmentManagerForTest(master);
+ protected AssignmentManager createAssignmentManager(MasterServices master,
+ LocalStore localStore) {
+ return new AssignmentManagerForTest(master, localStore);
}
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionAssignedToMultipleRegionServers.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionAssignedToMultipleRegionServers.java
index 44af256..d78420d 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionAssignedToMultipleRegionServers.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionAssignedToMultipleRegionServers.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -80,8 +81,8 @@ public class TestRegionAssignedToMultipleRegionServers {
private static final class AssignmentManagerForTest extends
AssignmentManager {
- public AssignmentManagerForTest(MasterServices master) {
- super(master);
+ public AssignmentManagerForTest(MasterServices master, LocalStore
localStore) {
+ super(master, localStore);
}
@Override
@@ -114,8 +115,9 @@ public class TestRegionAssignedToMultipleRegionServers {
}
@Override
- protected AssignmentManager createAssignmentManager(MasterServices master)
{
- return new AssignmentManagerForTest(master);
+ protected AssignmentManager createAssignmentManager(MasterServices master,
+ LocalStore localStore) {
+ return new AssignmentManagerForTest(master, localStore);
}
@Override
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportOnlineRegionsRace.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportOnlineRegionsRace.java
index 4dede89..51028b3 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportOnlineRegionsRace.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportOnlineRegionsRace.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -68,8 +69,8 @@ public class TestReportOnlineRegionsRace {
private static final class AssignmentManagerForTest extends
AssignmentManager {
- public AssignmentManagerForTest(MasterServices master) {
- super(master);
+ public AssignmentManagerForTest(MasterServices master, LocalStore
localStore) {
+ super(master, localStore);
}
@Override
@@ -110,8 +111,9 @@ public class TestReportOnlineRegionsRace {
}
@Override
- protected AssignmentManager createAssignmentManager(MasterServices master)
{
- return new AssignmentManagerForTest(master);
+ protected AssignmentManager createAssignmentManager(MasterServices master,
+ LocalStore localStore) {
+ return new AssignmentManagerForTest(master, localStore);
}
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionFromDeadServer.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionFromDeadServer.java
index 1de806f..32b7154 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionFromDeadServer.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionFromDeadServer.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -82,8 +83,8 @@ public class TestReportRegionStateTransitionFromDeadServer {
private static final class AssignmentManagerForTest extends
AssignmentManager {
- public AssignmentManagerForTest(MasterServices master) {
- super(master);
+ public AssignmentManagerForTest(MasterServices master, LocalStore
localStore) {
+ super(master, localStore);
}
@Override
@@ -121,8 +122,9 @@ public class TestReportRegionStateTransitionFromDeadServer {
}
@Override
- protected AssignmentManager createAssignmentManager(MasterServices master)
{
- return new AssignmentManagerForTest(master);
+ protected AssignmentManager createAssignmentManager(MasterServices master,
+ LocalStore localStore) {
+ return new AssignmentManagerForTest(master, localStore);
}
@Override
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionRetry.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionRetry.java
index 71c4693..5e61acb 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionRetry.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionRetry.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -60,8 +61,8 @@ public class TestReportRegionStateTransitionRetry {
private static final class AssignmentManagerForTest extends
AssignmentManager {
- public AssignmentManagerForTest(MasterServices master) {
- super(master);
+ public AssignmentManagerForTest(MasterServices master, LocalStore
localStore) {
+ super(master, localStore);
}
@Override
@@ -88,8 +89,9 @@ public class TestReportRegionStateTransitionRetry {
}
@Override
- protected AssignmentManager createAssignmentManager(MasterServices master)
{
- return new AssignmentManagerForTest(master);
+ protected AssignmentManager createAssignmentManager(MasterServices master,
+ LocalStore localStore) {
+ return new AssignmentManagerForTest(master, localStore);
}
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSCPGetRegionsRace.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSCPGetRegionsRace.java
index d676af9..0aed22f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSCPGetRegionsRace.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSCPGetRegionsRace.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -89,8 +90,8 @@ public class TestSCPGetRegionsRace {
private static final class AssignmentManagerForTest extends
AssignmentManager {
- public AssignmentManagerForTest(MasterServices master) {
- super(master);
+ public AssignmentManagerForTest(MasterServices master, LocalStore
localStore) {
+ super(master, localStore);
}
@Override
@@ -134,8 +135,9 @@ public class TestSCPGetRegionsRace {
}
@Override
- protected AssignmentManager createAssignmentManager(MasterServices master)
{
- return new AssignmentManagerForTest(master);
+ protected AssignmentManager createAssignmentManager(MasterServices master,
+ LocalStore localStore) {
+ return new AssignmentManagerForTest(master, localStore);
}
@Override
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestWakeUpUnexpectedProcedure.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestWakeUpUnexpectedProcedure.java
index 62e3161..9657d04 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestWakeUpUnexpectedProcedure.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestWakeUpUnexpectedProcedure.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.master.store.LocalStore;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.testclassification.LargeTests;
@@ -135,8 +136,8 @@ public class TestWakeUpUnexpectedProcedure {
private static final class AMForTest extends AssignmentManager {
- public AMForTest(MasterServices master) {
- super(master);
+ public AMForTest(MasterServices master, LocalStore localStore) {
+ super(master, localStore);
}
@Override
@@ -202,8 +203,9 @@ public class TestWakeUpUnexpectedProcedure {
}
@Override
- protected AssignmentManager createAssignmentManager(MasterServices master)
{
- return new AMForTest(master);
+ protected AssignmentManager createAssignmentManager(MasterServices master,
+ LocalStore localStore) {
+ return new AMForTest(master, localStore);
}
@Override
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStoreMigration.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStoreMigration.java
index effa751..489dba1 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStoreMigration.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStoreMigration.java
@@ -79,7 +79,7 @@ public class TestRegionProcedureStoreMigration {
Configuration conf = htu.getConfiguration();
conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, false);
// Runs on local filesystem. Test does not need sync. Turn off checks.
-
htu.getConfiguration().setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE,
false);
+ conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, false);
Path testDir = htu.getDataTestDir();
CommonFSUtils.setRootDir(conf, testDir);
walStore = new WALProcedureStore(conf, new LeaseRecovery() {
