This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-24950 in repository https://gitbox.apache.org/repos/asf/hbase.git
commit b16fcb52a2c639c4eb4c4a59be5f9d2b06f8d593 Author: Duo Zhang <[email protected]> AuthorDate: Thu Jun 25 23:17:03 2020 +0800 HBASE-24388 Store the locations of meta regions in master local store (#1746) --- .../apache/hadoop/hbase/CatalogFamilyFormat.java | 17 +++ .../org/apache/hadoop/hbase/master/HMaster.java | 64 ++++++++- .../hbase/master/assignment/AssignmentManager.java | 82 +++++++----- .../hbase/master/assignment/RegionStateStore.java | 143 ++++++++++++--------- .../hbase/master/region/MasterRegionFactory.java | 9 +- .../store/region/RegionProcedureStore.java | 5 +- .../client/TestFailedMetaReplicaAssigment.java | 11 +- .../hbase/master/TestCloseAnOpeningRegion.java | 8 +- .../hbase/master/TestClusterRestartFailover.java | 11 +- .../master/assignment/MockMasterServices.java | 12 +- .../assignment/TestOpenRegionProcedureBackoff.java | 10 +- .../assignment/TestOpenRegionProcedureHang.java | 10 +- .../assignment/TestRaceBetweenSCPAndDTP.java | 10 +- .../assignment/TestRaceBetweenSCPAndTRSP.java | 10 +- .../TestRegionAssignedToMultipleRegionServers.java | 10 +- .../assignment/TestReportOnlineRegionsRace.java | 10 +- ...tReportRegionStateTransitionFromDeadServer.java | 10 +- .../TestReportRegionStateTransitionRetry.java | 10 +- .../master/assignment/TestSCPGetRegionsRace.java | 10 +- .../assignment/TestWakeUpUnexpectedProcedure.java | 10 +- .../region/TestRegionProcedureStoreMigration.java | 2 +- 21 files changed, 309 insertions(+), 155 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/CatalogFamilyFormat.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/CatalogFamilyFormat.java index 3cf6cc0..978198b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/CatalogFamilyFormat.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/CatalogFamilyFormat.java @@ -28,6 +28,7 @@ import java.util.NavigableMap; import java.util.SortedMap; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.RegionReplicaUtil; @@ -35,6 +36,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -401,4 +403,19 @@ public class CatalogFamilyFormat { return CellUtil.matchingFamily(cell, HConstants.CATALOG_FAMILY) && PrivateCellUtil.qualifierStartsWith(cell, HConstants.MERGE_QUALIFIER_PREFIX); } + + public static Delete removeRegionReplica(byte[] metaRow, int replicaIndexToDeleteFrom, + int numReplicasToRemove) { + int absoluteIndex = replicaIndexToDeleteFrom + numReplicasToRemove; + long now = EnvironmentEdgeManager.currentTime(); + Delete deleteReplicaLocations = new Delete(metaRow); + for (int i = replicaIndexToDeleteFrom; i < absoluteIndex; i++) { + deleteReplicaLocations.addColumns(HConstants.CATALOG_FAMILY, getServerColumn(i), now); + deleteReplicaLocations.addColumns(HConstants.CATALOG_FAMILY, getSeqNumColumn(i), now); + deleteReplicaLocations.addColumns(HConstants.CATALOG_FAMILY, getStartCodeColumn(i), now); + deleteReplicaLocations.addColumns(HConstants.CATALOG_FAMILY, getServerNameColumn(i), now); + deleteReplicaLocations.addColumns(HConstants.CATALOG_FAMILY, getRegionStateColumn(i), now); + } + return deleteReplicaLocations; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 1791ce4..961c929 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -54,6 +54,11 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CatalogFamilyFormat; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellBuilderFactory; +import org.apache.hadoop.hbase.CellBuilderType; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.ClusterId; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.ClusterMetrics.Option; @@ -80,9 +85,11 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.CompactionState; import org.apache.hadoop.hbase.client.MasterSwitchType; import org.apache.hadoop.hbase.client.NormalizeTableFilterParams; +import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.RegionStatesCount; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.TableState; @@ -99,6 +106,7 @@ import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode; import org.apache.hadoop.hbase.master.assignment.AssignmentManager; import org.apache.hadoop.hbase.master.assignment.MergeTableRegionsProcedure; import org.apache.hadoop.hbase.master.assignment.RegionStateNode; +import org.apache.hadoop.hbase.master.assignment.RegionStateStore; import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure; import org.apache.hadoop.hbase.master.balancer.BalancerChore; @@ -179,6 +187,7 @@ import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; import org.apache.hadoop.hbase.regionserver.RSRpcServices; +import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.replication.ReplicationLoadSource; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; @@ -212,6 +221,7 @@ import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.VersionInfo; import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker; import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker; import org.apache.hadoop.hbase.zookeeper.SnapshotCleanupTracker; import org.apache.hadoop.hbase.zookeeper.ZKClusterId; @@ -393,7 +403,7 @@ public class HMaster extends HRegionServer implements MasterServices { private ProcedureExecutor<MasterProcedureEnv> procedureExecutor; private ProcedureStore procedureStore; - // the master local storage to store procedure data, etc. + // the master local storage to store procedure data, root table, etc. private MasterRegion masterRegion; // handle table states @@ -758,8 +768,50 @@ public class HMaster extends HRegionServer implements MasterServices { // Will be overriden in test to inject customized AssignmentManager @InterfaceAudience.Private - protected AssignmentManager createAssignmentManager(MasterServices master) { - return new AssignmentManager(master); + protected AssignmentManager createAssignmentManager(MasterServices master, + MasterRegion masterRegion) { + return new AssignmentManager(master, masterRegion); + } + + private void tryMigrateRootTableFromZooKeeper() throws IOException, KeeperException { + // try migrate data from zookeeper + try (RegionScanner scanner = + masterRegion.getScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) { + List<Cell> cells = new ArrayList<>(); + boolean moreRows = scanner.next(cells); + if (!cells.isEmpty() || moreRows) { + // notice that all replicas for a region are in the same row, so the migration can be + // done with in a one row put, which means if we have data in root table then we can make + // sure that the migration is done. + LOG.info("Root table already has data in it, skip migrating..."); + return; + } + } + // start migrating + byte[] row = CatalogFamilyFormat.getMetaKeyForRegion(RegionInfoBuilder.FIRST_META_REGIONINFO); + Put put = new Put(row); + List<String> metaReplicaNodes = zooKeeper.getMetaReplicaNodes(); + StringBuilder info = new StringBuilder("Migrating meta location:"); + for (String metaReplicaNode : metaReplicaNodes) { + int replicaId = zooKeeper.getZNodePaths().getMetaReplicaIdFromZNode(metaReplicaNode); + RegionState state = MetaTableLocator.getMetaRegionState(zooKeeper, replicaId); + info.append(" ").append(state); + put.setTimestamp(state.getStamp()); + MetaTableAccessor.addRegionInfo(put, state.getRegion()); + if (state.getServerName() != null) { + MetaTableAccessor.addLocation(put, state.getServerName(), HConstants.NO_SEQNUM, replicaId); + } + put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(put.getRow()) + .setFamily(HConstants.CATALOG_FAMILY) + .setQualifier(RegionStateStore.getStateColumn(replicaId)).setTimestamp(put.getTimestamp()) + .setType(Cell.Type.Put).setValue(Bytes.toBytes(state.getState().name())).build()); + } + if (!put.isEmpty()) { + LOG.info(info.toString()); + masterRegion.update(r -> r.put(put)); + } else { + LOG.info("No meta location avaiable on zookeeper, skip migrating..."); + } } /** @@ -775,6 +827,7 @@ public class HMaster extends HRegionServer implements MasterServices { * region server tracker * <ol type='i'> * <li>Create server manager</li> + * <li>Create root table</li> * <li>Create procedure executor, load the procedures, but do not start workers. We will start it * later after we finish scheduling SCPs to avoid scheduling duplicated SCPs for the same * server</li> @@ -856,13 +909,16 @@ public class HMaster extends HRegionServer implements MasterServices { // initialize master local region masterRegion = MasterRegionFactory.create(this); + + tryMigrateRootTableFromZooKeeper(); + createProcedureExecutor(); Map<Class<?>, List<Procedure<MasterProcedureEnv>>> procsByType = procedureExecutor.getActiveProceduresNoCopy().stream() .collect(Collectors.groupingBy(p -> p.getClass())); // Create Assignment Manager - this.assignmentManager = createAssignmentManager(this); + this.assignmentManager = createAssignmentManager(this, masterRegion); this.assignmentManager.start(); // TODO: TRSP can perform as the sub procedure for other procedures, so even if it is marked as // completed, it could still be in the procedure list. This is a bit strange but is another diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index 5622a58..73b6aa6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -34,6 +34,7 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CatalogFamilyFormat; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HConstants; @@ -48,6 +49,7 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.RegionStatesCount; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.exceptions.UnexpectedStateException; @@ -67,11 +69,13 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler; import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait; import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; +import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureEvent; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureInMemoryChore; import org.apache.hadoop.hbase.procedure2.util.StringUtils; +import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.regionserver.SequenceId; import org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer; import org.apache.hadoop.hbase.util.Bytes; @@ -79,8 +83,6 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.VersionInfo; -import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; -import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.yetus.audience.InterfaceAudience; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -196,18 +198,21 @@ public class AssignmentManager { private final int assignMaxAttempts; private final int assignRetryImmediatelyMaxAttempts; + private final MasterRegion masterRegion; + private final Object checkIfShouldMoveSystemRegionLock = new Object(); private Thread assignThread; - public AssignmentManager(final MasterServices master) { - this(master, new RegionStateStore(master)); + public AssignmentManager(MasterServices master, MasterRegion masterRegion) { + this(master, masterRegion, new RegionStateStore(master, masterRegion)); } - AssignmentManager(final MasterServices master, final RegionStateStore stateStore) { + AssignmentManager(MasterServices master, MasterRegion masterRegion, RegionStateStore stateStore) { this.master = master; this.regionStateStore = stateStore; this.metrics = new MetricsAssignmentManager(); + this.masterRegion = masterRegion; final Configuration conf = master.getConfiguration(); @@ -249,34 +254,47 @@ public class AssignmentManager { // Start the Assignment Thread startAssignmentThread(); + // load meta region states. + try (RegionScanner scanner = + masterRegion.getScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) { + List<Cell> cells = new ArrayList<>(); + boolean moreRows; + do { + moreRows = scanner.next(cells); + if (cells.isEmpty()) { + continue; + } + Result result = Result.create(cells); + cells.clear(); + RegionStateStore + .visitMetaEntry((r, regionInfo, state, regionLocation, lastHost, openSeqNum) -> { + RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionInfo); + regionNode.setState(state); + regionNode.setLastHost(lastHost); + regionNode.setRegionLocation(regionLocation); + regionNode.setOpenSeqNum(openSeqNum); + if (regionNode.getProcedure() != null) { + regionNode.getProcedure().stateLoaded(this, regionNode); + } + if (regionLocation != null) { + regionStates.addRegionToServer(regionNode); + } + if (RegionReplicaUtil.isDefaultReplica(regionInfo)) { + setMetaAssigned(regionInfo, state == State.OPEN); + } - // load meta region state - ZKWatcher zkw = master.getZooKeeper(); - // it could be null in some tests - if (zkw == null) { - return; - } - List<String> metaZNodes = zkw.getMetaReplicaNodes(); - LOG.debug("hbase:meta replica znodes: {}", metaZNodes); - for (String metaZNode : metaZNodes) { - int replicaId = zkw.getZNodePaths().getMetaReplicaIdFromZNode(metaZNode); - // here we are still in the early steps of active master startup. There is only one thread(us) - // can access AssignmentManager and create region node, so here we do not need to lock the - // region node. - RegionState regionState = MetaTableLocator.getMetaRegionState(zkw, replicaId); - RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionState.getRegion()); - regionNode.setRegionLocation(regionState.getServerName()); - regionNode.setState(regionState.getState()); - if (regionNode.getProcedure() != null) { - regionNode.getProcedure().stateLoaded(this, regionNode); - } - if (regionState.getServerName() != null) { - regionStates.addRegionToServer(regionNode); - } - if (RegionReplicaUtil.isDefaultReplica(replicaId)) { - setMetaAssigned(regionState.getRegion(), regionState.getState() == State.OPEN); - } - LOG.debug("Loaded hbase:meta {}", regionNode); + if (regionInfo.isFirst()) { + // for compatibility, mirror the meta region state to zookeeper + try { + regionStateStore.mirrorMetaLocation(regionInfo, regionLocation, state); + } catch (IOException e) { + LOG.warn("Failed to mirror region location for {} to zk", + regionNode.toShortString()); + } + } + LOG.debug("Loaded hbase:meta {}", regionNode); + }, result); + } while (moreRows); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java index 8818067..87c04da 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java @@ -56,8 +56,10 @@ import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.RegionState.State; +import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.util.StringUtils; +import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.replication.ReplicationBarrierFamilyFormat; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -92,10 +94,14 @@ public class RegionStateStore { private final MasterServices master; - public RegionStateStore(final MasterServices master) { + private final MasterRegion masterRegion; + + public RegionStateStore(MasterServices master, MasterRegion masterRegion) { this.master = master; + this.masterRegion = masterRegion; } + @FunctionalInterface public interface RegionStateVisitor { void visitRegionState(Result result, RegionInfo regionInfo, State state, ServerName regionLocation, ServerName lastHost, long openSeqNum); @@ -142,8 +148,8 @@ public class RegionStateStore { } } - private void visitMetaEntry(final RegionStateVisitor visitor, final Result result) - throws IOException { + public static void visitMetaEntry(final RegionStateVisitor visitor, final Result result) + throws IOException { final RegionLocations rl = CatalogFamilyFormat.getRegionLocations(result); if (rl == null) return; @@ -173,33 +179,14 @@ public class RegionStateStore { } void updateRegionLocation(RegionStateNode regionStateNode) throws IOException { - if (regionStateNode.getRegionInfo().isMetaRegion()) { - updateMetaLocation(regionStateNode.getRegionInfo(), regionStateNode.getRegionLocation(), - regionStateNode.getState()); - } else { - long openSeqNum = regionStateNode.getState() == State.OPEN ? regionStateNode.getOpenSeqNum() : - HConstants.NO_SEQNUM; - updateUserRegionLocation(regionStateNode.getRegionInfo(), regionStateNode.getState(), - regionStateNode.getRegionLocation(), openSeqNum, - // The regionStateNode may have no procedure in a test scenario; allow for this. - regionStateNode.getProcedure() != null ? regionStateNode.getProcedure().getProcId() : - Procedure.NO_PROC_ID); - } - } - - private void updateMetaLocation(RegionInfo regionInfo, ServerName serverName, State state) - throws IOException { - try { - MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName, regionInfo.getReplicaId(), - state); - } catch (KeeperException e) { - throw new IOException(e); - } - } - - private void updateUserRegionLocation(RegionInfo regionInfo, State state, - ServerName regionLocation, long openSeqNum, long pid) throws IOException { long time = EnvironmentEdgeManager.currentTime(); + long openSeqNum = regionStateNode.getState() == State.OPEN ? regionStateNode.getOpenSeqNum() : + HConstants.NO_SEQNUM; + RegionInfo regionInfo = regionStateNode.getRegionInfo(); + State state = regionStateNode.getState(); + ServerName regionLocation = regionStateNode.getRegionLocation(); + TransitRegionStateProcedure rit = regionStateNode.getProcedure(); + long pid = rit != null ? rit.getProcId() : Procedure.NO_PROC_ID; final int replicaId = regionInfo.getReplicaId(); final Put put = new Put(CatalogFamilyFormat.getMetaKeyForRegion(regionInfo), time); MetaTableAccessor.addRegionInfo(put, regionInfo); @@ -234,12 +221,32 @@ public class RegionStateStore { .build()); LOG.info(info.toString()); updateRegionLocation(regionInfo, state, put); + if (regionInfo.isMetaRegion() && regionInfo.isFirst()) { + // mirror the meta location to zookeeper + mirrorMetaLocation(regionInfo, regionLocation, state); + } + } + + public void mirrorMetaLocation(RegionInfo regionInfo, ServerName serverName, State state) + throws IOException { + try { + MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName, regionInfo.getReplicaId(), + state); + } catch (KeeperException e) { + throw new IOException(e); + } } private void updateRegionLocation(RegionInfo regionInfo, State state, Put put) throws IOException { - try (Table table = master.getConnection().getTable(TableName.META_TABLE_NAME)) { - table.put(put); + try { + if (regionInfo.isMetaRegion()) { + masterRegion.update(r -> r.put(put)); + } else { + try (Table table = master.getConnection().getTable(TableName.META_TABLE_NAME)) { + table.put(put); + } + } } catch (IOException e) { // TODO: Revist!!!! Means that if a server is loaded, then we will abort our host! // In tests we abort the Master! @@ -541,45 +548,63 @@ public class RegionStateStore { .addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); } + private Delete deleteRegionReplicas(Result result, int oldReplicaCount, int newReplicaCount, + long now) { + RegionInfo primaryRegionInfo = CatalogFamilyFormat.getRegionInfo(result); + if (primaryRegionInfo == null || primaryRegionInfo.isSplitParent()) { + return null; + } + Delete delete = new Delete(result.getRow()); + for (int i = newReplicaCount; i < oldReplicaCount; i++) { + delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerColumn(i), now); + delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getSeqNumColumn(i), now); + delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getStartCodeColumn(i), now); + delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerNameColumn(i), now); + delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getRegionStateColumn(i), + now); + } + return delete; + } + public void removeRegionReplicas(TableName tableName, int oldReplicaCount, int newReplicaCount) throws IOException { + Scan scan = getScanForUpdateRegionReplicas(tableName); + List<Delete> deletes = new ArrayList<>(); + long now = EnvironmentEdgeManager.currentTime(); if (TableName.isMetaTableName(tableName)) { - ZKWatcher zk = master.getZooKeeper(); - try { - for (int i = newReplicaCount; i < oldReplicaCount; i++) { - ZKUtil.deleteNode(zk, zk.getZNodePaths().getZNodeForReplica(i)); - } - } catch (KeeperException e) { - throw new IOException(e); + try (RegionScanner scanner = masterRegion.getScanner(scan)) { + List<Cell> cells = new ArrayList<>(); + boolean moreRows; + do { + cells.clear(); + moreRows = scanner.next(cells); + if (cells.isEmpty()) { + continue; + } + Result result = Result.create(cells); + Delete delete = deleteRegionReplicas(result, oldReplicaCount, newReplicaCount, now); + if (delete != null) { + deletes.add(delete); + } + } while (moreRows); } + debugLogMutations(deletes); + masterRegion.update(r -> { + for (Delete d : deletes) { + r.delete(d); + } + }); } else { - Scan scan = getScanForUpdateRegionReplicas(tableName); - List<Delete> deletes = new ArrayList<>(); - long now = EnvironmentEdgeManager.currentTime(); try (Table metaTable = getMetaTable(); ResultScanner scanner = metaTable.getScanner(scan)) { for (;;) { Result result = scanner.next(); if (result == null) { break; } - RegionInfo primaryRegionInfo = CatalogFamilyFormat.getRegionInfo(result); - if (primaryRegionInfo == null || primaryRegionInfo.isSplitParent()) { - continue; - } - Delete delete = new Delete(result.getRow()); - for (int i = newReplicaCount; i < oldReplicaCount; i++) { - delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerColumn(i), - now); - delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getSeqNumColumn(i), - now); - delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getStartCodeColumn(i), - now); - delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerNameColumn(i), - now); - delete.addColumns(HConstants.CATALOG_FAMILY, - CatalogFamilyFormat.getRegionStateColumn(i), now); + Delete delete = deleteRegionReplicas(result, oldReplicaCount, newReplicaCount, now); + if (delete != null) { + deletes.add(delete); } - deletes.add(delete); } debugLogMutations(deletes); metaTable.delete(deletes); @@ -634,7 +659,7 @@ public class RegionStateStore { } } - private static byte[] getStateColumn(int replicaId) { + public static byte[] getStateColumn(int replicaId) { return replicaId == 0 ? HConstants.STATE_QUALIFIER : Bytes.toBytes(HConstants.STATE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFactory.java index f1da308..cfa25f5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFactory.java @@ -20,11 +20,14 @@ package org.apache.hadoop.hbase.master.region; import java.io.IOException; import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; @@ -80,6 +83,10 @@ public final class MasterRegionFactory { public static final byte[] PROC_FAMILY = Bytes.toBytes("proc"); private static final TableDescriptor TABLE_DESC = TableDescriptorBuilder.newBuilder(TABLE_NAME) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(HConstants.CATALOG_FAMILY) + .setMaxVersions(HConstants.DEFAULT_HBASE_META_VERSIONS).setInMemory(true) + .setBlocksize(HConstants.DEFAULT_HBASE_META_BLOCK_SIZE).setBloomFilterType(BloomType.ROWCOL) + .setDataBlockEncoding(DataBlockEncoding.ROW_INDEX_V1).build()) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(PROC_FAMILY)).build(); public static MasterRegion create(Server server) throws IOException { @@ -100,7 +107,7 @@ public final class MasterRegionFactory { params.ringBufferSlotCount(conf.getInt(RING_BUFFER_SLOT_COUNT, DEFAULT_RING_BUFFER_SLOT_COUNT)); long rollPeriodMs = conf.getLong(ROLL_PERIOD_MS_KEY, DEFAULT_ROLL_PERIOD_MS); params.rollPeriodMs(rollPeriodMs).archivedWalSuffix(ARCHIVED_WAL_SUFFIX) - .archivedHFileSuffix(ARCHIVED_HFILE_SUFFIX); + .archivedHFileSuffix(ARCHIVED_HFILE_SUFFIX).useMetaCellComparator(true); return MasterRegion.create(params); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStore.java index 301b605..2d07dae 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStore.java @@ -442,8 +442,9 @@ public class RegionProcedureStore extends ProcedureStoreBase { Cell cell = cells.get(0); cells.clear(); if (cell.getValueLength() == 0) { - region.update(r -> r - .delete(new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()))); + region.update( + r -> r.delete(new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()) + .addFamily(PROC_FAMILY))); } } } catch (IOException e) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFailedMetaReplicaAssigment.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFailedMetaReplicaAssigment.java index e336554..fde362c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFailedMetaReplicaAssigment.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFailedMetaReplicaAssigment.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.master.assignment.AssignmentManager; import org.apache.hadoop.hbase.master.assignment.RegionStateNode; import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; import org.apache.hadoop.hbase.procedure2.ProcedureYieldException; @@ -129,16 +130,18 @@ public class TestFailedMetaReplicaAssigment { } @Override - public AssignmentManager createAssignmentManager(MasterServices master) { - return new BrokenMasterMetaAssignmentManager(master); + public AssignmentManager createAssignmentManager(MasterServices master, + MasterRegion masterRegion) { + return new BrokenMasterMetaAssignmentManager(master, masterRegion); } } public static class BrokenMasterMetaAssignmentManager extends AssignmentManager { MasterServices master; - public BrokenMasterMetaAssignmentManager(final MasterServices master) { - super(master); + public BrokenMasterMetaAssignmentManager(final MasterServices master, + MasterRegion masterRegion) { + super(master, masterRegion); this.master = master; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCloseAnOpeningRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCloseAnOpeningRegion.java index a79e8f3..73ff415 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCloseAnOpeningRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCloseAnOpeningRegion.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -69,12 +70,13 @@ public class TestCloseAnOpeningRegion { } @Override - protected AssignmentManager createAssignmentManager(MasterServices master) { - return new AssignmentManager(master) { + protected AssignmentManager createAssignmentManager(MasterServices master, + MasterRegion masterRegion) { + return new AssignmentManager(master, masterRegion) { @Override public ReportRegionStateTransitionResponse reportRegionStateTransition( - ReportRegionStateTransitionRequest req) throws PleaseHoldException { + ReportRegionStateTransitionRequest req) throws PleaseHoldException { ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(req); TransitionCode code = req.getTransition(0).getTransitionCode(); if (code == TransitionCode.OPENED && ARRIVE != null) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java index f9300ed..c224440 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java @@ -27,7 +27,6 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CompatibilityFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -40,6 +39,7 @@ import org.apache.hadoop.hbase.master.assignment.AssignmentManager; import org.apache.hadoop.hbase.master.assignment.ServerState; import org.apache.hadoop.hbase.master.assignment.ServerStateNode; import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; +import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -166,15 +166,16 @@ public class TestClusterRestartFailover extends AbstractTestRestartCluster { } @Override - protected AssignmentManager createAssignmentManager(MasterServices master) { - return new AssignmentManagerForTest(master); + protected AssignmentManager createAssignmentManager(MasterServices master, + MasterRegion masterRegion) { + return new AssignmentManagerForTest(master, masterRegion); } } private static final class AssignmentManagerForTest extends AssignmentManager { - public AssignmentManagerForTest(MasterServices master) { - super(master); + public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) { + super(master, masterRegion); } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java index 445aeaf..d22040b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java @@ -50,6 +50,8 @@ import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher; +import org.apache.hadoop.hbase.master.region.MasterRegion; +import org.apache.hadoop.hbase.master.region.MasterRegionFactory; import org.apache.hadoop.hbase.procedure2.ProcedureEvent; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; @@ -85,6 +87,7 @@ public class MockMasterServices extends MockNoopMasterServices { private final SplitWALManager splitWALManager; private final AssignmentManager assignmentManager; private final TableStateManager tableStateManager; + private final MasterRegion masterRegion; private MasterProcedureEnv procedureEnv; private ProcedureExecutor<MasterProcedureEnv> procedureExecutor; @@ -107,9 +110,10 @@ public class MockMasterServices extends MockNoopMasterServices { this.splitWALManager = conf.getBoolean(HBASE_SPLIT_WAL_COORDINATED_BY_ZK, DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK)? null: new SplitWALManager(this); - + this.masterRegion = MasterRegionFactory.create(this); // Mock an AM. - this.assignmentManager = new AssignmentManager(this, new MockRegionStateStore(this)); + this.assignmentManager = + new AssignmentManager(this, masterRegion, new MockRegionStateStore(this, masterRegion)); this.balancer = LoadBalancerFactory.getLoadBalancer(conf); this.serverManager = new ServerManager(this); this.tableStateManager = Mockito.mock(TableStateManager.class); @@ -290,8 +294,8 @@ public class MockMasterServices extends MockNoopMasterServices { } private static class MockRegionStateStore extends RegionStateStore { - public MockRegionStateStore(final MasterServices master) { - super(master); + public MockRegionStateStore(MasterServices master, MasterRegion masterRegion) { + super(master, masterRegion); } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureBackoff.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureBackoff.java index 12ea426..4c0eac0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureBackoff.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureBackoff.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -55,8 +56,8 @@ public class TestOpenRegionProcedureBackoff { private static final class AssignmentManagerForTest extends AssignmentManager { - public AssignmentManagerForTest(MasterServices master) { - super(master); + public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) { + super(master, masterRegion); } @Override @@ -75,8 +76,9 @@ public class TestOpenRegionProcedureBackoff { } @Override - protected AssignmentManager createAssignmentManager(MasterServices master) { - return new AssignmentManagerForTest(master); + protected AssignmentManager createAssignmentManager(MasterServices master, + MasterRegion masterRegion) { + return new AssignmentManagerForTest(master, masterRegion); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureHang.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureHang.java index 6fd4fb2..93711ba 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureHang.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureHang.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -74,8 +75,8 @@ public class TestOpenRegionProcedureHang { private static final class AssignmentManagerForTest extends AssignmentManager { - public AssignmentManagerForTest(MasterServices master) { - super(master); + public AssignmentManagerForTest(MasterServices master,MasterRegion masterRegion) { + super(master, masterRegion); } @Override @@ -110,8 +111,9 @@ public class TestOpenRegionProcedureHang { } @Override - protected AssignmentManager createAssignmentManager(MasterServices master) { - return new AssignmentManagerForTest(master); + protected AssignmentManager createAssignmentManager(MasterServices master, + MasterRegion masterRegion) { + return new AssignmentManagerForTest(master, masterRegion); } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndDTP.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndDTP.java index cca5663..21fb63e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndDTP.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndDTP.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure; import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; +import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -69,8 +70,8 @@ public class TestRaceBetweenSCPAndDTP { private static final class AssignmentManagerForTest extends AssignmentManager { - public AssignmentManagerForTest(MasterServices master) { - super(master); + public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) { + super(master,masterRegion); } @Override @@ -95,8 +96,9 @@ public class TestRaceBetweenSCPAndDTP { } @Override - protected AssignmentManager createAssignmentManager(MasterServices master) { - return new AssignmentManagerForTest(master); + protected AssignmentManager createAssignmentManager(MasterServices master, + MasterRegion masterRegion) { + return new AssignmentManagerForTest(master, masterRegion); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndTRSP.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndTRSP.java index 788df3f..ae21c75 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndTRSP.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndTRSP.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; +import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -69,8 +70,8 @@ public class TestRaceBetweenSCPAndTRSP { private static final class AssignmentManagerForTest extends AssignmentManager { - public AssignmentManagerForTest(MasterServices master) { - super(master); + public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) { + super(master, masterRegion); } @Override @@ -108,8 +109,9 @@ public class TestRaceBetweenSCPAndTRSP { } @Override - protected AssignmentManager createAssignmentManager(MasterServices master) { - return new AssignmentManagerForTest(master); + protected AssignmentManager createAssignmentManager(MasterServices master, + MasterRegion masterRegion) { + return new AssignmentManagerForTest(master, masterRegion); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionAssignedToMultipleRegionServers.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionAssignedToMultipleRegionServers.java index da632d0..0cc510f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionAssignedToMultipleRegionServers.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionAssignedToMultipleRegionServers.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -80,8 +81,8 @@ public class TestRegionAssignedToMultipleRegionServers { private static final class AssignmentManagerForTest extends AssignmentManager { - public AssignmentManagerForTest(MasterServices master) { - super(master); + public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) { + super(master, masterRegion); } @Override @@ -114,8 +115,9 @@ public class TestRegionAssignedToMultipleRegionServers { } @Override - protected AssignmentManager createAssignmentManager(MasterServices master) { - return new AssignmentManagerForTest(master); + protected AssignmentManager createAssignmentManager(MasterServices master, + MasterRegion masterRegion) { + return new AssignmentManagerForTest(master, masterRegion); } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportOnlineRegionsRace.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportOnlineRegionsRace.java index b25b83d..f17c09d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportOnlineRegionsRace.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportOnlineRegionsRace.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -68,8 +69,8 @@ public class TestReportOnlineRegionsRace { private static final class AssignmentManagerForTest extends AssignmentManager { - public AssignmentManagerForTest(MasterServices master) { - super(master); + public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) { + super(master, masterRegion); } @Override @@ -110,8 +111,9 @@ public class TestReportOnlineRegionsRace { } @Override - protected AssignmentManager createAssignmentManager(MasterServices master) { - return new AssignmentManagerForTest(master); + protected AssignmentManager createAssignmentManager(MasterServices master, + MasterRegion masterRegion) { + return new AssignmentManagerForTest(master, masterRegion); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionFromDeadServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionFromDeadServer.java index 68d1e35..7fbf28b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionFromDeadServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionFromDeadServer.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -82,8 +83,8 @@ public class TestReportRegionStateTransitionFromDeadServer { private static final class AssignmentManagerForTest extends AssignmentManager { - public AssignmentManagerForTest(MasterServices master) { - super(master); + public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) { + super(master, masterRegion); } @Override @@ -121,8 +122,9 @@ public class TestReportRegionStateTransitionFromDeadServer { } @Override - protected AssignmentManager createAssignmentManager(MasterServices master) { - return new AssignmentManagerForTest(master); + protected AssignmentManager createAssignmentManager(MasterServices master, + MasterRegion masterRegion) { + return new AssignmentManagerForTest(master, masterRegion); } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionRetry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionRetry.java index f493892..1aa0f34 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionRetry.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionRetry.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -60,8 +61,8 @@ public class TestReportRegionStateTransitionRetry { private static final class AssignmentManagerForTest extends AssignmentManager { - public AssignmentManagerForTest(MasterServices master) { - super(master); + public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) { + super(master, masterRegion); } @Override @@ -88,8 +89,9 @@ public class TestReportRegionStateTransitionRetry { } @Override - protected AssignmentManager createAssignmentManager(MasterServices master) { - return new AssignmentManagerForTest(master); + protected AssignmentManager createAssignmentManager(MasterServices master, + MasterRegion masterRegion) { + return new AssignmentManagerForTest(master, masterRegion); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSCPGetRegionsRace.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSCPGetRegionsRace.java index 8b85c2d..c4ad67c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSCPGetRegionsRace.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSCPGetRegionsRace.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; +import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -89,8 +90,8 @@ public class TestSCPGetRegionsRace { private static final class AssignmentManagerForTest extends AssignmentManager { - public AssignmentManagerForTest(MasterServices master) { - super(master); + public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) { + super(master, masterRegion); } @Override @@ -134,8 +135,9 @@ public class TestSCPGetRegionsRace { } @Override - protected AssignmentManager createAssignmentManager(MasterServices master) { - return new AssignmentManagerForTest(master); + protected AssignmentManager createAssignmentManager(MasterServices master, + MasterRegion masterRegion) { + return new AssignmentManagerForTest(master, masterRegion); } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestWakeUpUnexpectedProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestWakeUpUnexpectedProcedure.java index 2a19b0a..ee01223 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestWakeUpUnexpectedProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestWakeUpUnexpectedProcedure.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.RSRpcServices; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -135,8 +136,8 @@ public class TestWakeUpUnexpectedProcedure { private static final class AMForTest extends AssignmentManager { - public AMForTest(MasterServices master) { - super(master); + public AMForTest(MasterServices master, MasterRegion masterRegion) { + super(master, masterRegion); } @Override @@ -202,8 +203,9 @@ public class TestWakeUpUnexpectedProcedure { } @Override - protected AssignmentManager createAssignmentManager(MasterServices master) { - return new AMForTest(master); + protected AssignmentManager createAssignmentManager(MasterServices master, + MasterRegion masterRegion) { + return new AMForTest(master, masterRegion); } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStoreMigration.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStoreMigration.java index 694ca32..cd39e99 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStoreMigration.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStoreMigration.java @@ -80,7 +80,7 @@ public class TestRegionProcedureStoreMigration { Configuration conf = htu.getConfiguration(); conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, false); // Runs on local filesystem. Test does not need sync. Turn off checks. - htu.getConfiguration().setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, false); + conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, false); Path testDir = htu.getDataTestDir(); CommonFSUtils.setRootDir(conf, testDir); walStore = new WALProcedureStore(conf, new LeaseRecovery() {
