http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index ed19dc9..b3ca94d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -18,6 +18,8 @@ */ package org.apache.hadoop.hbase.regionserver; +import com.google.common.annotations.VisibleForTesting; + import java.io.FileNotFoundException; import java.io.IOException; import java.io.InterruptedIOException; @@ -25,17 +27,8 @@ import java.net.BindException; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.NavigableMap; -import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -79,7 +72,6 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.VersionInfoUtil; import org.apache.hadoop.hbase.conf.ConfigurationObserver; import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException; -import org.apache.hadoop.hbase.exceptions.MergeRegionException; import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException; import org.apache.hadoop.hbase.exceptions.ScannerResetException; import org.apache.hadoop.hbase.filter.ByteArrayComparable; @@ -127,12 +119,12 @@ import org.apache.hadoop.hbase.shaded.protobuf.ResponseConverter; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitOrMergeRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitOrMergeResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionResponse; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresRequest; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest; @@ -145,8 +137,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerIn import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetStoreFileRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetStoreFileResponse; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.MergeRegionsRequest; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.MergeRegionsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionRequest.RegionOpenInfo; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionResponse; @@ -204,6 +194,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MapReduceProtos.ScanMet import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse.TableQuotaSnapshot; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceViolationPolicy; import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.RequestHeader; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; @@ -221,8 +212,6 @@ import org.apache.hadoop.hbase.wal.WALSplitter; import org.apache.hadoop.hbase.zookeeper.ZKSplitLog; import org.apache.zookeeper.KeeperException; -import com.google.common.annotations.VisibleForTesting; - /** * Implements the regionserver RPC services. */ @@ -1476,6 +1465,36 @@ public class RSRpcServices implements HBaseRPCErrorHandler, } } + @Override + @QosPriority(priority=HConstants.ADMIN_QOS) + public CloseRegionForSplitOrMergeResponse closeRegionForSplitOrMerge( + final RpcController controller, + final CloseRegionForSplitOrMergeRequest request) throws ServiceException { + try { + checkOpen(); + + List<String> encodedRegionNameList = new ArrayList<>(); + for(int i = 0; i < request.getRegionCount(); i++) { + final String encodedRegionName = ProtobufUtil.getRegionEncodedName(request.getRegion(i)); + + // Can be null if we're calling close on a region that's not online + final Region targetRegion = regionServer.getFromOnlineRegions(encodedRegionName); + if ((targetRegion != null) && (targetRegion.getCoprocessorHost() != null)) { + targetRegion.getCoprocessorHost().preClose(false); + encodedRegionNameList.add(encodedRegionName); + } + } + requestCount.increment(); + LOG.info("Close and offline " + encodedRegionNameList + " regions."); + boolean closed = regionServer.closeAndOfflineRegionForSplitOrMerge(encodedRegionNameList); + CloseRegionForSplitOrMergeResponse.Builder builder = + CloseRegionForSplitOrMergeResponse.newBuilder().setClosed(closed); + return builder.build(); + } catch (IOException ie) { + throw new ServiceException(ie); + } + } + /** * Compact a region on the region server. * @@ -1626,8 +1645,6 @@ public class RSRpcServices implements HBaseRPCErrorHandler, if (request.hasCompactionState() && request.getCompactionState()) { builder.setCompactionState(region.getCompactionState()); } - builder.setSplittable(region.isSplittable()); - builder.setMergeable(region.isMergeable()); builder.setIsRecovering(region.isRecovering()); return builder.build(); } catch (IOException ie) { @@ -1838,11 +1855,8 @@ public class RSRpcServices implements HBaseRPCErrorHandler, // The region is already online. This should not happen any more. String error = "Received OPEN for the region:" + region.getRegionNameAsString() + ", which is already online"; - LOG.warn(error); - //regionServer.abort(error); - //throw new IOException(error); - builder.addOpeningState(RegionOpeningState.OPENED); - continue; + regionServer.abort(error); + throw new IOException(error); } LOG.info("Open " + region.getRegionNameAsString()); @@ -3382,62 +3396,4 @@ public class RSRpcServices implements HBaseRPCErrorHandler, throw new ServiceException(e); } } - - @Override - public ExecuteProceduresResponse executeProcedures(RpcController controller, - ExecuteProceduresRequest request) throws ServiceException { - ExecuteProceduresResponse.Builder builder = ExecuteProceduresResponse.newBuilder(); - if (request.getOpenRegionCount() > 0) { - for (OpenRegionRequest req: request.getOpenRegionList()) { - builder.addOpenRegion(openRegion(controller, req)); - } - } - if (request.getCloseRegionCount() > 0) { - for (CloseRegionRequest req: request.getCloseRegionList()) { - builder.addCloseRegion(closeRegion(controller, req)); - } - } - return builder.build(); - } - - /** - * Merge regions on the region server. - * - * @param controller the RPC controller - * @param request the request - * @return merge regions response - * @throws ServiceException - */ - @Override - @QosPriority(priority = HConstants.ADMIN_QOS) - // UNUSED AS OF AMv2 PURGE! - public MergeRegionsResponse mergeRegions(final RpcController controller, - final MergeRegionsRequest request) throws ServiceException { - try { - checkOpen(); - requestCount.increment(); - Region regionA = getRegion(request.getRegionA()); - Region regionB = getRegion(request.getRegionB()); - boolean forcible = request.getForcible(); - long masterSystemTime = request.hasMasterSystemTime() ? request.getMasterSystemTime() : -1; - regionA.startRegionOperation(Operation.MERGE_REGION); - regionB.startRegionOperation(Operation.MERGE_REGION); - if (regionA.getRegionInfo().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID || - regionB.getRegionInfo().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) { - throw new ServiceException(new MergeRegionException("Can't merge non-default replicas")); - } - LOG.info("Receiving merging request for " + regionA + ", " + regionB - + ",forcible=" + forcible); - regionA.flush(true); - regionB.flush(true); - regionServer.compactSplitThread.requestRegionsMerge(regionA, regionB, forcible, - masterSystemTime, RpcServer.getRequestUser()); - return MergeRegionsResponse.newBuilder().build(); - } catch (DroppedSnapshotException ex) { - regionServer.abort("Replay of WAL required. Forcing server shutdown", ex); - throw new ServiceException(ex); - } catch (IOException ie) { - throw new ServiceException(ie); - } - } }
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java index 6c4eca9..295b825 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java @@ -96,14 +96,6 @@ public interface Region extends ConfigurationObserver { /** @return True if region is read only */ boolean isReadOnly(); - /** @return true if region is splittable */ - boolean isSplittable(); - - /** - * @return true if region is mergeable - */ - boolean isMergeable(); - /** * Return the list of Stores managed by this region * <p>Use with caution. Exposed for use of fixup utilities. http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeRequest.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeRequest.java deleted file mode 100644 index e95932b..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeRequest.java +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hbase.regionserver; - -import java.security.PrivilegedAction; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; - -import com.google.common.base.Preconditions; - -/** - * Handles processing region merges. Put in a queue, owned by HRegionServer. - */ -// UNUSED: REMOVE!!! [email protected] -class RegionMergeRequest implements Runnable { - private static final Log LOG = LogFactory.getLog(RegionMergeRequest.class); - private final HRegionInfo region_a; - private final HRegionInfo region_b; - private final HRegionServer server; - private final boolean forcible; - private final User user; - - RegionMergeRequest(Region a, Region b, HRegionServer hrs, boolean forcible, - long masterSystemTime, User user) { - Preconditions.checkNotNull(hrs); - this.region_a = a.getRegionInfo(); - this.region_b = b.getRegionInfo(); - this.server = hrs; - this.forcible = forcible; - this.user = user; - } - - @Override - public String toString() { - return "MergeRequest,regions:" + region_a + ", " + region_b + ", forcible=" - + forcible; - } - - private void doMerge() { - boolean success = false; - //server.metricsRegionServer.incrMergeRequest(); - - if (user != null && user.getUGI() != null) { - user.getUGI().doAs (new PrivilegedAction<Void>() { - @Override - public Void run() { - requestRegionMerge(); - return null; - } - }); - } else { - requestRegionMerge(); - } - } - - private void requestRegionMerge() { - final TableName table = region_a.getTable(); - if (!table.equals(region_b.getTable())) { - LOG.error("Can't merge regions from two different tables: " + region_a + ", " + region_b); - return; - } - - // TODO: fake merged region for compat with the report protocol - final HRegionInfo merged = new HRegionInfo(table); - - // Send the split request to the master. the master will do the validation on the split-key. - // The parent region will be unassigned and the two new regions will be assigned. - // hri_a and hri_b objects may not reflect the regions that will be created, those objectes - // are created just to pass the information to the reportRegionStateTransition(). - if (!server.reportRegionStateTransition(TransitionCode.READY_TO_MERGE, merged, region_a, region_b)) { - LOG.error("Unable to ask master to merge: " + region_a + ", " + region_b); - } - } - - @Override - public void run() { - if (this.server.isStopping() || this.server.isStopped()) { - LOG.debug("Skipping merge because server is stopping=" - + this.server.isStopping() + " or stopped=" + this.server.isStopped()); - return; - } - - doMerge(); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java index 5afa652..54aeaa6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java @@ -183,6 +183,16 @@ public interface RegionServerServices extends OnlineRegions, FavoredNodesForRegi boolean reportRegionStateTransition(TransitionCode code, HRegionInfo... hris); /** + * Notify master that a region wants to be splitted. + */ + long requestRegionSplit(final HRegionInfo regionInfo, final byte[] splitRow); + + /** + * Check with master whether a procedure is completed (either succeed or fail) + */ + boolean isProcedureFinished(final long procId) throws IOException; + + /** * Returns a reference to the region server's RPC server */ RpcServerInterface getRpcServer(); http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionUnassigner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionUnassigner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionUnassigner.java index 8eb78a2..b347b4b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionUnassigner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionUnassigner.java @@ -48,7 +48,8 @@ class RegionUnassigner { return; } unassigning = true; - new Thread("RegionUnassigner." + regionInfo.getEncodedName()) { + new Thread("Unassign-" + regionInfo) { + @Override public void run() { LOG.info("Unassign " + regionInfo.getRegionNameAsString()); @@ -64,4 +65,4 @@ class RegionUnassigner { } }.start(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java index bd59c53..eb9811d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java @@ -18,16 +18,16 @@ */ package org.apache.hadoop.hbase.regionserver; +import java.io.IOException; import java.security.PrivilegedAction; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.ipc.RemoteException; import com.google.common.base.Preconditions; @@ -37,14 +37,14 @@ import com.google.common.base.Preconditions; @InterfaceAudience.Private class SplitRequest implements Runnable { private static final Log LOG = LogFactory.getLog(SplitRequest.class); - private final HRegionInfo parent; + private final HRegion parent; private final byte[] midKey; private final HRegionServer server; private final User user; SplitRequest(Region region, byte[] midKey, HRegionServer hrs, User user) { Preconditions.checkNotNull(hrs); - this.parent = region.getRegionInfo(); + this.parent = (HRegion)region; this.midKey = midKey; this.server = hrs; this.user = user; @@ -56,30 +56,67 @@ class SplitRequest implements Runnable { } private void doSplitting() { + boolean success = false; server.metricsRegionServer.incrSplitRequest(); - if (user != null && user.getUGI() != null) { - user.getUGI().doAs (new PrivilegedAction<Void>() { - @Override - public Void run() { - requestRegionSplit(); - return null; + long startTime = EnvironmentEdgeManager.currentTime(); + + try { + long procId; + if (user != null && user.getUGI() != null) { + procId = user.getUGI().doAs (new PrivilegedAction<Long>() { + @Override + public Long run() { + try { + return server.requestRegionSplit(parent.getRegionInfo(), midKey); + } catch (Exception e) { + LOG.error("Failed to complete region split ", e); + } + return (long)-1; + } + }); + } else { + procId = server.requestRegionSplit(parent.getRegionInfo(), midKey); + } + + if (procId != -1) { + // wait for the split to complete or get interrupted. If the split completes successfully, + // the procedure will return true; if the split fails, the procedure would throw exception. + // + try { + while (!(success = server.isProcedureFinished(procId))) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + LOG.warn("Split region " + parent + " is still in progress. Not waiting..."); + break; + } + } + } catch (IOException e) { + LOG.error("Split region " + parent + " failed.", e); } - }); - } else { - requestRegionSplit(); - } - } + } else { + LOG.error("Fail to split region " + parent); + } + } finally { + if (this.parent.getCoprocessorHost() != null) { + try { + this.parent.getCoprocessorHost().postCompleteSplit(); + } catch (IOException io) { + LOG.error("Split failed " + this, + io instanceof RemoteException ? ((RemoteException) io).unwrapRemoteException() : io); + } + } + + // Update regionserver metrics with the split transaction total running time + server.metricsRegionServer.updateSplitTime(EnvironmentEdgeManager.currentTime() - startTime); + + if (parent.shouldForceSplit()) { + parent.clearSplit(); + } - private void requestRegionSplit() { - final TableName table = parent.getTable(); - final HRegionInfo hri_a = new HRegionInfo(table, parent.getStartKey(), midKey); - final HRegionInfo hri_b = new HRegionInfo(table, midKey, parent.getEndKey()); - // Send the split request to the master. the master will do the validation on the split-key. - // The parent region will be unassigned and the two new regions will be assigned. - // hri_a and hri_b objects may not reflect the regions that will be created, those objects - // are created just to pass the information to the reportRegionStateTransition(). - if (!server.reportRegionStateTransition(TransitionCode.READY_TO_SPLIT, parent, hri_a, hri_b)) { - LOG.error("Unable to ask master to split " + parent.getRegionNameAsString()); + if (success) { + server.metricsRegionServer.incrSplitSuccess(); + } } } @@ -93,4 +130,4 @@ class SplitRequest implements Runnable { doSplitting(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java index 3ecc750..5ff7a1e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java @@ -125,4 +125,4 @@ public class CloseRegionHandler extends EventHandler { remove(this.regionInfo.getEncodedNameAsBytes(), Boolean.FALSE); } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index f1e42a6..dca02e4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -559,7 +559,7 @@ public class HBaseFsck extends Configured implements Closeable { errors.print("Number of requests: " + status.getRequestsCount()); errors.print("Number of regions: " + status.getRegionsCount()); - List<RegionState> rits = status.getRegionsInTransition(); + Set<RegionState> rits = status.getRegionsInTransition(); errors.print("Number of regions in transition: " + rits.size()); if (details) { for (RegionState state: rits) { http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java index 8ea7012..d7749c2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java @@ -41,7 +41,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +import org.apache.hadoop.hbase.master.AssignmentManager; /** * Utility methods for interacting with the regions. @@ -223,7 +223,7 @@ public abstract class ModifyRegionUtils { static ThreadPoolExecutor getRegionOpenAndInitThreadPool(final Configuration conf, final String threadNamePrefix, int regionNumber) { int maxThreads = Math.min(regionNumber, conf.getInt( - "hbase.hregion.open.and.init.threads.max", 16)); + "hbase.hregion.open.and.init.threads.max", 10)); ThreadPoolExecutor regionOpenAndInitThreadPool = Threads .getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS, new ThreadFactory() { @@ -236,4 +236,24 @@ public abstract class ModifyRegionUtils { }); return regionOpenAndInitThreadPool; } + + /** + * Triggers a bulk assignment of the specified regions + * + * @param assignmentManager the Assignment Manger + * @param regionInfos the list of regions to assign + * @throws IOException if an error occurred during the assignment + */ + public static void assignRegions(final AssignmentManager assignmentManager, + final List<HRegionInfo> regionInfos) throws IOException { + try { + assignmentManager.getRegionStates().createRegionStates(regionInfos); + assignmentManager.assign(regionInfos); + } catch (InterruptedException e) { + LOG.error("Caught " + e + " during round-robin assignment"); + InterruptedIOException ie = new InterruptedIOException(e.getMessage()); + ie.initCause(e); + throw ie; + } + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java index 517a0cd..e8069ec 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java @@ -289,8 +289,8 @@ public class WALSplitter { this.fileBeingSplit = logfile; try { long logLength = logfile.getLen(); - LOG.info("Splitting WAL=" + logPath + ", length=" + logLength + - ", distributedLogReplay=" + this.distributedLogReplay); + LOG.info("Splitting wal: " + logPath + ", length=" + logLength); + LOG.info("DistributedLogReplay = " + this.distributedLogReplay); status.setStatus("Opening log file"); if (reporter != null && !reporter.progress()) { progress_failed = true; @@ -298,7 +298,7 @@ public class WALSplitter { } in = getReader(logfile, skipErrors, reporter); if (in == null) { - LOG.warn("Nothing to split in WAL=" + logPath); + LOG.warn("Nothing to split in log file " + logPath); return true; } int numOpenedFilesBeforeReporting = conf.getInt("hbase.splitlog.report.openedfiles", 3); @@ -377,7 +377,7 @@ public class WALSplitter { iie.initCause(ie); throw iie; } catch (CorruptedLogFileException e) { - LOG.warn("Could not parse, corrupted WAL=" + logPath, e); + LOG.warn("Could not parse, corrupted log file " + logPath, e); if (this.csm != null) { // Some tests pass in a csm of null. this.csm.getSplitLogWorkerCoordination().markCorrupted(rootDir, @@ -397,7 +397,7 @@ public class WALSplitter { in.close(); } } catch (IOException exception) { - LOG.warn("Could not close WAL reader: " + exception.getMessage()); + LOG.warn("Could not close wal reader: " + exception.getMessage()); LOG.debug("exception details", exception); } try { @@ -1595,10 +1595,8 @@ public class WALSplitter { if (wap == null) { wap = getWriterAndPath(logEntry); if (wap == null) { - if (LOG.isTraceEnabled()) { - // This log spews the full edit. Can be massive in the log. Enable only debugging - // WAL lost edit issues. - LOG.trace("getWriterAndPath decided we don't need to write edits for " + logEntry); + if (LOG.isDebugEnabled()) { + LOG.debug("getWriterAndPath decided we don't need to write edits for " + logEntry); } return; } http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java index a6a5c17..69cd233 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java @@ -88,8 +88,8 @@ public class RegionServerTracker extends ZooKeeperListener { int magicLen = ProtobufUtil.lengthOfPBMagic(); ProtobufUtil.mergeFrom(rsInfoBuilder, data, magicLen, data.length - magicLen); } - if (LOG.isTraceEnabled()) { - LOG.trace("Added tracking of RS " + nodePath); + if (LOG.isDebugEnabled()) { + LOG.debug("Added tracking of RS " + nodePath); } } catch (KeeperException e) { LOG.warn("Get Rs info port from ephemeral node", e); http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 5c8b29b..afc070d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -86,10 +86,10 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.ipc.RpcServerInterface; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; import org.apache.hadoop.hbase.mapreduce.MapreduceTestingShim; +import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.RegionStates; import org.apache.hadoop.hbase.master.ServerManager; -import org.apache.hadoop.hbase.master.assignment.AssignmentManager; -import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.ChunkCreator; import org.apache.hadoop.hbase.regionserver.HRegion; @@ -3323,14 +3323,13 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { public void moveRegionAndWait(HRegionInfo destRegion, ServerName destServer) throws InterruptedException, IOException { HMaster master = getMiniHBaseCluster().getMaster(); - // TODO: Here we start the move. The move can take a while. - getAdmin().move(destRegion.getEncodedNameAsBytes(), + getHBaseAdmin().move(destRegion.getEncodedNameAsBytes(), Bytes.toBytes(destServer.getServerName())); while (true) { ServerName serverName = master.getAssignmentManager().getRegionStates() .getRegionServerOfRegion(destRegion); if (serverName != null && serverName.equals(destServer)) { - assertRegionOnServer(destRegion, serverName, 2000); + assertRegionOnServer(destRegion, serverName, 200); break; } Thread.sleep(10); @@ -3995,7 +3994,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { if (master == null) return false; AssignmentManager am = master.getAssignmentManager(); if (am == null) return false; - return !am.hasRegionsInTransition(); + final RegionStates regionStates = am.getRegionStates(); + return !regionStates.isRegionsInTransition(); } }; } http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java index a99345b..eefde94 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java @@ -302,6 +302,16 @@ public class MockRegionServerServices implements RegionServerServices { } @Override + public long requestRegionSplit(final HRegionInfo regionInfo, final byte[] splitRow) { + return -1; + } + + @Override + public boolean isProcedureFinished(final long procId) { + return false; + } + + @Override public boolean registerService(Service service) { // TODO Auto-generated method stub return false; http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java index cff1a8d..283d79d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java @@ -21,18 +21,13 @@ package org.apache.hadoop.hbase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.master.RegionStates; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.testclassification.FlakeyTests; @@ -42,18 +37,21 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.Threads; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + /** * Test whether region re-balancing works. (HBASE-71) */ -@Ignore // This is broken since new RegionServers does proper average of regions -// and because Master is treated as a regionserver though it hosts two regions only. @Category({FlakeyTests.class, LargeTests.class}) @RunWith(value = Parameterized.class) public class TestRegionRebalancing { http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java index f84d9c2..7b69db4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java @@ -42,18 +42,23 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.InvalidFamilyOperationException; +import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotEnabledException; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.exceptions.MergeRegionException; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsRequest; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreFile; -import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsRequest; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -98,7 +103,7 @@ public class TestAdmin1 { @Before public void setUp() throws Exception { - this.admin = TEST_UTIL.getAdmin(); + this.admin = TEST_UTIL.getHBaseAdmin(); } @After @@ -746,7 +751,7 @@ public class TestAdmin1 { desc = new HTableDescriptor(TABLE_2); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); - admin = TEST_UTIL.getAdmin(); + admin = TEST_UTIL.getHBaseAdmin(); admin.createTable(desc, startKey, endKey, expectedRegions); try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(TABLE_2)) { @@ -801,7 +806,7 @@ public class TestAdmin1 { desc = new HTableDescriptor(TABLE_3); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); - admin = TEST_UTIL.getAdmin(); + admin = TEST_UTIL.getHBaseAdmin(); admin.createTable(desc, startKey, endKey, expectedRegions); @@ -987,7 +992,7 @@ public class TestAdmin1 { sb.append("_").append(Integer.toString(rowCounts[i])); } assertFalse(admin.tableExists(tableName)); - try (final Table table = TEST_UTIL.createTable(tableName, familyNames, + try(final Table table = TEST_UTIL.createTable(tableName, familyNames, numVersions, blockSize); final RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { @@ -1052,7 +1057,8 @@ public class TestAdmin1 { } catch (IOException e) { e.printStackTrace(); } - if (regions == null) continue; + if (regions == null) + continue; count.set(regions.size()); if (count.get() >= 2) { LOG.info("Found: " + regions); http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java index 607fc61..0014401 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java @@ -54,7 +54,7 @@ import org.apache.hadoop.hbase.TableNotEnabledException; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.constraint.ConstraintException; -import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.Region; @@ -528,6 +528,8 @@ public class TestAdmin2 { List<HRegionInfo> tableRegions = localAdmin.getTableRegions(tableName); HRegionInfo hri = tableRegions.get(0); AssignmentManager am = master.getAssignmentManager(); + assertTrue("Region " + hri.getRegionNameAsString() + + " should be assigned properly", am.waitForAssignment(hri)); ServerName server = am.getRegionStates().getRegionServerOfRegion(hri); localAdmin.move(hri.getEncodedNameAsBytes(), Bytes.toBytes(server.getServerName())); assertEquals("Current region server and region server before move should be same.", server, http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi.java index 074a421..04bd224 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Random; -import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -35,14 +34,14 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.NoSuchProcedureException; import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.master.RegionStates; import org.apache.hadoop.hbase.master.ServerManager; -import org.apache.hadoop.hbase.master.assignment.AssignmentManager; -import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; @@ -53,7 +52,6 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Threads; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -323,10 +321,17 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase { desc.addFamily(new HColumnDescriptor(FAMILY)); admin.createTable(desc).get(); + // add region to meta. + Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME); + HRegionInfo hri = + new HRegionInfo(desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z")); + MetaTableAccessor.addRegionToMeta(meta, hri); + // assign region. HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); AssignmentManager am = master.getAssignmentManager(); - HRegionInfo hri = am.getRegionStates().getRegionsOfTable(tableName).get(0); + admin.assign(hri.getRegionName()).get(); + am.waitForAssignment(hri); // assert region on server RegionStates regionStates = am.getRegionStates(); @@ -335,32 +340,22 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase { assertTrue(regionStates.getRegionState(hri).isOpened()); // Region is assigned now. Let's assign it again. - // Master should not abort, and region should stay assigned. + // Master should not abort, and region should be assigned. admin.assign(hri.getRegionName()).get(); - try { - am.waitForAssignment(hri); - fail("Expected NoSuchProcedureException"); - } catch (NoSuchProcedureException e) { - // Expected - } + am.waitForAssignment(hri); assertTrue(regionStates.getRegionState(hri).isOpened()); // unassign region admin.unassign(hri.getRegionName(), true).get(); - try { - am.waitForAssignment(hri); - fail("Expected NoSuchProcedureException"); - } catch (NoSuchProcedureException e) { - // Expected - } - assertTrue(regionStates.getRegionState(hri).isClosed()); + am.waitForAssignment(hri); + assertTrue(regionStates.getRegionState(hri).isOpened()); } finally { TEST_UTIL.deleteTable(tableName); } } HRegionInfo createTableAndGetOneRegion(final TableName tableName) - throws IOException, InterruptedException, ExecutionException { + throws IOException, InterruptedException { HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(FAMILY)); admin.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), 5); @@ -382,12 +377,7 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase { } } - @Ignore @Test - // Turning off this tests in AMv2. Doesn't make sense.Offlining means something - // different now. - // You can't 'offline' a region unless you know what you are doing - // Will cause the Master to tell the regionserver to shut itself down because - // regionserver is reporting the state as OPEN. + @Test public void testOfflineRegion() throws Exception { final TableName tableName = TableName.valueOf("testOfflineRegion"); try { @@ -395,6 +385,8 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase { RegionStates regionStates = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates(); + ServerName serverName = regionStates.getRegionServerOfRegion(hri); + TEST_UTIL.assertRegionOnServer(hri, serverName, 200); admin.offline(hri.getRegionName()).get(); long timeoutTime = System.currentTimeMillis() + 3000; @@ -450,7 +442,7 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase { if (now > timeoutTime) { fail("Failed to move the region in time: " + regionStates.getRegionState(hri)); } - regionStates.wait(50); + regionStates.waitForUpdate(50); } } finally { TEST_UTIL.deleteTable(tableName); @@ -474,9 +466,6 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase { regionServerCount.incrementAndGet(); }); Assert.assertEquals(regionServerCount.get(), 2); - } catch (Exception e) { - LOG.info("Exception", e); - throw e; } finally { TEST_UTIL.deleteTable(tableName); } http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreaded.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreaded.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreaded.java index 2abc54d..913c2e9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreaded.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreaded.java @@ -47,14 +47,12 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; /** * Will split the table, and move region randomly when testing. */ -@Ignore // Can't move hbase:meta off master server in AMv2. TODO. @Category({ LargeTests.class, ClientTests.class }) public class TestAsyncTableGetMultiThreaded { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreadedWithBasicCompaction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreadedWithBasicCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreadedWithBasicCompaction.java index b9f11d5..8743266 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreadedWithBasicCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreadedWithBasicCompaction.java @@ -17,14 +17,13 @@ */ package org.apache.hadoop.hbase.client; +import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.MemoryCompactionPolicy; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.experimental.categories.Category; -@Ignore // Can't move hbase:meta off master server in AMv2. TODO. @Category({ LargeTests.class, ClientTests.class }) public class TestAsyncTableGetMultiThreadedWithBasicCompaction extends TestAsyncTableGetMultiThreaded { @@ -33,4 +32,5 @@ public class TestAsyncTableGetMultiThreadedWithBasicCompaction extends public static void setUp() throws Exception { setUp(MemoryCompactionPolicy.BASIC); } -} \ No newline at end of file + +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreadedWithEagerCompaction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreadedWithEagerCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreadedWithEagerCompaction.java index dd94398..ef75373 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreadedWithEagerCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreadedWithEagerCompaction.java @@ -17,14 +17,13 @@ */ package org.apache.hadoop.hbase.client; +import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.MemoryCompactionPolicy; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.experimental.categories.Category; -@Ignore // Can't move hbase:meta off master server in AMv2. TODO. @Category({ LargeTests.class, ClientTests.class }) public class TestAsyncTableGetMultiThreadedWithEagerCompaction extends TestAsyncTableGetMultiThreaded { @@ -33,4 +32,5 @@ public class TestAsyncTableGetMultiThreadedWithEagerCompaction extends public static void setUp() throws Exception { setUp(MemoryCompactionPolicy.EAGER); } -} \ No newline at end of file + +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java index 6e50312..023095f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java @@ -593,14 +593,12 @@ public class TestBlockEvictionFromClient { put.addColumn(FAMILY, QUALIFIER2, data2); table.put(put); region.flush(true); - LOG.info("About to SPLIT on " + Bytes.toString(ROW1)); TEST_UTIL.getAdmin().split(tableName, ROW1); List<HRegionInfo> tableRegions = TEST_UTIL.getAdmin().getTableRegions(tableName); // Wait for splits while (tableRegions.size() != 2) { tableRegions = TEST_UTIL.getAdmin().getTableRegions(tableName); Thread.sleep(100); - LOG.info("Waiting on SPLIT to complete..."); } region.compact(true); Iterator<CachedBlock> iterator = cache.iterator(); http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestEnableTable.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestEnableTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestEnableTable.java index e1277fa..80b7208 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestEnableTable.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestEnableTable.java @@ -18,15 +18,14 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.io.IOException; import java.util.concurrent.CountDownLatch; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -37,11 +36,16 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.coprocessor.MasterObserver; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; -import org.apache.hadoop.hbase.coprocessor.MasterObserver; import org.apache.hadoop.hbase.coprocessor.ObserverContext; -import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -53,9 +57,9 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; @Category({ MasterTests.class, MediumTests.class }) public class TestEnableTable { @@ -101,17 +105,15 @@ public class TestEnableTable { rs.getRegionServer().stop("stop"); cluster.waitForRegionServerToStop(rs.getRegionServer().getServerName(), 10000); - // We used to enable the table here but AMv2 would hang waiting on a RS to check-in. - // Revisit. + LOG.debug("Now enabling table " + tableName); + + admin.enableTable(tableName); + assertTrue(admin.isTableEnabled(tableName)); JVMClusterUtil.RegionServerThread rs2 = cluster.startRegionServer(); cluster.waitForRegionServerToStart(rs2.getRegionServer().getServerName().getHostname(), rs2.getRegionServer().getServerName().getPort(), 60000); - LOG.debug("Now enabling table " + tableName); - admin.enableTable(tableName); - assertTrue(admin.isTableEnabled(tableName)); - List<HRegionInfo> regions = TEST_UTIL.getAdmin().getTableRegions(tableName); assertEquals(1, regions.size()); for (HRegionInfo region : regions) { http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java index 9eaa716..7f44a2a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java @@ -677,8 +677,6 @@ public class TestFromClientSide3 { }); fail("This cp should fail because the target lock is blocked by previous put"); } catch (Throwable ex) { - // TODO!!!! Is this right? It catches everything including the above fail - // if it happens (which it seems too....) } }); cpService.shutdown(); http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java index e99ee07..bfe10b5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java @@ -18,12 +18,7 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import com.google.common.collect.Lists; import java.io.IOException; import java.lang.reflect.Field; @@ -65,6 +60,7 @@ import org.apache.hadoop.hbase.filter.FilterBase; import org.apache.hadoop.hbase.ipc.HBaseRpcController; import org.apache.hadoop.hbase.ipc.RpcClient; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; +import org.apache.hadoop.hbase.ipc.ServerTooBusyException; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.Region; @@ -86,7 +82,12 @@ import org.junit.experimental.categories.Category; import org.junit.rules.TestName; import org.junit.rules.TestRule; -import com.google.common.collect.Lists; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * This class is for testing HBaseConnectionManager features @@ -230,6 +231,8 @@ public class TestHCM { TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, RPC_RETRY); // simulate queue blocking in testDropTimeoutRequest TEST_UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HANDLER_COUNT, 1); + // Used in testServerBusyException + TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_PERSERVER_REQUESTS_THRESHOLD, 3); TEST_UTIL.startMiniCluster(2); } @@ -967,7 +970,7 @@ public class TestHCM { * that we really delete it. * @throws Exception */ - @Ignore @Test + @Test public void testRegionCaching() throws Exception{ TEST_UTIL.createMultiRegionTable(TABLE_NAME, FAM_NAM).close(); Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); @@ -1033,7 +1036,7 @@ public class TestHCM { Assert.assertNotNull(curServer.getOnlineRegion(regionName)); Assert.assertNull(destServer.getOnlineRegion(regionName)); Assert.assertFalse(TEST_UTIL.getMiniHBaseCluster().getMaster(). - getAssignmentManager().hasRegionsInTransition()); + getAssignmentManager().getRegionStates().isRegionsInTransition()); // Moving. It's possible that we don't have all the regions online at this point, so // the test must depends only on the region we're looking at. @@ -1046,7 +1049,7 @@ public class TestHCM { while (destServer.getOnlineRegion(regionName) == null || destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || - master.getAssignmentManager().hasRegionsInTransition()) { + master.getAssignmentManager().getRegionStates().isRegionsInTransition()) { // wait for the move to be finished Thread.sleep(1); } @@ -1105,7 +1108,7 @@ public class TestHCM { while (curServer.getOnlineRegion(regionName) == null || destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || - master.getAssignmentManager().hasRegionsInTransition()) { + master.getAssignmentManager().getRegionStates().isRegionsInTransition()) { // wait for the move to be finished Thread.sleep(1); } @@ -1290,7 +1293,7 @@ public class TestHCM { return prevNumRetriesVal; } - @Ignore @Test + @Test public void testMulti() throws Exception { Table table = TEST_UTIL.createMultiRegionTable(TABLE_NAME3, FAM_NAM); try { @@ -1325,8 +1328,6 @@ public class TestHCM { HRegionServer destServer = TEST_UTIL.getHBaseCluster().getRegionServer(destServerId); ServerName destServerName = destServer.getServerName(); - ServerName metaServerName = TEST_UTIL.getHBaseCluster().getServerHoldingMeta(); - assertTrue(!destServerName.equals(metaServerName)); //find another row in the cur server that is less than ROW_X List<Region> regions = curServer.getOnlineRegions(TABLE_NAME3); @@ -1352,11 +1353,11 @@ public class TestHCM { Assert.assertNotNull(curServer.getOnlineRegion(regionName)); Assert.assertNull(destServer.getOnlineRegion(regionName)); Assert.assertFalse(TEST_UTIL.getMiniHBaseCluster().getMaster(). - getAssignmentManager().hasRegionsInTransition()); + getAssignmentManager().getRegionStates().isRegionsInTransition()); // Moving. It's possible that we don't have all the regions online at this point, so - // the test depends only on the region we're looking at. - LOG.info("Move starting region=" + toMove.getRegionInfo().getRegionNameAsString()); + // the test must depends only on the region we're looking at. + LOG.info("Move starting region="+toMove.getRegionInfo().getRegionNameAsString()); TEST_UTIL.getAdmin().move( toMove.getRegionInfo().getEncodedNameAsBytes(), destServerName.getServerName().getBytes() @@ -1365,7 +1366,7 @@ public class TestHCM { while (destServer.getOnlineRegion(regionName) == null || destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || - master.getAssignmentManager().hasRegionsInTransition()) { + master.getAssignmentManager().getRegionStates().isRegionsInTransition()) { // wait for the move to be finished Thread.sleep(1); } @@ -1477,4 +1478,107 @@ public class TestHCM { table.close(); connection.close(); } -} \ No newline at end of file + + private class TestPutThread extends Thread { + Table table; + int getServerBusyException = 0; + + TestPutThread(Table table){ + this.table = table; + } + + @Override + public void run() { + try { + Put p = new Put(ROW); + p.addColumn(FAM_NAM, new byte[]{0}, new byte[]{0}); + table.put(p); + } catch (RetriesExhaustedWithDetailsException e) { + if (e.exceptions.get(0) instanceof ServerTooBusyException) { + getServerBusyException = 1; + } + } catch (IOException ignore) { + } + } + } + + private class TestGetThread extends Thread { + Table table; + int getServerBusyException = 0; + + TestGetThread(Table table){ + this.table = table; + } + + @Override + public void run() { + try { + Get g = new Get(ROW); + g.addColumn(FAM_NAM, new byte[] { 0 }); + table.get(g); + } catch (ServerTooBusyException e) { + getServerBusyException = 1; + } catch (IOException ignore) { + } + } + } + + @Test() + public void testServerBusyException() throws Exception { + HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName())); + hdt.addCoprocessor(SleepCoprocessor.class.getName()); + Configuration c = new Configuration(TEST_UTIL.getConfiguration()); + TEST_UTIL.createTable(hdt, new byte[][] { FAM_NAM }, c); + + TestGetThread tg1 = + new TestGetThread(TEST_UTIL.getConnection().getTable(hdt.getTableName())); + TestGetThread tg2 = + new TestGetThread(TEST_UTIL.getConnection().getTable(hdt.getTableName())); + TestGetThread tg3 = + new TestGetThread(TEST_UTIL.getConnection().getTable(hdt.getTableName())); + TestGetThread tg4 = + new TestGetThread(TEST_UTIL.getConnection().getTable(hdt.getTableName())); + TestGetThread tg5 = + new TestGetThread(TEST_UTIL.getConnection().getTable(hdt.getTableName())); + tg1.start(); + tg2.start(); + tg3.start(); + tg4.start(); + tg5.start(); + tg1.join(); + tg2.join(); + tg3.join(); + tg4.join(); + tg5.join(); + assertEquals(2, + tg1.getServerBusyException + tg2.getServerBusyException + tg3.getServerBusyException + + tg4.getServerBusyException + tg5.getServerBusyException); + + // Put has its own logic in HTable, test Put alone. We use AsyncProcess for Put (use multi at + // RPC level) and it wrap exceptions to RetriesExhaustedWithDetailsException. + + TestPutThread tp1 = + new TestPutThread(TEST_UTIL.getConnection().getTable(hdt.getTableName())); + TestPutThread tp2 = + new TestPutThread(TEST_UTIL.getConnection().getTable(hdt.getTableName())); + TestPutThread tp3 = + new TestPutThread(TEST_UTIL.getConnection().getTable(hdt.getTableName())); + TestPutThread tp4 = + new TestPutThread(TEST_UTIL.getConnection().getTable(hdt.getTableName())); + TestPutThread tp5 = + new TestPutThread(TEST_UTIL.getConnection().getTable(hdt.getTableName())); + tp1.start(); + tp2.start(); + tp3.start(); + tp4.start(); + tp5.start(); + tp1.join(); + tp2.join(); + tp3.join(); + tp4.join(); + tp5.join(); + assertEquals(2, + tp1.getServerBusyException + tp2.getServerBusyException + tp3.getServerBusyException + + tp4.getServerBusyException + tp5.getServerBusyException); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java index 3847e6e..a700ebe 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java @@ -29,6 +29,7 @@ import java.util.Collection; import java.util.List; import java.util.concurrent.ExecutorService; +import edu.umd.cs.findbugs.annotations.Nullable; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -44,7 +45,6 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.Waiter; -import org.apache.hadoop.hbase.master.NoSuchProcedureException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -52,6 +52,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.HBaseFsck; import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; import org.apache.hadoop.hbase.util.HBaseFsckRepair; +import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil; import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker; import org.apache.hadoop.hbase.zookeeper.ZKUtil; @@ -59,15 +60,12 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.zookeeper.KeeperException; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; import org.junit.rules.TestRule; -import edu.umd.cs.findbugs.annotations.Nullable; - /** * Tests the scenarios where replicas are enabled for the meta table */ @@ -107,11 +105,7 @@ public class TestMetaWithReplicas { for (int replicaId = 1; replicaId < 3; replicaId ++) { HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(HRegionInfo.FIRST_META_REGIONINFO, replicaId); - try { - TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().waitForAssignment(h); - } catch (NoSuchProcedureException e) { - LOG.info("Presume the procedure has been cleaned up so just proceed: " + e.toString()); - } + TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().waitForAssignment(h); } LOG.debug("All meta replicas assigned"); } @@ -262,7 +256,7 @@ public class TestMetaWithReplicas { } } - @Ignore @Test // Uses FSCK. Needs fixing after HBASE-14614. + @Test public void testChangingReplicaCount() throws Exception { // tests changing the replica count across master restarts // reduce the replica count from 3 to 2 @@ -281,9 +275,6 @@ public class TestMetaWithReplicas { assert(metaZnodes.size() == originalReplicaCount); //we should have what was configured before TEST_UTIL.getHBaseClusterInterface().getConf().setInt(HConstants.META_REPLICAS_NUM, newReplicaCount); - if (TEST_UTIL.getHBaseCluster().countServedRegions() < newReplicaCount) { - TEST_UTIL.getHBaseCluster().startRegionServer(); - } TEST_UTIL.getHBaseClusterInterface().startMaster(sn.getHostname(), 0); TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster(); TEST_UTIL.waitFor(10000, predicateMetaHasReplicas(newReplicaCount)); @@ -340,7 +331,7 @@ public class TestMetaWithReplicas { HbckTestingUtil.assertNoErrors(hbck); } - @Ignore @Test // Disabled. Relies on FSCK which needs work for AMv2. + @Test public void testHBaseFsckWithFewerMetaReplicas() throws Exception { ClusterConnection c = (ClusterConnection)ConnectionFactory.createConnection( TEST_UTIL.getConfiguration()); @@ -358,7 +349,7 @@ public class TestMetaWithReplicas { assertErrors(hbck, new ERROR_CODE[]{}); } - @Ignore @Test // The close silently doesn't work any more since HBASE-14614. Fix. + @Test public void testHBaseFsckWithFewerMetaReplicaZnodes() throws Exception { ClusterConnection c = (ClusterConnection)ConnectionFactory.createConnection( TEST_UTIL.getConfiguration()); @@ -392,7 +383,7 @@ public class TestMetaWithReplicas { fail("Expected TableNotFoundException"); } - @Ignore @Test // Disabled. Currently can't move hbase:meta in AMv2. + @Test public void testMetaAddressChange() throws Exception { // checks that even when the meta's location changes, the various // caches update themselves. Uses the master operations to test @@ -420,16 +411,13 @@ public class TestMetaWithReplicas { TEST_UTIL.getAdmin().move(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), Bytes.toBytes(moveToServer.getServerName())); int i = 0; - assert !moveToServer.equals(currentServer); - LOG.info("CurrentServer=" + currentServer + ", moveToServer=" + moveToServer); - final int max = 10000; do { Thread.sleep(10); data = ZKUtil.getData(zkw, primaryMetaZnode); currentServer = ProtobufUtil.toServerName(data); i++; - } while (!moveToServer.equals(currentServer) && i < max); //wait for 10 seconds overall - assert(i != max); + } while (!moveToServer.equals(currentServer) && i < 1000); //wait for 10 seconds overall + assert(i != 1000); TEST_UTIL.getAdmin().disableTable(tableName); assertTrue(TEST_UTIL.getAdmin().isTableDisabled(tableName)); } @@ -448,7 +436,7 @@ public class TestMetaWithReplicas { int i = 0; do { LOG.debug("Waiting for the replica " + hrl.getRegionInfo() + " to come up"); - Thread.sleep(10000); //wait for the detection/recovery + Thread.sleep(30000); //wait for the detection/recovery rl = conn.locateRegion(TableName.META_TABLE_NAME, Bytes.toBytes(""), false, true); hrl = rl.getRegionLocation(1); i++; @@ -457,11 +445,14 @@ public class TestMetaWithReplicas { } } - @Ignore @Test // Disabled because fsck and this needs work for AMv2 + @Test public void testHBaseFsckWithExcessMetaReplicas() throws Exception { // Create a meta replica (this will be the 4th one) and assign it HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica( HRegionInfo.FIRST_META_REGIONINFO, 3); + // create in-memory state otherwise master won't assign + TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().createRegionState(h); TEST_UTIL.assignRegion(h); HBaseFsckRepair.waitUntilAssigned(TEST_UTIL.getAdmin(), h); // check that problem exists http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java index ef00b24..1b18ee2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.client; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; @@ -44,7 +43,11 @@ import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; import org.apache.hadoop.hbase.filter.ColumnRangeFilter; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.RegionState.State; +import org.apache.hadoop.hbase.master.RegionStates; import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -616,33 +619,34 @@ public class TestScannersFromClientSide { byte[] regionName = hri.getRegionName(); int i = cluster.getServerWith(regionName); HRegionServer rs = cluster.getRegionServer(i); - LOG.info("Unassigning " + hri); - TEST_UTIL.getAdmin().unassign(hri.getRegionName(), true); + ProtobufUtil.closeRegion(null, + rs.getRSRpcServices(), rs.getServerName(), regionName); long startTime = EnvironmentEdgeManager.currentTime(); - long timeOut = 10000; - boolean offline = false; + long timeOut = 300000; while (true) { if (rs.getOnlineRegion(regionName) == null) { - offline = true; break; } assertTrue("Timed out in closing the testing region", EnvironmentEdgeManager.currentTime() < startTime + timeOut); + Thread.sleep(500); } - assertTrue(offline); - LOG.info("Assigning " + hri); - TEST_UTIL.getAdmin().assign(hri.getRegionName()); + + // Now open the region again. + HMaster master = cluster.getMaster(); + RegionStates states = master.getAssignmentManager().getRegionStates(); + states.regionOffline(hri); + states.updateRegionState(hri, State.OPENING); + ProtobufUtil.openRegion(null, rs.getRSRpcServices(), rs.getServerName(), hri); startTime = EnvironmentEdgeManager.currentTime(); while (true) { - rs = cluster.getRegionServer(cluster.getServerWith(regionName)); - if (rs != null && rs.getOnlineRegion(regionName) != null) { - offline = false; + if (rs.getOnlineRegion(regionName) != null) { break; } assertTrue("Timed out in open the testing region", EnvironmentEdgeManager.currentTime() < startTime + timeOut); + Thread.sleep(500); } - assertFalse(offline); // c0:0, c1:1 kvListExp = new ArrayList<>();
