Repository: hbase Updated Branches: refs/heads/master 047463c27 -> 58549428a
http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java index 548f4d5..10c2335 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java @@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -65,7 +66,6 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.coordination.ZKSplitTransactionCoordination; import org.apache.hadoop.hbase.coordination.ZkCloseRegionCoordination; import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager; @@ -120,15 +120,24 @@ public class TestSplitTransactionOnCluster { private static volatile boolean secondSplit = false; private static volatile boolean callRollBack = false; private static volatile boolean firstSplitCompleted = false; + private static boolean useZKForAssignment = true; - private static final HBaseTestingUtility TESTING_UTIL = + static final HBaseTestingUtility TESTING_UTIL = new HBaseTestingUtility(); - @BeforeClass public static void before() throws Exception { + static void setupOnce() throws Exception { TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000); + useZKForAssignment = TESTING_UTIL.getConfiguration().getBoolean( + "hbase.assignment.usezk", false); TESTING_UTIL.startMiniCluster(NB_SERVERS); } + @BeforeClass public static void before() throws Exception { + // Use ZK for region assignment + TESTING_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true); + setupOnce(); + } + @AfterClass public static void after() throws Exception { TESTING_UTIL.shutdownMiniCluster(); } @@ -175,6 +184,12 @@ public class TestSplitTransactionOnCluster { public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception { final TableName tableName = TableName.valueOf("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack"); + + if (!useZKForAssignment) { + // This test doesn't apply if not using ZK for assignment + return; + } + try { // Create table then get the single region for our new table. HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf")); @@ -304,7 +319,7 @@ public class TestSplitTransactionOnCluster { TESTING_UTIL.deleteTable(tableName); } } - @Test + @Test(timeout = 60000) @SuppressWarnings("deprecation") public void testSplitFailedCompactionAndSplit() throws Exception { final byte[] tableName = Bytes.toBytes("testSplitFailedCompactionAndSplit"); @@ -403,42 +418,46 @@ public class TestSplitTransactionOnCluster { AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true; // Now try splitting and it should work. split(hri, server, regionCount); - // Get daughters - List<HRegion> daughters = checkAndGetDaughters(tableName); - // Assert the ephemeral node is up in zk. + // Assert the ephemeral node is up in zk. String path = ZKAssign.getNodeName(TESTING_UTIL.getZooKeeperWatcher(), hri.getEncodedName()); RegionTransition rt = null; Stat stats = null; - // Wait till the znode moved to SPLIT - for (int i=0; i<100; i++) { - stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false); - rt = RegionTransition.parseFrom(ZKAssign.getData(TESTING_UTIL.getZooKeeperWatcher(), - hri.getEncodedName())); - if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) break; - Thread.sleep(100); + List<HRegion> daughters = null; + if (useZKForAssignment) { + daughters = checkAndGetDaughters(tableName); + + // Wait till the znode moved to SPLIT + for (int i=0; i<100; i++) { + stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false); + rt = RegionTransition.parseFrom(ZKAssign.getData(TESTING_UTIL.getZooKeeperWatcher(), + hri.getEncodedName())); + if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) break; + Thread.sleep(100); + } + LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats); + assertTrue(rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)); + // Now crash the server + cluster.abortRegionServer(tableRegionIndex); } - LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats); - assertTrue(rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)); - // Now crash the server - cluster.abortRegionServer(tableRegionIndex); waitUntilRegionServerDead(); - awaitDaughters(tableName, daughters.size()); + awaitDaughters(tableName, 2); + if (useZKForAssignment) { + regions = cluster.getRegions(tableName); + for (HRegion r: regions) { + assertTrue(daughters.contains(r)); + } - // Assert daughters are online. - regions = cluster.getRegions(tableName); - for (HRegion r: regions) { - assertTrue(daughters.contains(r)); - } - // Finally assert that the ephemeral SPLIT znode was cleaned up. - for (int i=0; i<100; i++) { - // wait a bit (10s max) for the node to disappear - stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false); - if (stats == null) break; - Thread.sleep(100); + // Finally assert that the ephemeral SPLIT znode was cleaned up. + for (int i=0; i<100; i++) { + // wait a bit (10s max) for the node to disappear + stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false); + if (stats == null) break; + Thread.sleep(100); + } + LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats); + assertTrue(stats == null); } - LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats); - assertTrue(stats == null); } finally { // Set this flag back. AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false; @@ -461,6 +480,8 @@ public class TestSplitTransactionOnCluster { int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri); + RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates(); + // Turn off balancer so it doesn't cut in and mess up our placements. this.admin.setBalancerRunning(false, true); // Turn off the meta scanner so it don't remove parent on us. @@ -475,8 +496,12 @@ public class TestSplitTransactionOnCluster { // Insert into zk a blocking znode, a znode of same name as region // so it gets in way of our splitting. ServerName fakedServer = ServerName.valueOf("any.old.server", 1234, -1); - ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(), - hri, fakedServer); + if (useZKForAssignment) { + ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(), + hri, fakedServer); + } else { + regionStates.updateRegionState(hri, RegionState.State.CLOSING); + } // Now try splitting.... should fail. And each should successfully // rollback. this.admin.split(hri.getRegionNameAsString()); @@ -488,9 +513,13 @@ public class TestSplitTransactionOnCluster { assertEquals(regionCount, ProtobufUtil.getOnlineRegions( server.getRSRpcServices()).size()); } - // Now clear the zknode - ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(), - hri, fakedServer); + if (useZKForAssignment) { + // Now clear the zknode + ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(), + hri, fakedServer); + } else { + regionStates.regionOnline(hri, server.getServerName()); + } // Now try splitting and it should work. split(hri, server, regionCount); // Get daughters @@ -682,6 +711,11 @@ public class TestSplitTransactionOnCluster { KeeperException, DeserializationException, ServiceException { final byte[] tableName = Bytes.toBytes("testMasterRestartWhenSplittingIsPartial"); + if (!useZKForAssignment) { + // This test doesn't apply if not using ZK for assignment + return; + } + // Create table then get the single region for our new table. HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); List<HRegion> regions = cluster.getRegions(tableName); @@ -841,7 +875,7 @@ public class TestSplitTransactionOnCluster { * @throws InterruptedException * @throws KeeperException */ - @Test + @Test(timeout = 60000) public void testSplitBeforeSettingSplittingInZK() throws Exception, InterruptedException, KeeperException { testSplitBeforeSettingSplittingInZKInternals(); @@ -912,7 +946,7 @@ public class TestSplitTransactionOnCluster { * If a table has regions that have no store files in a region, they should split successfully * into two regions with no store files. */ - @Test + @Test(timeout = 60000) public void testSplitRegionWithNoStoreFiles() throws Exception { final TableName tableName = @@ -1196,14 +1230,22 @@ public class TestSplitTransactionOnCluster { private void split(final HRegionInfo hri, final HRegionServer server, final int regionCount) throws IOException, InterruptedException { this.admin.split(hri.getRegionNameAsString()); - for (int i = 0; ProtobufUtil.getOnlineRegions( - server.getRSRpcServices()).size() <= regionCount && i < 300; i++) { - LOG.debug("Waiting on region to split"); - Thread.sleep(100); - } + try { + for (int i = 0; ProtobufUtil.getOnlineRegions( + server.getRSRpcServices()).size() <= regionCount && i < 300; i++) { + LOG.debug("Waiting on region to split"); + Thread.sleep(100); + } - assertFalse("Waited too long for split", - ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size() <= regionCount); + assertFalse("Waited too long for split", + ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size() <= regionCount); + } catch (RegionServerStoppedException e) { + if (useZKForAssignment) { + // If not using ZK for assignment, the exception may be expected. + LOG.error(e); + throw e; + } + } } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java new file mode 100644 index 0000000..d9e8480 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java @@ -0,0 +1,45 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.LargeTests; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +/** + * Like {@link TestRegionMergeTransaction} in that we're testing + * {@link RegionMergeTransaction} only the below tests are against a running + * cluster where {@link TestRegionMergeTransaction} is tests against bare + * {@link HRegion}. + */ +@Category(LargeTests.class) +public class TestZKLessMergeOnCluster extends TestRegionMergeTransactionOnCluster { + @BeforeClass + public static void beforeAllTests() throws Exception { + // Don't use ZK for region assignment + TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", false); + setupOnce(); + } + + @AfterClass + public static void afterAllTests() throws Exception { + TestRegionMergeTransactionOnCluster.afterAllTests(); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java new file mode 100644 index 0000000..1e3de64 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java @@ -0,0 +1,45 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.LargeTests; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +/** + * Like {@link TestSplitTransaction} in that we're testing {@link SplitTransaction} + * only the below tests are against a running cluster where {@link TestSplitTransaction} + * is tests against a bare {@link HRegion}. + */ +@Category(LargeTests.class) +public class TestZKLessSplitOnCluster extends TestSplitTransactionOnCluster { + @BeforeClass + public static void before() throws Exception { + // Don't use ZK for region assignment + TESTING_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", false); + setupOnce(); + } + + @AfterClass + public static void after() throws Exception { + TestSplitTransactionOnCluster.after(); + } +} + http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java index 43d5875..2990014 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java @@ -65,6 +65,7 @@ public class TestCloseRegionHandler { private int testIndex = 0; @BeforeClass public static void before() throws Exception { + HTU.getConfiguration().setBoolean("hbase.assignment.usezk", true); HTU.startMiniZKCluster(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java index 869d727..d472d57 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java @@ -56,6 +56,7 @@ public class TestOpenRegionHandler { private int testIndex = 0; @BeforeClass public static void before() throws Exception { + HTU.getConfiguration().setBoolean("hbase.assignment.usezk", true); HTU.startMiniZKCluster(); TEST_HTD = new HTableDescriptor(TableName.valueOf("TestOpenRegionHandler.java")); } http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java index 134a953..0a2e8fd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java @@ -1630,6 +1630,12 @@ public class TestHBaseFsck { assertFalse(fsck.shouldRerun()); assertErrors(fsck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN}); + + // fix the cluster so other tests won't be impacted + fsck = doFsck(conf, true); + assertTrue(fsck.shouldRerun()); + fsck = doFsck(conf, true); + assertNoErrors(fsck); } finally { deleteTable(table); } http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java index f805ca4..7ed8215 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java @@ -69,7 +69,7 @@ public class TestRecoverableZooKeeper { @Test public void testSetDataVersionMismatchInLoop() throws Exception { - String znode = "/hbase/region-in-transition/9af7cfc9b15910a0b3d714bf40a3248f"; + String znode = "/hbase/splitWAL/9af7cfc9b15910a0b3d714bf40a3248f"; Configuration conf = TEST_UTIL.getConfiguration(); Properties properties = ZKConfig.makeZKProps(conf); ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "testSetDataVersionMismatchInLoop",
