Repository: hbase Updated Branches: refs/heads/master 918599ef1 -> 80438924f
HBASE-19879 Promote TestAcidGuaranteesXXX to LargeTests Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/80438924 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/80438924 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/80438924 Branch: refs/heads/master Commit: 80438924ff5066bf809cf0318168025cf646d415 Parents: 918599e Author: zhangduo <zhang...@apache.org> Authored: Mon Jan 29 13:04:58 2018 +0800 Committer: zhangduo <zhang...@apache.org> Committed: Mon Jan 29 14:55:04 2018 +0800 ---------------------------------------------------------------------- .../hadoop/hbase/AcidGuaranteesTestBase.java | 134 +++++++++++++++++++ .../TestAcidGuaranteesWithAdaptivePolicy.java | 6 +- .../TestAcidGuaranteesWithBasicPolicy.java | 6 +- .../TestAcidGuaranteesWithEagerPolicy.java | 6 +- ...TestAcidGuaranteesWithNoInMemCompaction.java | 117 +--------------- 5 files changed, 147 insertions(+), 122 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/80438924/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestBase.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestBase.java new file mode 100644 index 0000000..c0aa1a0 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestBase.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.apache.hadoop.hbase.AcidGuaranteesTestTool.FAMILIES; +import static org.apache.hadoop.hbase.AcidGuaranteesTestTool.TABLE_NAME; + +import java.util.List; +import java.util.stream.Stream; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.CompactingMemStore; +import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.MemStoreLAB; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; + +/** + * Test case that uses multiple threads to read and write multifamily rows into a table, verifying + * that reads never see partially-complete writes. This can run as a junit test, or with a main() + * function which runs against a real cluster (eg for testing with failures, region movement, etc) + */ +public abstract class AcidGuaranteesTestBase { + + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private AcidGuaranteesTestTool tool = new AcidGuaranteesTestTool(); + + protected abstract MemoryCompactionPolicy getMemoryCompactionPolicy(); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Set small flush size for minicluster so we exercise reseeking scanners + Configuration conf = UTIL.getConfiguration(); + conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(128 * 1024)); + // prevent aggressive region split + conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, + ConstantSizeRegionSplitPolicy.class.getName()); + conf.setInt("hfile.format.version", 3); // for mob tests + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Before + public void setUp() throws Exception { + MemoryCompactionPolicy policy = getMemoryCompactionPolicy(); + TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TABLE_NAME) + .setValue(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, policy.name()); + if (policy == MemoryCompactionPolicy.EAGER) { + builder.setValue(MemStoreLAB.USEMSLAB_KEY, "false"); + builder.setValue(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, "0.9"); + } + Stream.of(FAMILIES).map(ColumnFamilyDescriptorBuilder::of) + .forEachOrdered(builder::addColumnFamily); + UTIL.getAdmin().createTable(builder.build()); + tool.setConf(UTIL.getConfiguration()); + } + + @After + public void tearDown() throws Exception { + UTIL.deleteTable(TABLE_NAME); + } + + private void runTestAtomicity(long millisToRun, int numWriters, int numGetters, int numScanners, + int numUniqueRows) throws Exception { + runTestAtomicity(millisToRun, numWriters, numGetters, numScanners, numUniqueRows, false); + } + + private void runTestAtomicity(long millisToRun, int numWriters, int numGetters, int numScanners, + int numUniqueRows, boolean useMob) throws Exception { + List<String> args = Lists.newArrayList("-millis", String.valueOf(millisToRun), "-numWriters", + String.valueOf(numWriters), "-numGetters", String.valueOf(numGetters), "-numScanners", + String.valueOf(numScanners), "-numUniqueRows", String.valueOf(numUniqueRows), "-crazyFlush"); + if (useMob) { + args.add("-useMob"); + } + tool.run(args.toArray(new String[0])); + } + + @Test + public void testGetAtomicity() throws Exception { + runTestAtomicity(20000, 5, 5, 0, 3); + } + + @Test + public void testScanAtomicity() throws Exception { + runTestAtomicity(20000, 5, 0, 5, 3); + } + + @Test + public void testMixedAtomicity() throws Exception { + runTestAtomicity(20000, 5, 2, 2, 3); + } + + @Test + public void testMobGetAtomicity() throws Exception { + runTestAtomicity(20000, 5, 5, 0, 3, true); + } + + @Test + public void testMobScanAtomicity() throws Exception { + runTestAtomicity(20000, 5, 0, 5, 3, true); + } + + @Test + public void testMobMixedAtomicity() throws Exception { + runTestAtomicity(20000, 5, 2, 2, 3, true); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/80438924/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java index b46a28c..ff77081 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java @@ -17,12 +17,12 @@ */ package org.apache.hadoop.hbase; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.junit.ClassRule; import org.junit.experimental.categories.Category; -@Category({ MediumTests.class }) -public class TestAcidGuaranteesWithAdaptivePolicy extends TestAcidGuaranteesWithNoInMemCompaction { +@Category(LargeTests.class) +public class TestAcidGuaranteesWithAdaptivePolicy extends AcidGuaranteesTestBase { @ClassRule public static final HBaseClassTestRule CLASS_RULE = http://git-wip-us.apache.org/repos/asf/hbase/blob/80438924/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java index 7993649..02c6a98 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java @@ -17,12 +17,12 @@ */ package org.apache.hadoop.hbase; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.junit.ClassRule; import org.junit.experimental.categories.Category; -@Category({ MediumTests.class }) -public class TestAcidGuaranteesWithBasicPolicy extends TestAcidGuaranteesWithNoInMemCompaction { +@Category(LargeTests.class) +public class TestAcidGuaranteesWithBasicPolicy extends AcidGuaranteesTestBase { @ClassRule public static final HBaseClassTestRule CLASS_RULE = http://git-wip-us.apache.org/repos/asf/hbase/blob/80438924/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java index c65eada..5f2e245 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java @@ -17,12 +17,12 @@ */ package org.apache.hadoop.hbase; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.junit.ClassRule; import org.junit.experimental.categories.Category; -@Category({ MediumTests.class }) -public class TestAcidGuaranteesWithEagerPolicy extends TestAcidGuaranteesWithNoInMemCompaction { +@Category(LargeTests.class) +public class TestAcidGuaranteesWithEagerPolicy extends AcidGuaranteesTestBase { @ClassRule public static final HBaseClassTestRule CLASS_RULE = http://git-wip-us.apache.org/repos/asf/hbase/blob/80438924/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java index 001f5c2..4b2bcd0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java @@ -17,128 +17,19 @@ */ package org.apache.hadoop.hbase; -import static org.apache.hadoop.hbase.AcidGuaranteesTestTool.FAMILIES; -import static org.apache.hadoop.hbase.AcidGuaranteesTestTool.TABLE_NAME; - -import java.util.List; -import java.util.stream.Stream; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.regionserver.CompactingMemStore; -import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; -import org.apache.hadoop.hbase.regionserver.MemStoreLAB; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.junit.ClassRule; -import org.junit.Test; import org.junit.experimental.categories.Category; -import org.apache.hbase.thirdparty.com.google.common.collect.Lists; - -/** - * Test case that uses multiple threads to read and write multifamily rows into a table, verifying - * that reads never see partially-complete writes. This can run as a junit test, or with a main() - * function which runs against a real cluster (eg for testing with failures, region movement, etc) - */ -@Category({ MediumTests.class }) -public class TestAcidGuaranteesWithNoInMemCompaction { +@Category(LargeTests.class) +public class TestAcidGuaranteesWithNoInMemCompaction extends AcidGuaranteesTestBase { @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestAcidGuaranteesWithNoInMemCompaction.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - - private AcidGuaranteesTestTool tool = new AcidGuaranteesTestTool(); - + @Override protected MemoryCompactionPolicy getMemoryCompactionPolicy() { return MemoryCompactionPolicy.NONE; } - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - // Set small flush size for minicluster so we exercise reseeking scanners - Configuration conf = UTIL.getConfiguration(); - conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(128 * 1024)); - // prevent aggressive region split - conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, - ConstantSizeRegionSplitPolicy.class.getName()); - conf.setInt("hfile.format.version", 3); // for mob tests - UTIL.startMiniCluster(1); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - UTIL.shutdownMiniCluster(); - } - - @Before - public void setUp() throws Exception { - MemoryCompactionPolicy policy = getMemoryCompactionPolicy(); - TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TABLE_NAME) - .setValue(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, policy.name()); - if (policy == MemoryCompactionPolicy.EAGER) { - builder.setValue(MemStoreLAB.USEMSLAB_KEY, "false"); - builder.setValue(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, "0.9"); - } - Stream.of(FAMILIES).map(ColumnFamilyDescriptorBuilder::of) - .forEachOrdered(builder::addColumnFamily); - UTIL.getAdmin().createTable(builder.build()); - tool.setConf(UTIL.getConfiguration()); - } - - @After - public void tearDown() throws Exception { - UTIL.deleteTable(TABLE_NAME); - } - - private void runTestAtomicity(long millisToRun, int numWriters, int numGetters, int numScanners, - int numUniqueRows) throws Exception { - runTestAtomicity(millisToRun, numWriters, numGetters, numScanners, numUniqueRows, false); - } - - private void runTestAtomicity(long millisToRun, int numWriters, int numGetters, int numScanners, - int numUniqueRows, boolean useMob) throws Exception { - List<String> args = Lists.newArrayList("-millis", String.valueOf(millisToRun), "-numWriters", - String.valueOf(numWriters), "-numGetters", String.valueOf(numGetters), "-numScanners", - String.valueOf(numScanners), "-numUniqueRows", String.valueOf(numUniqueRows), "-crazyFlush"); - if (useMob) { - args.add("-useMob"); - } - tool.run(args.toArray(new String[0])); - } - - @Test - public void testGetAtomicity() throws Exception { - runTestAtomicity(20000, 5, 5, 0, 3); - } - - @Test - public void testScanAtomicity() throws Exception { - runTestAtomicity(20000, 5, 0, 5, 3); - } - - @Test - public void testMixedAtomicity() throws Exception { - runTestAtomicity(20000, 5, 2, 2, 3); - } - - @Test - public void testMobGetAtomicity() throws Exception { - runTestAtomicity(20000, 5, 5, 0, 3, true); - } - - @Test - public void testMobScanAtomicity() throws Exception { - runTestAtomicity(20000, 5, 0, 5, 3, true); - } - - @Test - public void testMobMixedAtomicity() throws Exception { - runTestAtomicity(20000, 5, 2, 2, 3, true); - } }
