Repository: hbase Updated Branches: refs/heads/branch-2 ca6e7e68f -> 66e650c39
HBASE-19266 TestAcidGuarantees should cover adaptive in-memory compaction Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/66e650c3 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/66e650c3 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/66e650c3 Branch: refs/heads/branch-2 Commit: 66e650c39b7d794e9cd774add7b9f2be9d064e8f Parents: ca6e7e6 Author: Chia-Ping Tsai <chia7...@gmail.com> Authored: Fri Nov 24 00:16:50 2017 +0800 Committer: Chia-Ping Tsai <chia7...@gmail.com> Committed: Fri Nov 24 00:19:06 2017 +0800 ---------------------------------------------------------------------- .../hadoop/hbase/AcidGuaranteesTestTool.java | 3 +- .../apache/hadoop/hbase/TestAcidGuarantees.java | 149 ------------------- .../TestAcidGuaranteesWithAdaptivePolicy.java | 35 +++++ .../TestAcidGuaranteesWithBasicPolicy.java | 31 ++++ .../TestAcidGuaranteesWithEagerPolicy.java | 35 +++++ ...TestAcidGuaranteesWithNoInMemCompaction.java | 140 +++++++++++++++++ 6 files changed, 242 insertions(+), 151 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/66e650c3/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestTool.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestTool.java index 5e00e8c..6a21438 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestTool.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestTool.java @@ -27,7 +27,6 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; - import org.apache.commons.cli.CommandLine; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -62,7 +61,7 @@ import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; @InterfaceAudience.Private public class AcidGuaranteesTestTool extends AbstractHBaseTool { - private static final Log LOG = LogFactory.getLog(TestAcidGuarantees.class); + private static final Log LOG = LogFactory.getLog(AcidGuaranteesTestTool.class); public static final TableName TABLE_NAME = TableName.valueOf("TestAcidGuarantees"); public static final byte[] FAMILY_A = Bytes.toBytes("A"); http://git-wip-us.apache.org/repos/asf/hbase/blob/66e650c3/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java deleted file mode 100644 index e42794e..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase; - -import static org.apache.hadoop.hbase.AcidGuaranteesTestTool.FAMILIES; -import static org.apache.hadoop.hbase.AcidGuaranteesTestTool.TABLE_NAME; - -import java.util.List; -import java.util.stream.Stream; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.regionserver.CompactingMemStore; -import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; -import org.apache.hadoop.hbase.regionserver.MemStoreLAB; -import org.apache.hadoop.hbase.testclassification.FlakeyTests; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameter; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -/** - * Test case that uses multiple threads to read and write multifamily rows into a table, verifying - * that reads never see partially-complete writes. This can run as a junit test, or with a main() - * function which runs against a real cluster (eg for testing with failures, region movement, etc) - */ -@Category({ FlakeyTests.class, LargeTests.class }) -@RunWith(Parameterized.class) -public class TestAcidGuarantees { - - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - - @Parameterized.Parameters(name = "{index}: compType={0}") - public static Object[] data() { - return new Object[] { "NONE", "BASIC", "EAGER" }; - } - - @Parameter - public String compType; - - private AcidGuaranteesTestTool tool = new AcidGuaranteesTestTool(); - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - // Set small flush size for minicluster so we exercise reseeking scanners - Configuration conf = UTIL.getConfiguration(); - conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(128 * 1024)); - // prevent aggressive region split - conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, - ConstantSizeRegionSplitPolicy.class.getName()); - conf.setInt("hfile.format.version", 3); // for mob tests - UTIL.startMiniCluster(1); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - UTIL.shutdownMiniCluster(); - } - - @Before - public void setUp() throws Exception { - TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TABLE_NAME) - .setValue(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, compType); - if (MemoryCompactionPolicy.valueOf(compType) == MemoryCompactionPolicy.EAGER) { - builder.setValue(MemStoreLAB.USEMSLAB_KEY, "false"); - builder.setValue(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, "0.9"); - } - Stream.of(FAMILIES).map(ColumnFamilyDescriptorBuilder::of) - .forEachOrdered(builder::addColumnFamily); - UTIL.getAdmin().createTable(builder.build()); - tool.setConf(UTIL.getConfiguration()); - } - - @After - public void tearDown() throws Exception { - UTIL.deleteTable(TABLE_NAME); - } - - private void runTestAtomicity(long millisToRun, int numWriters, int numGetters, int numScanners, - int numUniqueRows) throws Exception { - runTestAtomicity(millisToRun, numWriters, numGetters, numScanners, numUniqueRows, false); - } - - private void runTestAtomicity(long millisToRun, int numWriters, int numGetters, int numScanners, - int numUniqueRows, boolean useMob) throws Exception { - List<String> args = Lists.newArrayList("-millis", String.valueOf(millisToRun), "-numWriters", - String.valueOf(numWriters), "-numGetters", String.valueOf(numGetters), "-numScanners", - String.valueOf(numScanners), "-numUniqueRows", String.valueOf(numUniqueRows), "-crazyFlush"); - if (useMob) { - args.add("-useMob"); - } - tool.run(args.toArray(new String[0])); - } - - @Test - public void testGetAtomicity() throws Exception { - runTestAtomicity(20000, 5, 5, 0, 3); - } - - @Test - public void testScanAtomicity() throws Exception { - runTestAtomicity(20000, 5, 0, 5, 3); - } - - @Test - public void testMixedAtomicity() throws Exception { - runTestAtomicity(20000, 5, 2, 2, 3); - } - - @Test - public void testMobGetAtomicity() throws Exception { - runTestAtomicity(20000, 5, 5, 0, 3, true); - } - - @Test - public void testMobScanAtomicity() throws Exception { - runTestAtomicity(20000, 5, 0, 5, 3, true); - } - - @Test - public void testMobMixedAtomicity() throws Exception { - runTestAtomicity(20000, 5, 2, 2, 3, true); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/66e650c3/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java new file mode 100644 index 0000000..3bc417e --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java @@ -0,0 +1,35 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.hadoop.hbase.testclassification.FlakeyTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.Ignore; +import org.junit.experimental.categories.Category; + +@Category({ FlakeyTests.class, MediumTests.class }) +// TODO: HBASE-19266 disables this test as the adaptive policy causes the negative size of MemStore +@Ignore +public class TestAcidGuaranteesWithAdaptivePolicy extends TestAcidGuaranteesWithNoInMemCompaction { + + @Override + protected MemoryCompactionPolicy getMemoryCompactionPolicy() { + return MemoryCompactionPolicy.ADAPTIVE; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/66e650c3/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java new file mode 100644 index 0000000..78737a8 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java @@ -0,0 +1,31 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.experimental.categories.Category; + +@Category({ MediumTests.class }) +public class TestAcidGuaranteesWithBasicPolicy extends TestAcidGuaranteesWithNoInMemCompaction { + + @Override + protected MemoryCompactionPolicy getMemoryCompactionPolicy() { + return MemoryCompactionPolicy.BASIC; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/66e650c3/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java new file mode 100644 index 0000000..b2c87f6 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java @@ -0,0 +1,35 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.hadoop.hbase.testclassification.FlakeyTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.Ignore; +import org.junit.experimental.categories.Category; + +@Category({ FlakeyTests.class, MediumTests.class }) +// TODO: HBASE-19266 disables this test as the eager policy causes the negative size of MemStore +@Ignore +public class TestAcidGuaranteesWithEagerPolicy extends TestAcidGuaranteesWithNoInMemCompaction { + + @Override + protected MemoryCompactionPolicy getMemoryCompactionPolicy() { + return MemoryCompactionPolicy.EAGER; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/66e650c3/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java new file mode 100644 index 0000000..9ce2337 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java @@ -0,0 +1,140 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.apache.hadoop.hbase.AcidGuaranteesTestTool.FAMILIES; +import static org.apache.hadoop.hbase.AcidGuaranteesTestTool.TABLE_NAME; + +import java.util.List; +import java.util.stream.Stream; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.CompactingMemStore; +import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.MemStoreLAB; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; + +/** + * Test case that uses multiple threads to read and write multifamily rows into a table, verifying + * that reads never see partially-complete writes. This can run as a junit test, or with a main() + * function which runs against a real cluster (eg for testing with failures, region movement, etc) + */ +@Category({ MediumTests.class }) +public class TestAcidGuaranteesWithNoInMemCompaction { + + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private AcidGuaranteesTestTool tool = new AcidGuaranteesTestTool(); + + protected MemoryCompactionPolicy getMemoryCompactionPolicy() { + return MemoryCompactionPolicy.NONE; + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Set small flush size for minicluster so we exercise reseeking scanners + Configuration conf = UTIL.getConfiguration(); + conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(128 * 1024)); + // prevent aggressive region split + conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, + ConstantSizeRegionSplitPolicy.class.getName()); + conf.setInt("hfile.format.version", 3); // for mob tests + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Before + public void setUp() throws Exception { + MemoryCompactionPolicy policy = getMemoryCompactionPolicy(); + TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TABLE_NAME) + .setValue(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, policy.name()); + if (policy == MemoryCompactionPolicy.EAGER) { + builder.setValue(MemStoreLAB.USEMSLAB_KEY, "false"); + builder.setValue(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, "0.9"); + } + Stream.of(FAMILIES).map(ColumnFamilyDescriptorBuilder::of) + .forEachOrdered(builder::addColumnFamily); + UTIL.getAdmin().createTable(builder.build()); + tool.setConf(UTIL.getConfiguration()); + } + + @After + public void tearDown() throws Exception { + UTIL.deleteTable(TABLE_NAME); + } + + private void runTestAtomicity(long millisToRun, int numWriters, int numGetters, int numScanners, + int numUniqueRows) throws Exception { + runTestAtomicity(millisToRun, numWriters, numGetters, numScanners, numUniqueRows, false); + } + + private void runTestAtomicity(long millisToRun, int numWriters, int numGetters, int numScanners, + int numUniqueRows, boolean useMob) throws Exception { + List<String> args = Lists.newArrayList("-millis", String.valueOf(millisToRun), "-numWriters", + String.valueOf(numWriters), "-numGetters", String.valueOf(numGetters), "-numScanners", + String.valueOf(numScanners), "-numUniqueRows", String.valueOf(numUniqueRows), "-crazyFlush"); + if (useMob) { + args.add("-useMob"); + } + tool.run(args.toArray(new String[0])); + } + + @Test + public void testGetAtomicity() throws Exception { + runTestAtomicity(20000, 5, 5, 0, 3); + } + + @Test + public void testScanAtomicity() throws Exception { + runTestAtomicity(20000, 5, 0, 5, 3); + } + + @Test + public void testMixedAtomicity() throws Exception { + runTestAtomicity(20000, 5, 2, 2, 3); + } + + @Test + public void testMobGetAtomicity() throws Exception { + runTestAtomicity(20000, 5, 5, 0, 3, true); + } + + @Test + public void testMobScanAtomicity() throws Exception { + runTestAtomicity(20000, 5, 0, 5, 3, true); + } + + @Test + public void testMobMixedAtomicity() throws Exception { + runTestAtomicity(20000, 5, 2, 2, 3, true); + } +}
