Repository: hbase Updated Branches: refs/heads/0.98 c480b2a4e -> 940bde52b
http://git-wip-us.apache.org/repos/asf/hbase/blob/940bde52/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompaction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompaction.java index 8afe621..21480d5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompaction.java @@ -1,5 +1,4 @@ /** - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -26,9 +25,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration; import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionPolicy; +import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionRequest; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -37,6 +40,10 @@ import org.junit.experimental.categories.Category; public class TestDateTieredCompaction extends TestCompactionPolicy { ArrayList<StoreFile> sfCreate(long[] minTimestamps, long[] maxTimestamps, long[] sizes) throws IOException { + ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge(); + EnvironmentEdgeManager.injectEdge(timeMachine); + // Has to be > 0 and < now. + timeMachine.setValue(1); ArrayList<Long> ageInDisk = new ArrayList<Long>(); for (int i = 0; i < sizes.length; i++) { ageInDisk.add(0L); @@ -57,29 +64,47 @@ public class TestDateTieredCompaction extends TestCompactionPolicy { super.config(); // Set up policy - conf.setLong(CompactionConfiguration.MAX_AGE_KEY, 100); + conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, + "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine"); + conf.setLong(CompactionConfiguration.MAX_AGE_MILLIS_KEY, 100); conf.setLong(CompactionConfiguration.INCOMING_WINDOW_MIN_KEY, 3); conf.setLong(CompactionConfiguration.BASE_WINDOW_MILLIS_KEY, 6); conf.setInt(CompactionConfiguration.WINDOWS_PER_TIER_KEY, 4); - conf.set(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, - DateTieredCompactionPolicy.class.getName()); + conf.setBoolean(CompactionConfiguration.SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, false); // Special settings for compaction policy per window - this.conf.setInt(CompactionConfiguration.MIN_KEY, 2); - this.conf.setInt(CompactionConfiguration.MAX_KEY, 12); - this.conf.setFloat(CompactionConfiguration.RATIO_KEY, 1.2F); + conf.setInt(CompactionConfiguration.MIN_KEY, 2); + conf.setInt(CompactionConfiguration.MAX_KEY, 12); + conf.setFloat(CompactionConfiguration.RATIO_KEY, 1.2F); + + conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 20); + conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 10); } - void compactEquals(long now, ArrayList<StoreFile> candidates, long... expected) - throws IOException { - Assert.assertTrue(((DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy()) - .needsCompaction(candidates, ImmutableList.<StoreFile> of(), now)); - - List<StoreFile> actual = - ((DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy()) - .applyCompactionPolicy(candidates, false, false, now); - - Assert.assertEquals(Arrays.toString(expected), Arrays.toString(getSizes(actual))); + void compactEquals(long now, ArrayList<StoreFile> candidates, long[] expectedFileSizes, + long[] expectedBoundaries, boolean isMajor, boolean toCompact) throws IOException { + ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge(); + EnvironmentEdgeManager.injectEdge(timeMachine); + timeMachine.setValue(now); + DateTieredCompactionRequest request; + if (isMajor) { + for (StoreFile file : candidates) { + ((MockStoreFile)file).setIsMajor(true); + } + Assert.assertEquals(toCompact, ((DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy()) + .shouldPerformMajorCompaction(candidates)); + request = (DateTieredCompactionRequest) ((DateTieredCompactionPolicy) store.storeEngine + .getCompactionPolicy()).selectMajorCompaction(candidates); + } else { + Assert.assertEquals(toCompact, ((DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy()) + .needsCompaction(candidates, ImmutableList.<StoreFile> of())); + request = (DateTieredCompactionRequest) ((DateTieredCompactionPolicy) store.storeEngine + .getCompactionPolicy()).selectMinorCompaction(candidates, false, false); + } + List<StoreFile> actual = Lists.newArrayList(request.getFiles()); + Assert.assertEquals(Arrays.toString(expectedFileSizes), Arrays.toString(getSizes(actual))); + Assert.assertEquals(Arrays.toString(expectedBoundaries), + Arrays.toString(request.getBoundaries().toArray())); } /** @@ -92,7 +117,8 @@ public class TestDateTieredCompaction extends TestCompactionPolicy { long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 }; - compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 13, 12, 11, 10); + compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 }, + new long[] { Long.MIN_VALUE, 12 }, false, true); } /** @@ -105,7 +131,22 @@ public class TestDateTieredCompaction extends TestCompactionPolicy { long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }; long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11 }; - compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 25, 24, 23, 22, 21, 20); + compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21, 22, 23, + 24, 25 }, new long[] { Long.MIN_VALUE, 6}, false, true); + } + + /** + * Test for file on the upper bound of incoming window + * @throws IOException with error + */ + @Test + public void OnUpperBoundOfIncomingWindow() throws IOException { + long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18 }; + long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 }; + + compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 }, + new long[] { Long.MIN_VALUE, 12 }, false, true); } /** @@ -115,23 +156,25 @@ public class TestDateTieredCompaction extends TestCompactionPolicy { @Test public void NewerThanIncomingWindow() throws IOException { long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18 }; + long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 19 }; long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 }; - compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 13, 12, 11, 10); + compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 }, + new long[] { Long.MIN_VALUE, 12}, false, true); } /** - * If there is no T1 window, we don't build 2 + * If there is no T1 window, we don't build T2 * @throws IOException with error */ @Test public void NoT2() throws IOException { long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - long[] maxTimestamps = new long[] { 44, 60, 61, 92, 95, 100 }; + long[] maxTimestamps = new long[] { 44, 60, 61, 97, 100, 193 }; long[] sizes = new long[] { 0, 20, 21, 22, 23, 1 }; - compactEquals(100, sfCreate(minTimestamps, maxTimestamps, sizes), 23, 22); + compactEquals(194, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 22, 23 }, + new long[] { Long.MIN_VALUE, 96}, false, true); } @Test @@ -140,7 +183,8 @@ public class TestDateTieredCompaction extends TestCompactionPolicy { long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 }; long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 30, 31, 32, 2, 1 }; - compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), 32, 31, 30); + compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31, 32 }, + new long[] { Long.MIN_VALUE, 120 }, false, true); } /** @@ -153,7 +197,8 @@ public class TestDateTieredCompaction extends TestCompactionPolicy { long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }; long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 280, 23, 24, 1 }; - compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 22, 21, 20); + compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21, 22 }, + new long[] { Long.MIN_VALUE }, false, true); } /** @@ -166,7 +211,8 @@ public class TestDateTieredCompaction extends TestCompactionPolicy { long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 }; long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 350, 30, 31, 2, 1 }; - compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), 31, 30); + compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31 }, + new long[] { Long.MIN_VALUE }, false, true); } /** @@ -179,33 +225,101 @@ public class TestDateTieredCompaction extends TestCompactionPolicy { long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 8, 9, 10, 11, 12 }; long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 280, 23, 24, 1 }; - compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 24, 23); + compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 23, 24 }, + new long[] { Long.MIN_VALUE }, false, true); } /** - * Older than now(161) - maxAge(100) - * @throws IOException with error - */ - @Test - public void olderThanMaxAge() throws IOException { - long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 }; - long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 }; - - compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), 31, 30, 33, 42, 41, 40); - } + * Older than now(161) - maxAge(100) + * @throws IOException with error + */ + @Test + public void olderThanMaxAge() throws IOException { + long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 }; + long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 }; + + compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 40, 41, 42, 33, + 30, 31 }, new long[] { Long.MIN_VALUE, 96 }, false, true); + } /** * Out-of-order data * @throws IOException with error */ @Test - public void OutOfOrder() throws IOException { + public void outOfOrder() throws IOException { long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; long[] maxTimestamps = new long[] { 0, 13, 3, 10, 11, 1, 2, 12, 14, 15 }; long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 28, 23, 24, 1 }; - compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 1, 24, 23, 28, 22, 34, - 33, 32, 31); + compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 31, 32, 33, 34, + 22, 28, 23, 24, 1 }, new long[] { Long.MIN_VALUE, 12 }, false, true); + } + + /** + * Negative epoch time + * @throws IOException with error + */ + @Test + public void negativeEpochtime() throws IOException { + long[] minTimestamps = + new long[] { -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000 }; + long[] maxTimestamps = new long[] { -28, -11, -10, -9, -8, -7, -6, -5, -4, -3 }; + long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 25, 23, 24, 1 }; + + compactEquals(1, sfCreate(minTimestamps, maxTimestamps, sizes), + new long[] { 31, 32, 33, 34, 22, 25, 23, 24, 1 }, + new long[] { Long.MIN_VALUE, -24 }, false, true); + } + + /** + * Major compaction + * @throws IOException with error + */ + @Test + public void majorCompation() throws IOException { + long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 }; + long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 }; + + compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 0, 50, 51, 40,41, 42, + 33, 30, 31, 2, 1 }, new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true); + } + + /** + * Major compaction with negative numbers + * @throws IOException with error + */ + @Test + public void negativeForMajor() throws IOException { + long[] minTimestamps = + new long[] { -155, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100 }; + long[] maxTimestamps = new long[] { -8, -7, -6, -5, -4, -3, -2, -1, 0, 6, 13 }; + long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 }; + + compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 0, 50, 51, 40, + 41, 42, 33, 30, 31, 2, 1 }, + new long[] { Long.MIN_VALUE, -144, -120, -96, -72, -48, -24, 0, 6, 12 }, true, true); + } + + /** + * Major compaction with maximum values + * @throws IOException with error + */ + @Test + public void maxValuesForMajor() throws IOException { + conf.setLong(CompactionConfiguration.BASE_WINDOW_MILLIS_KEY, Long.MAX_VALUE / 2); + conf.setInt(CompactionConfiguration.WINDOWS_PER_TIER_KEY, 2); + store.storeEngine.getCompactionPolicy().setConf(conf); + long[] minTimestamps = + new long[] { Long.MIN_VALUE, -100 }; + long[] maxTimestamps = new long[] { -8, Long.MAX_VALUE }; + long[] sizes = new long[] { 0, 1 }; + + compactEquals(Long.MAX_VALUE, sfCreate(minTimestamps, maxTimestamps, sizes), + new long[] { 0, 1 }, + new long[] { Long.MIN_VALUE, -4611686018427387903L, 0, 4611686018427387903L, + 9223372036854775806L }, true, true); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/940bde52/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java index 11a63b4..4448572 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java @@ -25,6 +25,8 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -89,6 +91,10 @@ public class TestDefaultCompactSelection extends TestCompactionPolicy { conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1); conf.setFloat("hbase.hregion.majorcompaction.jitter", 0); store.storeEngine.getCompactionPolicy().setConf(conf); + ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge(); + EnvironmentEdgeManager.injectEdge(timeMachine); + // Has to be > 0 and < now. + timeMachine.setValue(1); try { // trigger an aged major compaction compactEquals(sfCreate(50,25,12,12), 50, 25, 12, 12); http://git-wip-us.apache.org/repos/asf/hbase/blob/940bde52/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java index 412e5a7..9a4bb8e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java @@ -42,7 +42,7 @@ public class EverythingPolicy extends RatioBasedCompactionPolicy { } @Override - final ArrayList<StoreFile> applyCompactionPolicy(final ArrayList<StoreFile> candidates, + protected final ArrayList<StoreFile> applyCompactionPolicy(final ArrayList<StoreFile> candidates, final boolean mayUseOffPeak, final boolean mayBeStuck) throws IOException { if (candidates.size() < comConf.getMinFilesToCompact()) {
