Updated Branches: refs/heads/trunk 98eb543a9 -> f92fb2241
move testStandardColumnCompactions and testSuperColumnCompactions to LongSompactionsTest Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f92fb224 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f92fb224 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f92fb224 Branch: refs/heads/trunk Commit: f92fb2241fc2ea5c7fb25ee885337404de3fbb49 Parents: 98eb543 Author: Jonathan Ellis <jbel...@apache.org> Authored: Fri Sep 21 10:05:51 2012 -0500 Committer: Jonathan Ellis <jbel...@apache.org> Committed: Fri Sep 21 10:05:51 2012 -0500 ---------------------------------------------------------------------- .../db/compaction/LongCompactionSpeedTest.java | 102 ------- .../db/compaction/LongCompactionsTest.java | 217 +++++++++++++++ .../cassandra/db/compaction/CompactionsTest.java | 107 +------- 3 files changed, 218 insertions(+), 208 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/f92fb224/test/long/org/apache/cassandra/db/compaction/LongCompactionSpeedTest.java ---------------------------------------------------------------------- diff --git a/test/long/org/apache/cassandra/db/compaction/LongCompactionSpeedTest.java b/test/long/org/apache/cassandra/db/compaction/LongCompactionSpeedTest.java deleted file mode 100644 index 8338c2c..0000000 --- a/test/long/org/apache/cassandra/db/compaction/LongCompactionSpeedTest.java +++ /dev/null @@ -1,102 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ -package org.apache.cassandra.db.compaction; - -import java.util.*; - -import org.apache.cassandra.config.Schema; -import org.junit.Test; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import org.apache.cassandra.db.*; -import org.apache.cassandra.io.sstable.SSTableReader; -import org.apache.cassandra.io.sstable.SSTableUtils; - -public class LongCompactionSpeedTest extends SchemaLoader -{ - public static final String TABLE1 = "Keyspace1"; - - /** - * Test compaction with a very wide row. - */ - @Test - public void testCompactionWide() throws Exception - { - testCompaction(2, 1, 200000); - } - - /** - * Test compaction with lots of skinny rows. - */ - @Test - public void testCompactionSlim() throws Exception - { - testCompaction(2, 200000, 1); - } - - /** - * Test compaction with lots of small sstables. - */ - @Test - public void testCompactionMany() throws Exception - { - testCompaction(100, 800, 5); - } - - protected void testCompaction(int sstableCount, int rowsPerSSTable, int colsPerRow) throws Exception - { - CompactionManager.instance.disableAutoCompaction(); - - Table table = Table.open(TABLE1); - ColumnFamilyStore store = table.getColumnFamilyStore("Standard1"); - - ArrayList<SSTableReader> sstables = new ArrayList<SSTableReader>(); - for (int k = 0; k < sstableCount; k++) - { - SortedMap<String,ColumnFamily> rows = new TreeMap<String,ColumnFamily>(); - for (int j = 0; j < rowsPerSSTable; j++) - { - String key = String.valueOf(j); - IColumn[] cols = new IColumn[colsPerRow]; - for (int i = 0; i < colsPerRow; i++) - { - // last sstable has highest timestamps - cols[i] = Util.column(String.valueOf(i), String.valueOf(i), k); - } - rows.put(key, SSTableUtils.createCF(Long.MIN_VALUE, Integer.MIN_VALUE, cols)); - } - SSTableReader sstable = SSTableUtils.prepare().write(rows); - sstables.add(sstable); - store.addSSTable(sstable); - } - - // give garbage collection a bit of time to catch up - Thread.sleep(1000); - - long start = System.currentTimeMillis(); - final int gcBefore = (int) (System.currentTimeMillis() / 1000) - Schema.instance.getCFMetaData(TABLE1, "Standard1").getGcGraceSeconds(); - new CompactionTask(store, sstables, gcBefore).execute(null); - System.out.println(String.format("%s: sstables=%d rowsper=%d colsper=%d: %d ms", - this.getClass().getName(), - sstableCount, - rowsPerSSTable, - colsPerRow, - System.currentTimeMillis() - start)); - } -} http://git-wip-us.apache.org/repos/asf/cassandra/blob/f92fb224/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java ---------------------------------------------------------------------- diff --git a/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java new file mode 100644 index 0000000..4407170 --- /dev/null +++ b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java @@ -0,0 +1,217 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ +package org.apache.cassandra.db.compaction; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.Schema; +import org.junit.Test; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.filter.QueryPath; +import org.apache.cassandra.io.sstable.SSTableReader; +import org.apache.cassandra.io.sstable.SSTableUtils; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; + +import static junit.framework.Assert.assertEquals; + +public class LongCompactionsTest extends SchemaLoader +{ + public static final String TABLE1 = "Keyspace1"; + + /** + * Test compaction with a very wide row. + */ + @Test + public void testCompactionWide() throws Exception + { + testCompaction(2, 1, 200000); + } + + /** + * Test compaction with lots of skinny rows. + */ + @Test + public void testCompactionSlim() throws Exception + { + testCompaction(2, 200000, 1); + } + + /** + * Test compaction with lots of small sstables. + */ + @Test + public void testCompactionMany() throws Exception + { + testCompaction(100, 800, 5); + } + + protected void testCompaction(int sstableCount, int rowsPerSSTable, int colsPerRow) throws Exception + { + CompactionManager.instance.disableAutoCompaction(); + + Table table = Table.open(TABLE1); + ColumnFamilyStore store = table.getColumnFamilyStore("Standard1"); + + ArrayList<SSTableReader> sstables = new ArrayList<SSTableReader>(); + for (int k = 0; k < sstableCount; k++) + { + SortedMap<String,ColumnFamily> rows = new TreeMap<String,ColumnFamily>(); + for (int j = 0; j < rowsPerSSTable; j++) + { + String key = String.valueOf(j); + IColumn[] cols = new IColumn[colsPerRow]; + for (int i = 0; i < colsPerRow; i++) + { + // last sstable has highest timestamps + cols[i] = Util.column(String.valueOf(i), String.valueOf(i), k); + } + rows.put(key, SSTableUtils.createCF(Long.MIN_VALUE, Integer.MIN_VALUE, cols)); + } + SSTableReader sstable = SSTableUtils.prepare().write(rows); + sstables.add(sstable); + store.addSSTable(sstable); + } + + // give garbage collection a bit of time to catch up + Thread.sleep(1000); + + long start = System.currentTimeMillis(); + final int gcBefore = (int) (System.currentTimeMillis() / 1000) - Schema.instance.getCFMetaData(TABLE1, "Standard1").getGcGraceSeconds(); + new CompactionTask(store, sstables, gcBefore).execute(null); + System.out.println(String.format("%s: sstables=%d rowsper=%d colsper=%d: %d ms", + this.getClass().getName(), + sstableCount, + rowsPerSSTable, + colsPerRow, + System.currentTimeMillis() - start)); + } + + @Test + public void testStandardColumnCompactions() throws IOException, ExecutionException, InterruptedException + { + // this test does enough rows to force multiple block indexes to be used + Table table = Table.open(TABLE1); + ColumnFamilyStore cfs = table.getColumnFamilyStore("Standard1"); + + final int ROWS_PER_SSTABLE = 10; + final int SSTABLES = DatabaseDescriptor.getIndexInterval() * 3 / ROWS_PER_SSTABLE; + + // disable compaction while flushing + cfs.disableAutoCompaction(); + + long maxTimestampExpected = Long.MIN_VALUE; + Set<DecoratedKey> inserted = new HashSet<DecoratedKey>(); + for (int j = 0; j < SSTABLES; j++) { + for (int i = 0; i < ROWS_PER_SSTABLE; i++) { + DecoratedKey key = Util.dk(String.valueOf(i % 2)); + RowMutation rm = new RowMutation(TABLE1, key.key); + long timestamp = j * ROWS_PER_SSTABLE + i; + rm.add(new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i / 2))), + ByteBufferUtil.EMPTY_BYTE_BUFFER, + timestamp); + maxTimestampExpected = Math.max(timestamp, maxTimestampExpected); + rm.apply(); + inserted.add(key); + } + cfs.forceBlockingFlush(); + CompactionsTest.assertMaxTimestamp(cfs, maxTimestampExpected); + assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(cfs).size()); + } + + forceCompactions(cfs); + + assertEquals(inserted.size(), Util.getRangeSlice(cfs).size()); + + // make sure max timestamp of compacted sstables is recorded properly after compaction. + CompactionsTest.assertMaxTimestamp(cfs, maxTimestampExpected); + cfs.truncate(); + } + + @Test + public void testSuperColumnCompactions() throws IOException, ExecutionException, InterruptedException + { + Table table = Table.open(TABLE1); + ColumnFamilyStore cfs = table.getColumnFamilyStore("Super1"); + + final int ROWS_PER_SSTABLE = 10; + final int SSTABLES = DatabaseDescriptor.getIndexInterval() * 3 / ROWS_PER_SSTABLE; + + //disable compaction while flushing + cfs.disableAutoCompaction(); + + long maxTimestampExpected = Long.MIN_VALUE; + Set<DecoratedKey> inserted = new HashSet<DecoratedKey>(); + ByteBuffer superColumn = ByteBufferUtil.bytes("TestSuperColumn"); + for (int j = 0; j < SSTABLES; j++) + { + for (int i = 0; i < ROWS_PER_SSTABLE; i++) + { + DecoratedKey key = Util.dk(String.valueOf(i % 2)); + RowMutation rm = new RowMutation(TABLE1, key.key); + long timestamp = j * ROWS_PER_SSTABLE + i; + rm.add(new QueryPath("Super1", superColumn, ByteBufferUtil.bytes((long)(i / 2))), + ByteBufferUtil.EMPTY_BYTE_BUFFER, + timestamp); + maxTimestampExpected = Math.max(timestamp, maxTimestampExpected); + rm.apply(); + inserted.add(key); + } + cfs.forceBlockingFlush(); + CompactionsTest.assertMaxTimestamp(cfs, maxTimestampExpected); + assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(cfs, superColumn).size()); + } + + forceCompactions(cfs); + + assertEquals(inserted.size(), Util.getRangeSlice(cfs, superColumn).size()); + + // make sure max timestamp of compacted sstables is recorded properly after compaction. + CompactionsTest.assertMaxTimestamp(cfs, maxTimestampExpected); + } + + private void forceCompactions(ColumnFamilyStore cfs) throws ExecutionException, InterruptedException + { + // re-enable compaction with thresholds low enough to force a few rounds + cfs.setCompactionThresholds(2, 4); + + // loop submitting parallel compactions until they all return 0 + do + { + ArrayList<Future<?>> compactions = new ArrayList<Future<?>>(); + for (int i = 0; i < 10; i++) + compactions.add(CompactionManager.instance.submitBackground(cfs)); + // another compaction attempt will be launched in the background by + // each completing compaction: not much we can do to control them here + FBUtilities.waitOnFutures(compactions); + } while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0); + + if (cfs.getSSTables().size() > 1) + { + CompactionManager.instance.performMaximal(cfs); + } + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/f92fb224/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java index dc5a256..6795f87 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java @@ -62,47 +62,6 @@ public class CompactionsTest extends SchemaLoader testBlacklisting(LeveledCompactionStrategy.class.getCanonicalName()); } - @Test - public void testStandardColumnCompactions() throws IOException, ExecutionException, InterruptedException - { - // this test does enough rows to force multiple block indexes to be used - Table table = Table.open(TABLE1); - ColumnFamilyStore cfs = table.getColumnFamilyStore("Standard1"); - - final int ROWS_PER_SSTABLE = 10; - final int SSTABLES = DatabaseDescriptor.getIndexInterval() * 3 / ROWS_PER_SSTABLE; - - // disable compaction while flushing - cfs.disableAutoCompaction(); - - long maxTimestampExpected = Long.MIN_VALUE; - Set<DecoratedKey> inserted = new HashSet<DecoratedKey>(); - for (int j = 0; j < SSTABLES; j++) { - for (int i = 0; i < ROWS_PER_SSTABLE; i++) { - DecoratedKey key = Util.dk(String.valueOf(i % 2)); - RowMutation rm = new RowMutation(TABLE1, key.key); - long timestamp = j * ROWS_PER_SSTABLE + i; - rm.add(new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i / 2))), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp); - maxTimestampExpected = Math.max(timestamp, maxTimestampExpected); - rm.apply(); - inserted.add(key); - } - cfs.forceBlockingFlush(); - assertMaxTimestamp(cfs, maxTimestampExpected); - assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(cfs).size()); - } - - forceCompactions(cfs); - - assertEquals(inserted.size(), Util.getRangeSlice(cfs).size()); - - // make sure max timestamp of compacted sstables is recorded properly after compaction. - assertMaxTimestamp(cfs, maxTimestampExpected); - cfs.truncate(); - } - /** * Test to see if sstable has enough expired columns, it is compacted itself. */ @@ -205,48 +164,6 @@ public class CompactionsTest extends SchemaLoader } @Test - public void testSuperColumnCompactions() throws IOException, ExecutionException, InterruptedException - { - Table table = Table.open(TABLE1); - ColumnFamilyStore cfs = table.getColumnFamilyStore("Super1"); - - final int ROWS_PER_SSTABLE = 10; - final int SSTABLES = DatabaseDescriptor.getIndexInterval() * 3 / ROWS_PER_SSTABLE; - - //disable compaction while flushing - cfs.disableAutoCompaction(); - - long maxTimestampExpected = Long.MIN_VALUE; - Set<DecoratedKey> inserted = new HashSet<DecoratedKey>(); - ByteBuffer superColumn = ByteBufferUtil.bytes("TestSuperColumn"); - for (int j = 0; j < SSTABLES; j++) - { - for (int i = 0; i < ROWS_PER_SSTABLE; i++) - { - DecoratedKey key = Util.dk(String.valueOf(i % 2)); - RowMutation rm = new RowMutation(TABLE1, key.key); - long timestamp = j * ROWS_PER_SSTABLE + i; - rm.add(new QueryPath("Super1", superColumn, ByteBufferUtil.bytes((long)(i / 2))), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp); - maxTimestampExpected = Math.max(timestamp, maxTimestampExpected); - rm.apply(); - inserted.add(key); - } - cfs.forceBlockingFlush(); - assertMaxTimestamp(cfs, maxTimestampExpected); - assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(cfs, superColumn).size()); - } - - forceCompactions(cfs); - - assertEquals(inserted.size(), Util.getRangeSlice(cfs, superColumn).size()); - - // make sure max timestamp of compacted sstables is recorded properly after compaction. - assertMaxTimestamp(cfs, maxTimestampExpected); - } - - @Test public void testSuperColumnTombstones() throws IOException, ExecutionException, InterruptedException { Table table = Table.open(TABLE1); @@ -283,7 +200,7 @@ public class CompactionsTest extends SchemaLoader assert sc.getSubColumns().isEmpty(); } - public void assertMaxTimestamp(ColumnFamilyStore cfs, long maxTimestampExpected) + public static void assertMaxTimestamp(ColumnFamilyStore cfs, long maxTimestampExpected) { long maxTimestampObserved = Long.MIN_VALUE; for (SSTableReader sstable : cfs.getSSTables()) @@ -291,28 +208,6 @@ public class CompactionsTest extends SchemaLoader assertEquals(maxTimestampExpected, maxTimestampObserved); } - private void forceCompactions(ColumnFamilyStore cfs) throws ExecutionException, InterruptedException - { - // re-enable compaction with thresholds low enough to force a few rounds - cfs.setCompactionThresholds(2, 4); - - // loop submitting parallel compactions until they all return 0 - do - { - ArrayList<Future<?>> compactions = new ArrayList<Future<?>>(); - for (int i = 0; i < 10; i++) - compactions.add(CompactionManager.instance.submitBackground(cfs)); - // another compaction attempt will be launched in the background by - // each completing compaction: not much we can do to control them here - FBUtilities.waitOnFutures(compactions); - } while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0); - - if (cfs.getSSTables().size() > 1) - { - CompactionManager.instance.performMaximal(cfs); - } - } - @Test public void testEchoedRow() throws IOException, ExecutionException, InterruptedException {
