Author: brandonwilliams Date: Mon Mar 21 23:56:35 2011 New Revision: 1084028
URL: http://svn.apache.org/viewvc?rev=1084028&view=rev Log: add cache loading to row/key cache tests. Patch by Pavel Yaskevich and Matthew Dennis, reviewed by Pavel Yaskevich for CASSANDRA-2227 Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/cache/InstrumentedCache.java cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/CleanupHelper.java cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/KeyCacheTest.java cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/RowCacheTest.java Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/cache/InstrumentedCache.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/cache/InstrumentedCache.java?rev=1084028&r1=1084027&r2=1084028&view=diff ============================================================================== --- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/cache/InstrumentedCache.java (original) +++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/cache/InstrumentedCache.java Mon Mar 21 23:56:35 2011 @@ -21,6 +21,7 @@ package org.apache.cassandra.cache; */ +import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; @@ -139,4 +140,9 @@ public class InstrumentedCache<K, V> { return map.keySet(); } + + public Set<Map.Entry<K, V>> getEntrySet() + { + return map.entrySet(); + } } Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1084028&r1=1084027&r2=1084028&view=diff ============================================================================== --- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original) +++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Mon Mar 21 23:56:35 2011 @@ -35,6 +35,7 @@ import org.apache.commons.lang.StringUti import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.cache.JMXInstrumentedCache; import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor; import org.apache.cassandra.concurrent.NamedThreadFactory; import org.apache.cassandra.concurrent.StageManager; @@ -1799,6 +1800,16 @@ public class ColumnFamilyStore implement return ssTables.getKeyCache().getSize(); } + public JMXInstrumentedCache<DecoratedKey, ColumnFamily> getRowCache() + { + return ssTables.getRowCache(); + } + + public JMXInstrumentedCache<Pair<Descriptor, DecoratedKey>, Long> getKeyCache() + { + return ssTables.getKeyCache(); + } + public static Iterable<ColumnFamilyStore> all() { Iterable<ColumnFamilyStore>[] stores = new Iterable[DatabaseDescriptor.getTables().size()]; Modified: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/CleanupHelper.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/CleanupHelper.java?rev=1084028&r1=1084027&r2=1084028&view=diff ============================================================================== --- cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/CleanupHelper.java (original) +++ cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/CleanupHelper.java Mon Mar 21 23:56:35 2011 @@ -20,15 +20,21 @@ package org.apache.cassandra; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.RowMutation; +import org.apache.cassandra.db.Table; import org.apache.cassandra.db.commitlog.CommitLog; +import org.apache.cassandra.db.filter.QueryPath; import org.apache.cassandra.io.util.FileUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.cassandra.utils.ByteBufferUtil; public class CleanupHelper extends SchemaLoader { @@ -76,4 +82,30 @@ public class CleanupHelper extends Schem throw new RuntimeException(e); } } + + protected void insertData(String keyspace, String columnFamily, int offset, int numberOfRows) throws IOException + { + for (int i = offset; i < offset + numberOfRows; i++) + { + ByteBuffer key = ByteBufferUtil.bytes("key" + i); + RowMutation rowMutation = new RowMutation(keyspace, key); + QueryPath path = new QueryPath(columnFamily, null, ByteBufferUtil.bytes("col" + i)); + + rowMutation.add(path, ByteBufferUtil.bytes("val" + i), System.currentTimeMillis()); + rowMutation.applyUnsafe(); + } + } + + /* usually used to populate the cache */ + protected void readData(String keyspace, String columnFamily, int offset, int numberOfRows) throws IOException + { + ColumnFamilyStore store = Table.open(keyspace).getColumnFamilyStore(columnFamily); + for (int i = offset; i < offset + numberOfRows; i++) + { + DecoratedKey key = Util.dk("key" + i); + QueryPath path = new QueryPath(columnFamily, null, ByteBufferUtil.bytes("col" + i)); + + store.getColumnFamily(key, path, ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.EMPTY_BYTE_BUFFER, false, 1); + } + } } Modified: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/KeyCacheTest.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/KeyCacheTest.java?rev=1084028&r1=1084027&r2=1084028&view=diff ============================================================================== --- cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/KeyCacheTest.java (original) +++ cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/KeyCacheTest.java Mon Mar 21 23:56:35 2011 @@ -22,6 +22,8 @@ package org.apache.cassandra.db; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.ExecutionException; import org.junit.Test; @@ -29,22 +31,71 @@ import org.junit.Test; import org.apache.cassandra.CleanupHelper; import org.apache.cassandra.Util; import org.apache.cassandra.db.filter.QueryPath; +import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.Pair; public class KeyCacheTest extends CleanupHelper { private static final String TABLE1 = "KeyCacheSpace"; + private static final String COLUMN_FAMILY1 = "Standard1"; + private static final String COLUMN_FAMILY2 = "Standard2"; @Test public void testKeyCache50() throws IOException, ExecutionException, InterruptedException { - testKeyCache("Standard1", 64); + testKeyCache(COLUMN_FAMILY1, 64); } @Test public void testKeyCache100() throws IOException, ExecutionException, InterruptedException { - testKeyCache("Standard2", 128); + testKeyCache(COLUMN_FAMILY2, 128); + } + + @Test + public void testKeyCacheLoad() throws Exception + { + CompactionManager.instance.disableAutoCompaction(); + + ColumnFamilyStore store = Table.open(TABLE1).getColumnFamilyStore(COLUMN_FAMILY2); + + // empty the cache + store.invalidateKeyCache(); + assert store.getKeyCacheSize() == 0; + + // insert data and force to disk + insertData(TABLE1, COLUMN_FAMILY2, 0, 100); + store.forceBlockingFlush(); + + // populate the cache + readData(TABLE1, COLUMN_FAMILY2, 0, 100); + assert store.getKeyCacheSize() == 100; + + // really? our caches don't implement the map interface? (hence no .addAll) + Map<Pair<Descriptor, DecoratedKey>, Long> savedMap = new HashMap<Pair<Descriptor, DecoratedKey>, Long>(); + for (Map.Entry<Pair<Descriptor, DecoratedKey>, Long> entry : store.getKeyCache().getEntrySet()) + { + savedMap.put(entry.getKey(), entry.getValue()); + } + + // force the cache to disk + store.submitKeyCacheWrite().get(); + + // empty the cache again to make sure values came from disk + store.invalidateKeyCache(); + assert store.getKeyCacheSize() == 0; + + // load the cache from disk + store.unregisterMBean(); // unregistering old MBean to test how key cache will be loaded + ColumnFamilyStore newStore = ColumnFamilyStore.createColumnFamilyStore(Table.open(TABLE1), COLUMN_FAMILY2); + assert newStore.getKeyCacheSize() == 100; + + assert savedMap.size() == 100; + for (Map.Entry<Pair<Descriptor, DecoratedKey>, Long> entry : savedMap.entrySet()) + { + assert newStore.getKeyCache().get(entry.getKey()).equals(entry.getValue()); + } } public void testKeyCache(String cfName, int expectedCacheSize) throws IOException, ExecutionException, InterruptedException @@ -87,4 +138,5 @@ public class KeyCacheTest extends Cleanu CompactionManager.instance.submitMajor(store, 0, Integer.MAX_VALUE).get(); keyCacheSize = store.getKeyCacheCapacity(); assert keyCacheSize == 1 : keyCacheSize; - }} + } +} Modified: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/RowCacheTest.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/RowCacheTest.java?rev=1084028&r1=1084027&r2=1084028&view=diff ============================================================================== --- cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/RowCacheTest.java (original) +++ cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/RowCacheTest.java Mon Mar 21 23:56:35 2011 @@ -18,33 +18,33 @@ package org.apache.cassandra.db; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Collection; -import org.apache.cassandra.CleanupHelper; +import org.junit.Test; +import org.apache.cassandra.CleanupHelper; import org.apache.cassandra.Util; import org.apache.cassandra.db.filter.QueryPath; import org.apache.cassandra.utils.ByteBufferUtil; -import org.junit.Test; - public class RowCacheTest extends CleanupHelper { + private String KEYSPACE = "RowCacheSpace"; + private String COLUMN_FAMILY_WITH_CACHE = "CachedCF"; + private String COLUMN_FAMILY_WITHOUT_CACHE = "CFWithoutCache"; + @Test public void testRowCache() throws Exception { - String KEYSPACE = "RowCacheSpace"; - String COLUMN_FAMILY_WITH_CACHE = "CachedCF"; - String COLUMN_FAMILY_WITHOUT_CACHE = "CFWithoutCache"; - CompactionManager.instance.disableAutoCompaction(); Table table = Table.open(KEYSPACE); ColumnFamilyStore cachedStore = table.getColumnFamilyStore(COLUMN_FAMILY_WITH_CACHE); ColumnFamilyStore noCacheStore = table.getColumnFamilyStore(COLUMN_FAMILY_WITHOUT_CACHE); + // empty the row cache + cachedStore.invalidateRowCache(); + // inserting 100 rows into both column families insertData(KEYSPACE, COLUMN_FAMILY_WITH_CACHE, 0, 100); insertData(KEYSPACE, COLUMN_FAMILY_WITHOUT_CACHE, 0, 100); @@ -109,17 +109,37 @@ public class RowCacheTest extends Cleanu } } - private void insertData(String keyspace, String columnFamily, int offset, int numberOfRows) throws IOException + @Test + public void testRowCacheLoad() throws Exception { - for (int i = offset; i < offset + numberOfRows; i++) - { - ByteBuffer key = ByteBufferUtil.bytes("key" + i); - RowMutation rowMutation = new RowMutation(keyspace, key); - QueryPath path = new QueryPath(columnFamily, null, ByteBufferUtil.bytes("col" + i)); + CompactionManager.instance.disableAutoCompaction(); + + ColumnFamilyStore store = Table.open(KEYSPACE).getColumnFamilyStore(COLUMN_FAMILY_WITH_CACHE); + + // empty the cache + store.invalidateRowCache(); + assert store.getRowCacheSize() == 0; + + // insert data and fill the cache + insertData(KEYSPACE, COLUMN_FAMILY_WITH_CACHE, 0, 100); + readData(KEYSPACE, COLUMN_FAMILY_WITH_CACHE, 0, 100); + assert store.getRowCacheSize() == 100; - rowMutation.add(path, ByteBufferUtil.bytes("val" + i), System.currentTimeMillis()); - rowMutation.applyUnsafe(); + // force the cache to disk + store.submitRowCacheWrite().get(); + + // empty the cache again to make sure values came from disk + store.invalidateRowCache(); + assert store.getRowCacheSize() == 0; + + // load the cache from disk + store.initRowCache(); + assert store.getRowCacheSize() == 100; + + for (int i = 0; i < 100; i++) + { + // verify the correct data was found + assert store.getRawCachedRow(Util.dk("key" + i)).getColumn(ByteBufferUtil.bytes("col" + i)).value().equals(ByteBufferUtil.bytes("val" + i)); } } - }
