Author: jbellis
Date: Mon Apr 27 18:24:18 2009
New Revision: 769087
URL: http://svn.apache.org/viewvc?rev=769087&view=rev
Log:
add SSTable tests. in particular we want to be sure to test bloomfilter
[de]serialize since that's going to change a lot.
patch by jbellis; reviewed by Eric Evans for CASSANDRA-37
Removed:
incubator/cassandra/trunk/src/org/apache/cassandra/test/SSTableTest.java
Modified:
incubator/cassandra/trunk/src/org/apache/cassandra/db/BinaryMemtable.java
incubator/cassandra/trunk/src/org/apache/cassandra/db/ColumnFamilyStore.java
incubator/cassandra/trunk/src/org/apache/cassandra/db/FileStruct.java
incubator/cassandra/trunk/src/org/apache/cassandra/db/Memtable.java
incubator/cassandra/trunk/src/org/apache/cassandra/db/Table.java
incubator/cassandra/trunk/src/org/apache/cassandra/io/SSTable.java
incubator/cassandra/trunk/src/org/apache/cassandra/loader/Loader.java
incubator/cassandra/trunk/src/org/apache/cassandra/loader/PreLoad.java
incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnComparatorFactoryTest.java
incubator/cassandra/trunk/test/unit/org/apache/cassandra/io/SSTableTest.java
Modified:
incubator/cassandra/trunk/src/org/apache/cassandra/db/BinaryMemtable.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/org/apache/cassandra/db/BinaryMemtable.java?rev=769087&r1=769086&r2=769087&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/org/apache/cassandra/db/BinaryMemtable.java
(original)
+++ incubator/cassandra/trunk/src/org/apache/cassandra/db/BinaryMemtable.java
Mon Apr 27 18:24:18 2009
@@ -31,6 +31,7 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.utils.BloomFilter;
import org.apache.cassandra.io.SSTable;
+import org.apache.cassandra.service.StorageService;
import org.apache.log4j.Logger;
import org.cliffc.high_scale_lib.NonBlockingHashMap;
@@ -144,7 +145,7 @@
* Use the SSTable to write the contents of the TreeMap
* to disk.
*/
- SSTable ssTable = new SSTable(directory, filename);
+ SSTable ssTable = new SSTable(directory, filename,
StorageService.getPartitioner());
List<String> keys = new ArrayList<String>( columnFamilies_.keySet() );
Collections.sort(keys);
/* Use this BloomFilter to decide if a key exists in a SSTable */
Modified:
incubator/cassandra/trunk/src/org/apache/cassandra/db/ColumnFamilyStore.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/org/apache/cassandra/db/ColumnFamilyStore.java?rev=769087&r1=769086&r2=769087&view=diff
==============================================================================
---
incubator/cassandra/trunk/src/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++
incubator/cassandra/trunk/src/org/apache/cassandra/db/ColumnFamilyStore.java
Mon Apr 27 18:24:18 2009
@@ -267,7 +267,7 @@
boolean bVal = SSTable.isKeyInFile(key, file);
if ( !bVal )
continue;
- SSTable ssTable = new SSTable(file);
+ SSTable ssTable = new SSTable(file,
StorageService.getPartitioner());
ssTable.touch(key, fData);
}
}
@@ -556,7 +556,7 @@
private ColumnFamily fetchColumnFamily(String key, String cf, IFilter
filter, String ssTableFile) throws IOException
{
- SSTable ssTable = new SSTable(ssTableFile);
+ SSTable ssTable = new SSTable(ssTableFile,
StorageService.getPartitioner());
long start = System.currentTimeMillis();
DataInputBuffer bufIn;
bufIn = filter.next(key, cf, ssTable);
@@ -1120,7 +1120,7 @@
if( target != null )
rangeFileLocation =
rangeFileLocation + System.getProperty("file.separator") + "bootstrap";
FileUtils.createDirectory(rangeFileLocation);
- ssTableRange = new
SSTable(rangeFileLocation, mergedFileName);
+ ssTableRange = new
SSTable(rangeFileLocation, mergedFileName, StorageService.getPartitioner());
}
try
{
@@ -1326,7 +1326,7 @@
if ( ssTable == null )
{
- ssTable = new SSTable(compactionFileLocation,
mergedFileName);
+ ssTable = new SSTable(compactionFileLocation,
mergedFileName, StorageService.getPartitioner());
}
ssTable.append(lastkey, bufOut);
Modified: incubator/cassandra/trunk/src/org/apache/cassandra/db/FileStruct.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/org/apache/cassandra/db/FileStruct.java?rev=769087&r1=769086&r2=769087&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/org/apache/cassandra/db/FileStruct.java
(original)
+++ incubator/cassandra/trunk/src/org/apache/cassandra/db/FileStruct.java Mon
Apr 27 18:24:18 2009
@@ -24,6 +24,7 @@
import org.apache.cassandra.io.DataOutputBuffer;
import org.apache.cassandra.io.IFileReader;
import org.apache.cassandra.io.SSTable;
+import org.apache.cassandra.io.Coordinate;
import org.apache.cassandra.dht.IPartitioner;
@@ -74,6 +75,37 @@
return partitioner.getDecoratedKeyComparator().compare(key, f.key);
}
+ public void seekTo(String seekKey)
+ {
+ try
+ {
+ Coordinate range = SSTable.getCoordinates(seekKey, reader,
partitioner);
+ reader.seek(range.end_);
+ long position = reader.getPositionFromBlockIndex(seekKey);
+ if (position == -1)
+ {
+ reader.seek(range.start_);
+ }
+ else
+ {
+ reader.seek(position);
+ }
+
+ while (!exhausted)
+ {
+ advance();
+ if (key.compareTo(seekKey) >= 0)
+ {
+ break;
+ }
+ }
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException("corrupt sstable", e);
+ }
+ }
+
/*
* Read the next key from the data file, skipping block indexes.
* Caller must check isExhausted after each call to see if further
Modified: incubator/cassandra/trunk/src/org/apache/cassandra/db/Memtable.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/org/apache/cassandra/db/Memtable.java?rev=769087&r1=769086&r2=769087&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/org/apache/cassandra/db/Memtable.java
(original)
+++ incubator/cassandra/trunk/src/org/apache/cassandra/db/Memtable.java Mon Apr
27 18:24:18 2009
@@ -372,7 +372,7 @@
String directory = DatabaseDescriptor.getDataFileLocation();
String filename = cfStore.getNextFileName();
- SSTable ssTable = new SSTable(directory, filename);
+ SSTable ssTable = new SSTable(directory, filename,
StorageService.getPartitioner());
// sort keys in the order they would be in when decorated
final IPartitioner partitioner = StorageService.getPartitioner();
Modified: incubator/cassandra/trunk/src/org/apache/cassandra/db/Table.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/org/apache/cassandra/db/Table.java?rev=769087&r1=769086&r2=769087&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/org/apache/cassandra/db/Table.java (original)
+++ incubator/cassandra/trunk/src/org/apache/cassandra/db/Table.java Mon Apr 27
18:24:18 2009
@@ -267,7 +267,7 @@
* list of the associated Column Family. Also merge the CBF
into the
* sampler.
*/
- SSTable ssTable = new SSTable(streamContext.getTargetFile() );
+ SSTable ssTable = new SSTable(streamContext.getTargetFile(),
StorageService.getPartitioner());
ssTable.close();
logger_.debug("Merging the counting bloom filter in the
sampler ...");
String[] peices = FBUtilities.strip(fileName, "-");
Modified: incubator/cassandra/trunk/src/org/apache/cassandra/io/SSTable.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/org/apache/cassandra/io/SSTable.java?rev=769087&r1=769086&r2=769087&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/org/apache/cassandra/io/SSTable.java
(original)
+++ incubator/cassandra/trunk/src/org/apache/cassandra/io/SSTable.java Mon Apr
27 18:24:18 2009
@@ -153,15 +153,17 @@
{
private final String decoratedKey;
private long position_;
+ private IPartitioner partitioner;
- public KeyPositionInfo(String decoratedKey)
+ public KeyPositionInfo(String decoratedKey, IPartitioner partitioner)
{
this.decoratedKey = decoratedKey;
+ this.partitioner = partitioner;
}
- public KeyPositionInfo(String decoratedKey, long position)
+ public KeyPositionInfo(String decoratedKey, IPartitioner partitioner,
long position)
{
- this(decoratedKey);
+ this(decoratedKey, partitioner);
position_ = position;
}
@@ -177,8 +179,7 @@
public int compareTo(KeyPositionInfo kPosInfo)
{
- IPartitioner p = StorageService.getPartitioner();
- return p.getDecoratedKeyComparator().compare(decoratedKey,
kPosInfo.decoratedKey);
+ return
partitioner.getDecoratedKeyComparator().compare(decoratedKey,
kPosInfo.decoratedKey);
}
public String toString()
@@ -197,7 +198,7 @@
* We do this so that we don't read the index file into memory multiple
* times.
*/
- private static Map<String, List<KeyPositionInfo>> indexMetadataMap_ = new
Hashtable<String, List<KeyPositionInfo>>();
+ static Map<String, List<KeyPositionInfo>> indexMetadataMap_ = new
Hashtable<String, List<KeyPositionInfo>>();
/**
* This method deletes both the specified data file
@@ -287,7 +288,7 @@
SSTable ssTable = null;
try
{
- ssTable = new SSTable(filename);
+ ssTable = new SSTable(filename,
StorageService.getPartitioner());
}
catch ( IOException ex )
{
@@ -336,7 +337,7 @@
return bVal;
}
- private String dataFile_;
+ String dataFile_;
private IFileWriter dataWriter_;
private String lastWrittenKey_;
private long firstBlockPosition_ = 0L;
@@ -345,15 +346,17 @@
private SortedMap<String, BlockMetadata> blockIndex_;
/* Holds all the block indicies for this SSTable */
private List<SortedMap<String, BlockMetadata>> blockIndexes_;
+ private IPartitioner partitioner_;
/**
* This ctor basically gets passed in the full path name
* of the data file associated with this SSTable. Use this
* ctor to read the data in this file.
*/
- public SSTable(String dataFileName) throws IOException
+ public SSTable(String dataFileName, IPartitioner partitioner) throws
IOException
{
dataFile_ = dataFileName;
+ partitioner_ = partitioner;
init();
}
@@ -361,10 +364,11 @@
* This ctor is used for writing data into the SSTable. Use this
* version for non DB writes to the SSTable.
*/
- public SSTable(String directory, String filename) throws IOException
+ public SSTable(String directory, String filename, IPartitioner
partitioner) throws IOException
{
- dataFile_ = directory + System.getProperty("file.separator") +
filename + "-Data.db";
- blockIndex_ = new TreeMap<String,
BlockMetadata>(StorageService.getPartitioner().getReverseDecoratedKeyComparator());
+ dataFile_ = directory + System.getProperty("file.separator") +
filename + "-Data.db";
+ partitioner_ = partitioner;
+ blockIndex_ = new TreeMap<String,
BlockMetadata>(partitioner_.getReverseDecoratedKeyComparator());
blockIndexes_ = new ArrayList<SortedMap<String, BlockMetadata>>();
dataWriter_ = SequenceFile.bufferedWriter(dataFile_, 4*1024*1024);
SSTable.positionAfterFirstBlockIndex_ =
dataWriter_.getCurrentPosition();
@@ -461,7 +465,7 @@
/* size of data associated with the key */
bufIn.readLong();
/* load the actual position of the block index
into the index map */
- keyPositionInfos.add( new
KeyPositionInfo(largestKeyInBlock, currentPosition) );
+ keyPositionInfos.add( new
KeyPositionInfo(largestKeyInBlock, partitioner_, currentPosition) );
}
else
{
@@ -532,8 +536,8 @@
try
{
/* Morph the key */
- String decoratedKey =
StorageService.getPartitioner().decorateKey(clientKey);
- Coordinate fileCoordinate = getCoordinates(decoratedKey,
dataReader);
+ String decoratedKey = partitioner_.decorateKey(clientKey);
+ Coordinate fileCoordinate = getCoordinates(decoratedKey,
dataReader, partitioner_);
/* Get offset of key from block Index */
dataReader.seek(fileCoordinate.end_);
BlockMetadata blockMetadata =
dataReader.getBlockMetadata(decoratedKey);
@@ -566,7 +570,7 @@
{
if (decoratedKey == null )
throw new IOException("Keys must not be null.");
- Comparator<String> c =
StorageService.getPartitioner().getDecoratedKeyComparator();
+ Comparator<String> c = partitioner_.getDecoratedKeyComparator();
if ( lastWrittenKey_ != null && c.compare(lastWrittenKey_,
decoratedKey) > 0 )
{
logger_.info("Last written key : " + lastWrittenKey_);
@@ -585,7 +589,7 @@
if ( indexKeysWritten_ == indexInterval_ )
{
blockIndexes_.add(blockIndex_);
- blockIndex_ = new TreeMap<String,
BlockMetadata>(StorageService.getPartitioner().getReverseDecoratedKeyComparator());
+ blockIndex_ = new TreeMap<String,
BlockMetadata>(partitioner_.getReverseDecoratedKeyComparator());
indexKeysWritten_ = 0;
}
}
@@ -638,7 +642,7 @@
SSTable.indexMetadataMap_.put(dataFile_, keyPositionInfos);
}
- keyPositionInfos.add(new KeyPositionInfo(blockIndex.firstKey(),
position));
+ keyPositionInfos.add(new KeyPositionInfo(blockIndex.firstKey(),
partitioner_, position));
blockIndex.clear();
}
@@ -656,7 +660,7 @@
afterAppend(decoratedKey, currentPosition, value.length );
}
- public static Coordinate getCoordinates(String decoratedKey, IFileReader
dataReader) throws IOException
+ public static Coordinate getCoordinates(String decoratedKey, IFileReader
dataReader, IPartitioner partitioner) throws IOException
{
List<KeyPositionInfo> indexInfo =
indexMetadataMap_.get(dataReader.getFileName());
int size = (indexInfo == null) ? 0 : indexInfo.size();
@@ -664,7 +668,7 @@
long end = dataReader.getEOF();
if ( size > 0 )
{
- int index = Collections.binarySearch(indexInfo, new
KeyPositionInfo(decoratedKey));
+ int index = Collections.binarySearch(indexInfo, new
KeyPositionInfo(decoratedKey, partitioner));
if ( index < 0 )
{
/*
@@ -728,8 +732,8 @@
// dataReader = SequenceFile.chksumReader(dataFile_, 4*1024*1024);
/* Morph key into actual key based on the partition type. */
- String decoratedKey =
StorageService.getPartitioner().decorateKey(clientKey);
- Coordinate fileCoordinate = getCoordinates(decoratedKey,
dataReader);
+ String decoratedKey = partitioner_.decorateKey(clientKey);
+ Coordinate fileCoordinate = getCoordinates(decoratedKey,
dataReader, partitioner_);
/*
* we have the position we have to read from in order to get the
* column family, get the column family and column(s) needed.
Modified: incubator/cassandra/trunk/src/org/apache/cassandra/loader/Loader.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/org/apache/cassandra/loader/Loader.java?rev=769087&r1=769086&r2=769087&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/org/apache/cassandra/loader/Loader.java
(original)
+++ incubator/cassandra/trunk/src/org/apache/cassandra/loader/Loader.java Mon
Apr 27 18:24:18 2009
@@ -297,7 +297,7 @@
/* Load the indexes into memory */
for ( String df : ssTables )
{
- SSTable ssTable = new SSTable(df);
+ SSTable ssTable = new SSTable(df,
StorageService.getPartitioner());
ssTable.close();
}
/* We should have only one file since we just compacted. */
Modified: incubator/cassandra/trunk/src/org/apache/cassandra/loader/PreLoad.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/org/apache/cassandra/loader/PreLoad.java?rev=769087&r1=769086&r2=769087&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/org/apache/cassandra/loader/PreLoad.java
(original)
+++ incubator/cassandra/trunk/src/org/apache/cassandra/loader/PreLoad.java Mon
Apr 27 18:24:18 2009
@@ -98,7 +98,7 @@
/* Load the indexes into memory */
for ( String df : ssTables )
{
- SSTable ssTable = new SSTable(df);
+ SSTable ssTable = new SSTable(df,
StorageService.getPartitioner());
ssTable.close();
}
/* We should have only one file since we just compacted. */
Modified:
incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnComparatorFactoryTest.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnComparatorFactoryTest.java?rev=769087&r1=769086&r2=769087&view=diff
==============================================================================
---
incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnComparatorFactoryTest.java
(original)
+++
incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnComparatorFactoryTest.java
Mon Apr 27 18:24:18 2009
@@ -5,7 +5,7 @@
import java.util.Comparator;
public class ColumnComparatorFactoryTest {
- public Comparator<IColumn> nameComparator =
ColumnComparatorFactory.getComparator(ColumnComparatorFactory.ComparatorType.NAME);
+ public Comparator<IColumn> nameComparator =
ColumnComparatorFactory.getComparator(ColumnComparatorFactory.ComparatorType.NAME);
@Test
public void testLT() {
Modified:
incubator/cassandra/trunk/test/unit/org/apache/cassandra/io/SSTableTest.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/test/unit/org/apache/cassandra/io/SSTableTest.java?rev=769087&r1=769086&r2=769087&view=diff
==============================================================================
---
incubator/cassandra/trunk/test/unit/org/apache/cassandra/io/SSTableTest.java
(original)
+++
incubator/cassandra/trunk/test/unit/org/apache/cassandra/io/SSTableTest.java
Mon Apr 27 18:24:18 2009
@@ -1,10 +1,9 @@
package org.apache.cassandra.io;
import org.apache.cassandra.ServerTest;
-import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.dht.OrderPreservingPartitioner;
import org.apache.cassandra.db.FileStruct;
import org.apache.cassandra.utils.BloomFilter;
-import org.apache.commons.collections.CollectionUtils;
import org.testng.annotations.Test;
import java.io.File;
@@ -12,14 +11,13 @@
import java.util.*;
public class SSTableTest extends ServerTest {
- /*
@Test
public void testSingleWrite() throws IOException {
File f = File.createTempFile("sstable", "");
SSTable ssTable;
// write test data
- ssTable = new SSTable(f.getParent(), f.getName());
+ ssTable = new SSTable(f.getParent(), f.getName(), new
OrderPreservingPartitioner());
BloomFilter bf = new BloomFilter(1000, 8);
Random random = new Random();
byte[] bytes = new byte[1024];
@@ -27,20 +25,21 @@
String key = Integer.toString(1);
ssTable.append(key, bytes);
- bf.fill(key);
+ bf.add(key);
ssTable.close(bf);
- // TODO this is broken because SST/SequenceFile now assume that only
CFs are written
-
// verify
- ssTable = new SSTable(f.getPath() + "-Data.db");
- DataInputBuffer bufIn = ssTable.next(key, "Test:C");
- byte[] bytes2 = new byte[1024];
- bufIn.readFully(bytes2);
+ SSTable.indexMetadataMap_.clear();
+ ssTable = new SSTable(f.getPath() + "-Data.db", new
OrderPreservingPartitioner());
+ FileStruct fs = new
FileStruct(SequenceFile.bufferedReader(ssTable.dataFile_, 128 * 1024), new
OrderPreservingPartitioner());
+ fs.seekTo(key);
+ int size = fs.getBufIn().readInt();
+ byte[] bytes2 = new byte[size];
+ fs.getBufIn().readFully(bytes2);
assert Arrays.equals(bytes2, bytes);
}
- // @Test
+ @Test
public void testManyWrites() throws IOException {
File f = File.createTempFile("sstable", "");
SSTable ssTable;
@@ -52,7 +51,7 @@
}
// write
- ssTable = new SSTable(f.getParent(), f.getName());
+ ssTable = new SSTable(f.getParent(), f.getName(), new
OrderPreservingPartitioner());
BloomFilter bf = new BloomFilter(1000, 8);
for (String key: map.navigableKeySet())
{
@@ -60,19 +59,19 @@
}
ssTable.close(bf);
- // TODO this is broken because SST/SequenceFile now assume that only
CFs are written
-
// verify
+ SSTable.indexMetadataMap_.clear();
List<String> keys = new ArrayList(map.keySet());
Collections.shuffle(keys);
- ssTable = new SSTable(f.getPath() + "-Data.db");
- for (String key: keys)
+ ssTable = new SSTable(f.getPath() + "-Data.db", new
OrderPreservingPartitioner());
+ FileStruct fs = new
FileStruct(SequenceFile.bufferedReader(ssTable.dataFile_, 128 * 1024), new
OrderPreservingPartitioner());
+ for (String key : keys)
{
- DataInputBuffer bufIn = ssTable.next(key, "Test:C");
- byte[] bytes2 = new byte[map.get(key).length];
- bufIn.readFully(bytes2);
+ fs.seekTo(key);
+ int size = fs.getBufIn().readInt();
+ byte[] bytes2 = new byte[size];
+ fs.getBufIn().readFully(bytes2);
assert Arrays.equals(bytes2, map.get(key));
}
}
- */
}
