Author: gdusbabek
Date: Wed Apr 14 19:52:28 2010
New Revision: 934152
URL: http://svn.apache.org/viewvc?rev=934152&view=rev
Log:
convert IPartitioner disk key format to bytes. Patch by Stu Hood, reviewed by
Gary Dusbabek. CASSANDRA-767
Removed:
cassandra/trunk/src/java/org/apache/cassandra/io/IteratingRow.java
Modified:
cassandra/trunk/src/java/org/apache/cassandra/db/DBConstants.java
cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java
cassandra/trunk/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java
cassandra/trunk/src/java/org/apache/cassandra/dht/IPartitioner.java
cassandra/trunk/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java
cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedReader.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedScanner.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java
cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/DBConstants.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/DBConstants.java?rev=934152&r1=934151&r2=934152&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/DBConstants.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/DBConstants.java Wed Apr
14 19:52:28 2010
@@ -18,7 +18,7 @@
*/
package org.apache.cassandra.db;
-class DBConstants
+public class DBConstants
{
public static final int boolSize_ = 1;
public static final int intSize_ = 4;
Modified:
cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java?rev=934152&r1=934151&r2=934152&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java
Wed Apr 14 19:52:28 2010
@@ -34,6 +34,8 @@ import org.apache.cassandra.io.sstable.S
import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.utils.BloomFilter;
+import org.apache.cassandra.utils.FBUtilities;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,7 +67,7 @@ public class SSTableNamesIterator extend
file = ssTable.getFileDataInput(decoratedKey,
DatabaseDescriptor.getIndexedReadBufferSizeInKB() * 1024);
if (file == null)
return;
- DecoratedKey keyInDisk =
ssTable.getPartitioner().convertFromDiskFormat(file.readUTF());
+ DecoratedKey keyInDisk =
ssTable.getPartitioner().convertFromDiskFormat(FBUtilities.readShortByteArray(file));
assert keyInDisk.equals(decoratedKey)
: String.format("%s != %s in %s", keyInDisk,
decoratedKey, file.getPath());
file.readInt(); // data size
Modified:
cassandra/trunk/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java?rev=934152&r1=934151&r2=934152&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java
Wed Apr 14 19:52:28 2010
@@ -44,15 +44,15 @@ public class CollatingOrderPreservingPar
return new DecoratedKey<BytesToken>(getToken(key), key);
}
- public DecoratedKey<BytesToken> convertFromDiskFormat(String key)
+ public DecoratedKey<BytesToken> convertFromDiskFormat(byte[] fromdisk)
{
- return new DecoratedKey<BytesToken>(getToken(key), key);
+ String key = new String(fromdisk, FBUtilities.UTF8);
+ return new DecoratedKey<BytesToken>(getToken(key), fromdisk);
}
- public String convertToDiskFormat(DecoratedKey<BytesToken> key)
+ public byte[] convertToDiskFormat(DecoratedKey<BytesToken> key)
{
- // FIXME
- return new String(key.key, FBUtilities.UTF8);
+ return key.key;
}
public BytesToken midpoint(BytesToken ltoken, BytesToken rtoken)
Modified: cassandra/trunk/src/java/org/apache/cassandra/dht/IPartitioner.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/dht/IPartitioner.java?rev=934152&r1=934151&r2=934152&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/dht/IPartitioner.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/dht/IPartitioner.java Wed Apr
14 19:52:28 2010
@@ -24,13 +24,12 @@ import org.apache.cassandra.db.Decorated
public interface IPartitioner<T extends Token>
{
-
/**
* Convert the on disk representation to a DecoratedKey object
* @param key On disk representation
* @return DecoratedKey object
*/
- public DecoratedKey<T> convertFromDiskFormat(String key);
+ public DecoratedKey<T> convertFromDiskFormat(byte[] key);
/**
* Convert the DecoratedKey to the on disk format used for
@@ -38,7 +37,7 @@ public interface IPartitioner<T extends
* @param key The DecoratedKey in question
* @return
*/
- public String convertToDiskFormat(DecoratedKey<T> key);
+ public byte[] convertToDiskFormat(DecoratedKey<T> key);
/**
* Transform key to object representation of the on-disk format.
@@ -46,6 +45,7 @@ public interface IPartitioner<T extends
* @param key the raw, client-facing key
* @return decorated version of key
*/
+ @Deprecated
public DecoratedKey<T> decorateKey(String key);
/**
@@ -66,6 +66,7 @@ public interface IPartitioner<T extends
* (This is NOT a method to create a Token from its string representation;
* for that, use TokenFactory.fromString.)
*/
+ @Deprecated
public T getToken(String key);
/**
Modified:
cassandra/trunk/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java?rev=934152&r1=934151&r2=934152&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java
Wed Apr 14 19:52:28 2010
@@ -40,15 +40,15 @@ public class OrderPreservingPartitioner
return new DecoratedKey<StringToken>(new StringToken(key), key);
}
- public DecoratedKey<StringToken> convertFromDiskFormat(String key)
+ public DecoratedKey<StringToken> convertFromDiskFormat(byte[] fromdisk)
{
- return new DecoratedKey<StringToken>(new StringToken(key), key);
+ String key = new String(fromdisk, FBUtilities.UTF8);
+ return new DecoratedKey<StringToken>(new StringToken(key), fromdisk);
}
- public String convertToDiskFormat(DecoratedKey<StringToken> key)
+ public byte[] convertToDiskFormat(DecoratedKey<StringToken> key)
{
- // FIXME
- return new String(key.key, FBUtilities.UTF8);
+ return key.key;
}
public StringToken midpoint(StringToken ltoken, StringToken rtoken)
Modified:
cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java?rev=934152&r1=934151&r2=934152&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java
Wed Apr 14 19:52:28 2010
@@ -19,10 +19,11 @@
package org.apache.cassandra.dht;
import java.math.BigInteger;
+import java.util.Arrays;
import java.util.Comparator;
-import java.util.regex.Pattern;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.DBConstants;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.GuidGenerator;
@@ -37,26 +38,46 @@ public class RandomPartitioner implement
public static final BigIntegerToken MINIMUM = new BigIntegerToken("0");
- private static final String DELIMITER = ":";
+ private static final byte DELIMITER_BYTE = ":".getBytes()[0];
public DecoratedKey<BigIntegerToken> decorateKey(String key)
{
return new DecoratedKey<BigIntegerToken>(getToken(key), key);
}
- public DecoratedKey<BigIntegerToken> convertFromDiskFormat(String key)
+ public DecoratedKey<BigIntegerToken> convertFromDiskFormat(byte[] fromdisk)
{
- int splitPoint = key.indexOf(DELIMITER);
- String first = key.substring(0, splitPoint);
- String second = key.substring(splitPoint+1);
-
- return new DecoratedKey<BigIntegerToken>(new BigIntegerToken(first),
second);
- }
+ // find the delimiter position
+ int splitPoint = -1;
+ for (int i = 0; i < fromdisk.length; i++)
+ {
+ if (fromdisk[i] == DELIMITER_BYTE)
+ {
+ splitPoint = i;
+ break;
+ }
+ }
+ assert splitPoint != -1;
- public String convertToDiskFormat(DecoratedKey<BigIntegerToken> key)
- {
- // FIXME
- return key.token + DELIMITER + new String(key.key, FBUtilities.UTF8);
+ // and decode the token and key
+ String token = new String(fromdisk, 0, splitPoint, FBUtilities.UTF8);
+ byte[] key = Arrays.copyOfRange(fromdisk, splitPoint + 1,
fromdisk.length);
+ return new DecoratedKey<BigIntegerToken>(new BigIntegerToken(token),
key);
+ }
+
+ public byte[] convertToDiskFormat(DecoratedKey<BigIntegerToken> key)
+ {
+ // encode token prefix and calculate final length (with delimiter)
+ byte[] prefix = key.token.toString().getBytes(FBUtilities.UTF8);
+ int length = prefix.length + 1 + key.key.length;
+ assert length <= FBUtilities.MAX_UNSIGNED_SHORT;
+
+ // copy into output bytes
+ byte[] todisk = new byte[length];
+ System.arraycopy(prefix, 0, todisk, 0, prefix.length);
+ todisk[prefix.length] = DELIMITER_BYTE;
+ System.arraycopy(key.key, 0, todisk, prefix.length + 1,
key.key.length);
+ return todisk;
}
public BigIntegerToken midpoint(BigIntegerToken ltoken, BigIntegerToken
rtoken)
Modified:
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedReader.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedReader.java?rev=934152&r1=934151&r2=934152&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedReader.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedReader.java
Wed Apr 14 19:52:28 2010
@@ -173,7 +173,7 @@ class RowIndexedReader extends SSTableRe
{
break;
}
- DecoratedKey decoratedKey =
partitioner.convertFromDiskFormat(input.readUTF());
+ DecoratedKey decoratedKey =
partitioner.convertFromDiskFormat(FBUtilities.readShortByteArray(input));
long dataPosition = input.readLong();
long nextIndexPosition = input.getFilePointer();
// read the next index entry to see how big the row is
@@ -184,7 +184,7 @@ class RowIndexedReader extends SSTableRe
}
else
{
- input.readUTF();
+ FBUtilities.readShortByteArray(input);
nextDataPosition = input.readLong();
input.seek(nextIndexPosition);
}
@@ -231,7 +231,8 @@ class RowIndexedReader extends SSTableRe
public PositionSize getPosition(DecoratedKey decoratedKey)
{
// first, check bloom filter
- if (!bf.isPresent(partitioner.convertToDiskFormat(decoratedKey)))
+ // FIXME: expecting utf8
+ if (!bf.isPresent(new
String(partitioner.convertToDiskFormat(decoratedKey), FBUtilities.UTF8)))
return null;
// next, the key cache
@@ -294,9 +295,10 @@ class RowIndexedReader extends SSTableRe
}
else
{
+ int keylength =
StorageService.getPartitioner().convertToDiskFormat(kp.key).length;
long nextUnspannedPostion = input.getAbsolutePosition()
- + 2 +
FBUtilities.encodedUTF8Length(StorageService.getPartitioner().convertToDiskFormat(kp.key))
- + 8;
+ + DBConstants.shortSize_ +
keylength
+ + DBConstants.longSize_;
input = indexInputAt(nextUnspannedPostion);
}
oldInput.close();
@@ -307,7 +309,7 @@ class RowIndexedReader extends SSTableRe
}
// read key & data position from index entry
- DecoratedKey indexDecoratedKey =
partitioner.convertFromDiskFormat(input.readUTF());
+ DecoratedKey indexDecoratedKey =
partitioner.convertFromDiskFormat(FBUtilities.readShortByteArray(input));
long dataPosition = input.readLong();
int v = indexDecoratedKey.compareTo(decoratedKey);
@@ -388,7 +390,7 @@ class RowIndexedReader extends SSTableRe
DecoratedKey indexDecoratedKey;
try
{
- indexDecoratedKey =
partitioner.convertFromDiskFormat(input.readUTF());
+ indexDecoratedKey =
partitioner.convertFromDiskFormat(FBUtilities.readShortByteArray(input));
}
catch (EOFException e)
{
Modified:
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedScanner.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedScanner.java?rev=934152&r1=934151&r2=934152&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedScanner.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedScanner.java
Wed Apr 14 19:52:28 2010
@@ -29,6 +29,7 @@ import org.apache.cassandra.db.filter.IC
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.io.util.BufferedRandomAccessFile;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -166,7 +167,7 @@ public class RowIndexedScanner extends S
file.seek(finishedAt);
assert !file.isEOF();
- DecoratedKey key =
StorageService.getPartitioner().convertFromDiskFormat(file.readUTF());
+ DecoratedKey key =
StorageService.getPartitioner().convertFromDiskFormat(FBUtilities.readShortByteArray(file));
int dataSize = file.readInt();
dataStart = file.getFilePointer();
finishedAt = dataStart + dataSize;
Modified:
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=934152&r1=934151&r2=934152&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
Wed Apr 14 19:52:28 2010
@@ -74,11 +74,12 @@ public class SSTableWriter extends SSTab
private void afterAppend(DecoratedKey decoratedKey, long dataPosition, int
dataSize) throws IOException
{
- String diskKey = partitioner.convertToDiskFormat(decoratedKey);
- bf.add(diskKey);
+ byte[] diskKey = partitioner.convertToDiskFormat(decoratedKey);
+ // FIXME: needs format change
+ bf.add(new String(diskKey, FBUtilities.UTF8));
lastWrittenKey = decoratedKey;
long indexPosition = indexFile.getFilePointer();
- indexFile.writeUTF(diskKey);
+ FBUtilities.writeShortByteArray(diskKey, indexFile);
indexFile.writeLong(dataPosition);
if (logger.isTraceEnabled())
logger.trace("wrote " + decoratedKey + " at " + dataPosition);
@@ -92,7 +93,7 @@ public class SSTableWriter extends SSTab
public void append(DecoratedKey decoratedKey, DataOutputBuffer buffer)
throws IOException
{
long currentPosition = beforeAppend(decoratedKey);
- dataFile.writeUTF(partitioner.convertToDiskFormat(decoratedKey));
+
FBUtilities.writeShortByteArray(partitioner.convertToDiskFormat(decoratedKey),
dataFile);
int length = buffer.getLength();
assert length > 0;
dataFile.writeInt(length);
@@ -103,7 +104,7 @@ public class SSTableWriter extends SSTab
public void append(DecoratedKey decoratedKey, byte[] value) throws
IOException
{
long currentPosition = beforeAppend(decoratedKey);
- dataFile.writeUTF(partitioner.convertToDiskFormat(decoratedKey));
+
FBUtilities.writeShortByteArray(partitioner.convertToDiskFormat(decoratedKey),
dataFile);
assert value.length > 0;
dataFile.writeInt(value.length);
dataFile.write(value);
Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java?rev=934152&r1=934151&r2=934152&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java Wed
Apr 14 19:52:28 2010
@@ -164,7 +164,7 @@ public class SSTableExport
BufferedRandomAccessFile input = new
BufferedRandomAccessFile(SSTable.indexFilename(ssTableFile), "r");
while (!input.isEOF())
{
- DecoratedKey decoratedKey =
partitioner.convertFromDiskFormat(input.readUTF());
+ DecoratedKey decoratedKey =
partitioner.convertFromDiskFormat(FBUtilities.readShortByteArray(input));
long dataPosition = input.readLong();
outs.println(asStr(decoratedKey.key));
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java?rev=934152&r1=934151&r2=934152&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java Wed
Apr 14 19:52:28 2010
@@ -362,34 +362,6 @@ public class FBUtilities
return sb.toString();
}
- public static String mapToString(Map<?,?> map)
- {
- StringBuilder sb = new StringBuilder("{");
-
- for (Map.Entry entry : map.entrySet())
- {
- sb.append(entry.getKey()).append(":
").append(entry.getValue()).append(", ");
- }
-
- return sb.append("}").toString();
- }
-
- public static void writeNullableString(String key, DataOutput dos) throws
IOException
- {
- dos.writeBoolean(key == null);
- if (key != null)
- {
- dos.writeUTF(key);
- }
- }
-
- public static String readNullableString(DataInput dis) throws IOException
- {
- if (dis.readBoolean())
- return null;
- return dis.readUTF();
- }
-
public static void renameWithConfirm(String tmpFilename, String filename)
throws IOException
{
if (!new File(tmpFilename).renameTo(new File(filename)))
Modified:
cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java?rev=934152&r1=934151&r2=934152&view=diff
==============================================================================
---
cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
(original)
+++
cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
Wed Apr 14 19:52:28 2010
@@ -13,6 +13,7 @@ import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
public class SSTableReaderTest extends CleanupHelper
@@ -44,7 +45,7 @@ public class SSTableReaderTest extends C
String key = String.valueOf(j);
DecoratedKey dk = StorageService.getPartitioner().decorateKey(key);
FileDataInput file = sstable.getFileDataInput(dk,
DatabaseDescriptor.getIndexedReadBufferSizeInKB() * 1024);
- DecoratedKey keyInDisk =
sstable.getPartitioner().convertFromDiskFormat(file.readUTF());
+ DecoratedKey keyInDisk =
sstable.getPartitioner().convertFromDiskFormat(FBUtilities.readShortByteArray(file));
assert keyInDisk.equals(dk) : String.format("%s != %s in %s",
keyInDisk, dk, file.getPath());
}
