Author: gdusbabek
Date: Wed Apr 14 19:53:43 2010
New Revision: 934157
URL: http://svn.apache.org/viewvc?rev=934157&view=rev
Log:
use hex keys in sstable import/export. Patch by Stu Hood, reviewed by Gary
Dusbabek. CASSANDRA-767
Modified:
cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
cassandra/trunk/test/resources/SimpleCF.json
cassandra/trunk/test/resources/SuperCF.json
cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java?rev=934157&r1=934156&r2=934157&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java Wed
Apr 14 19:53:43 2010
@@ -38,7 +38,7 @@ import org.apache.cassandra.service.Stor
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.utils.FBUtilities.bytesToHex;
-import static org.apache.cassandra.utils.FBUtilities.UTF8;
+import static org.apache.cassandra.utils.FBUtilities.hexToBytes;
import org.apache.commons.cli.*;
/**
@@ -82,13 +82,6 @@ public class SSTableExport
return String.format("%s: ", quote(val));
}
- @Deprecated
- private static String asStr(byte[] val)
- {
- // FIXME: should not interpret as a string
- return new String(val, FBUtilities.UTF8);
- }
-
private static String serializeColumns(Collection<IColumn> cols,
AbstractType comp)
{
StringBuilder json = new StringBuilder("[");
@@ -119,7 +112,7 @@ public class SSTableExport
{
ColumnFamily cf = row.getColumnFamily();
AbstractType comparator = cf.getComparator();
- StringBuilder json = new StringBuilder(asKey(asStr(row.getKey().key)));
+ StringBuilder json = new
StringBuilder(asKey(bytesToHex(row.getKey().key)));
if (cf.isSuper())
{
@@ -167,7 +160,7 @@ public class SSTableExport
{
DecoratedKey decoratedKey =
partitioner.convertFromDiskFormat(FBUtilities.readShortByteArray(input));
long dataPosition = input.readLong();
- outs.println(asStr(decoratedKey.key));
+ outs.println(bytesToHex(decoratedKey.key));
}
outs.flush();
@@ -213,8 +206,7 @@ public class SSTableExport
{
if (excludeSet.contains(key))
continue;
- // FIXME: assuming string keys
- DecoratedKey<?> dk = partitioner.decorateKey(key.getBytes(UTF8));
+ DecoratedKey<?> dk = partitioner.decorateKey(hexToBytes(key));
scanner.seekTo(dk);
i++;
@@ -275,7 +267,7 @@ public class SSTableExport
while(scanner.hasNext())
{
SSTableIdentityIterator row = (SSTableIdentityIterator)
scanner.next();
- if (excludeSet.contains(asStr(row.getKey().key)))
+ if (excludeSet.contains(bytesToHex(row.getKey().key)))
continue;
try
{
@@ -288,12 +280,12 @@ public class SSTableExport
}
catch (IOException ioexcep)
{
- System.err.println("WARNING: Corrupt row " +
asStr(row.getKey().key) + " (skipping).");
+ System.err.println("WARNING: Corrupt row " +
bytesToHex(row.getKey().key) + " (skipping).");
continue;
}
catch (OutOfMemoryError oom)
{
- System.err.println("ERROR: Out of memory deserializing row " +
asStr(row.getKey().key));
+ System.err.println("ERROR: Out of memory deserializing row " +
bytesToHex(row.getKey().key));
continue;
}
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java?rev=934157&r1=934156&r2=934157&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java Wed
Apr 14 19:53:43 2010
@@ -31,7 +31,6 @@ import org.apache.cassandra.io.util.Data
import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.utils.FBUtilities.hexToBytes;
-import static org.apache.cassandra.utils.FBUtilities.UTF8;
import org.apache.commons.cli.*;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
@@ -77,13 +76,6 @@ public class SSTableImport
}
}
- @Deprecated
- private static String asStr(byte[] val)
- {
- // FIXME: should not interpret as a string
- return new String(val, FBUtilities.UTF8);
- }
-
/**
* Add columns to a column family.
*
@@ -151,22 +143,21 @@ public class SSTableImport
JSONObject json = (JSONObject)JSONValue.parseWithException(new
FileReader(jsonFile));
SSTableWriter writer = new SSTableWriter(ssTablePath, json.size(),
partitioner);
- List<DecoratedKey<?>> decoratedKeys = new
ArrayList<DecoratedKey<?>>();
+ SortedMap<DecoratedKey,String> decoratedKeys = new
TreeMap<DecoratedKey,String>();
+ // sort by dk representation, but hold onto the hex version
for (String key : (Set<String>)json.keySet())
- // FIXME: assuming string keys
- decoratedKeys.add(partitioner.decorateKey(key.getBytes(UTF8)));
- Collections.sort(decoratedKeys);
+ decoratedKeys.put(partitioner.decorateKey(hexToBytes(key)),
key);
- for (DecoratedKey<?> rowKey : decoratedKeys)
+ for (Map.Entry<DecoratedKey, String> rowKey :
decoratedKeys.entrySet())
{
if (cfType.equals("Super"))
- addToSuperCF((JSONObject)json.get(asStr(rowKey.key)),
cfamily);
+ addToSuperCF((JSONObject)json.get(rowKey.getValue()),
cfamily);
else
- addToStandardCF((JSONArray)json.get(asStr(rowKey.key)),
cfamily);
+ addToStandardCF((JSONArray)json.get(rowKey.getValue()),
cfamily);
ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
- writer.append(rowKey, dob);
+ writer.append(rowKey.getKey(), dob);
dob.reset();
cfamily.clear();
}
Modified: cassandra/trunk/test/resources/SimpleCF.json
URL:
http://svn.apache.org/viewvc/cassandra/trunk/test/resources/SimpleCF.json?rev=934157&r1=934156&r2=934157&view=diff
==============================================================================
--- cassandra/trunk/test/resources/SimpleCF.json (original)
+++ cassandra/trunk/test/resources/SimpleCF.json Wed Apr 14 19:53:43 2010
@@ -1,4 +1,4 @@
{
- "rowA": [["636f6c4141", "76616c4141", 1, false], ["636f6c4142", "76616c4142",
1, false]],
- "rowB": [["636f6c4241", "76616c4241", 1, false], ["636f6c4242", "76616c4242",
1, false]]
+ "726f7741": [["636f6c4141", "76616c4141", 1, false], ["636f6c4142",
"76616c4142", 1, false]],
+ "726f7742": [["636f6c4241", "76616c4241", 1, false], ["636f6c4242",
"76616c4242", 1, false]]
}
Modified: cassandra/trunk/test/resources/SuperCF.json
URL:
http://svn.apache.org/viewvc/cassandra/trunk/test/resources/SuperCF.json?rev=934157&r1=934156&r2=934157&view=diff
==============================================================================
--- cassandra/trunk/test/resources/SuperCF.json (original)
+++ cassandra/trunk/test/resources/SuperCF.json Wed Apr 14 19:53:43 2010
@@ -1,4 +1,4 @@
{
- "rowA": {"737570657241": {"deletedAt": -9223372036854775808, "subColumns":
[["636f6c4141", "76616c75654141", 1, false], ["636f6c4142", "76616c75654142",
1, false]]}},
- "rowB": {"737570657242": {"deletedAt": -9223372036854775808, "subColumns":
[["636f6c4241", "76616c75654241", 1, false], ["636f6c4242", "76616c75654242",
1, false]]}}
+ "726f7741": {"737570657241": {"deletedAt": -9223372036854775808,
"subColumns": [["636f6c4141", "76616c75654141", 1, false], ["636f6c4142",
"76616c75654142", 1, false]]}},
+ "726f7742": {"737570657242": {"deletedAt": -9223372036854775808,
"subColumns": [["636f6c4241", "76616c75654241", 1, false], ["636f6c4242",
"76616c75654242", 1, false]]}}
}
Modified:
cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java?rev=934157&r1=934156&r2=934157&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
(original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
Wed Apr 14 19:53:43 2010
@@ -35,6 +35,7 @@ import org.apache.cassandra.io.sstable.S
import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.io.util.DataOutputBuffer;
import static org.apache.cassandra.io.sstable.SSTableUtils.tempSSTableFile;
+import static org.apache.cassandra.utils.FBUtilities.bytesToHex;
import static org.apache.cassandra.utils.FBUtilities.hexToBytes;
import static org.junit.Assert.assertTrue;
@@ -48,6 +49,11 @@ import org.junit.Test;
public class SSTableExportTest extends SchemaLoader
{
+ public String asHex(String str)
+ {
+ return bytesToHex(str.getBytes());
+ }
+
@Test
public void testEnumeratekeys() throws IOException
{
@@ -84,8 +90,7 @@ public class SSTableExportTest extends S
String output = new String(buf);
String sep = System.getProperty("line.separator");
- // FIXME: string keys
- assert output.equals("rowA" + sep + "rowB" + sep) : output;
+ assert output.equals(asHex("rowA") + sep + asHex("rowB") + sep) :
output;
}
@Test
@@ -121,20 +126,19 @@ public class SSTableExportTest extends S
// Export to JSON and verify
File tempJson = File.createTempFile("Standard1", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new
String[]{"rowExclude"});
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new
String[]{asHex("rowExclude")});
JSONObject json = (JSONObject)JSONValue.parse(new
FileReader(tempJson));
- // FIXME: string keys
- JSONArray rowA = (JSONArray)json.get("rowA");
+ JSONArray rowA = (JSONArray)json.get(asHex("rowA"));
JSONArray colA = (JSONArray)rowA.get(0);
assert Arrays.equals(hexToBytes((String)colA.get(1)),
"valA".getBytes());
- JSONArray rowB = (JSONArray)json.get("rowB");
+ JSONArray rowB = (JSONArray)json.get(asHex("rowB"));
JSONArray colB = (JSONArray)rowB.get(0);
assert !(Boolean)colB.get(3);
- JSONArray rowExclude = (JSONArray)json.get("rowExclude");
+ JSONArray rowExclude = (JSONArray)json.get(asHex("rowExclude"));
assert rowExclude == null;
}
@@ -172,15 +176,15 @@ public class SSTableExportTest extends S
// Export to JSON and verify
File tempJson = File.createTempFile("Super4", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new
String[]{"rowExclude"});
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new
String[]{asHex("rowExclude")});
JSONObject json = (JSONObject)JSONValue.parse(new
FileReader(tempJson));
- JSONObject rowA = (JSONObject)json.get("rowA");
+ JSONObject rowA = (JSONObject)json.get(asHex("rowA"));
JSONObject superA =
(JSONObject)rowA.get(cfamily.getComparator().getString("superA".getBytes()));
JSONArray subColumns = (JSONArray)superA.get("subColumns");
JSONArray colA = (JSONArray)subColumns.get(0);
- JSONObject rowExclude = (JSONObject)json.get("rowExclude");
+ JSONObject rowExclude = (JSONObject)json.get(asHex("rowExclude"));
assert Arrays.equals(hexToBytes((String)colA.get(1)),
"valA".getBytes());
assert !(Boolean)colA.get(3);
assert rowExclude == null;
@@ -213,7 +217,7 @@ public class SSTableExportTest extends S
// Export to JSON and verify
File tempJson = File.createTempFile("Standard1", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new
String[]{"rowExclude"});
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new
String[]{asHex("rowExclude")});
// Import JSON to another SSTable file
File tempSS2 = tempSSTableFile("Keyspace1", "Standard1");
