Author: eevans
Date: Mon Jan 24 17:41:14 2011
New Revision: 1062896
URL: http://svn.apache.org/viewvc?rev=1062896&view=rev
Log:
generate more human-readable json
Patch by Pavel Yaskevich; reviewed by eevans for CASSANDRA-1933
Modified:
cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java?rev=1062896&r1=1062895&r2=1062896&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java Mon
Jan 24 17:41:14 2011
@@ -24,6 +24,7 @@ import java.io.PrintStream;
import java.nio.ByteBuffer;
import java.util.*;
+import org.apache.cassandra.config.CFMetaData;
import org.apache.commons.cli.*;
import org.apache.cassandra.config.ConfigurationException;
@@ -80,22 +81,28 @@ public class SSTableExport
return String.format("%s: ", quote(val));
}
- private static void serializeColumns(PrintStream outs, Collection<IColumn>
cols, AbstractType comp)
+ private static void serializeColumns(PrintStream outs, Collection<IColumn>
columns, AbstractType comparator, CFMetaData cfMetaData)
{
outs.print("[");
- Iterator<IColumn> iter = cols.iterator();
+ Iterator<IColumn> iter = columns.iterator();
+
while (iter.hasNext())
{
outs.print("[");
IColumn column = iter.next();
- outs.print(quote(bytesToHex(column.name())));
+
+ ByteBuffer name = column.name();
+ AbstractType validator = cfMetaData.getValueValidator(name);
+
+ outs.print(quote(comparator.getString(name)));
outs.print(", ");
- outs.print(quote(bytesToHex(column.value())));
+ outs.print(quote(validator.getString(column.value())));
outs.print(", ");
outs.print(column.timestamp());
outs.print(", ");
outs.print(column.isMarkedForDelete());
+
if (column instanceof ExpiringColumn)
{
outs.print(", ");
@@ -103,9 +110,13 @@ public class SSTableExport
outs.print(", ");
outs.print(column.getLocalDeletionTime());
}
+
outs.print("]");
+
if (iter.hasNext())
+ {
outs.print(", ");
+ }
}
outs.print("]");
@@ -113,35 +124,47 @@ public class SSTableExport
private static void serializeRow(PrintStream outs, SSTableIdentityIterator
row) throws IOException
{
- ColumnFamily cf = row.getColumnFamilyWithColumns();
- AbstractType comparator = cf.getComparator();
- outs.print(asKey(bytesToHex(row.getKey().key)));
+ ColumnFamily columnFamily = row.getColumnFamilyWithColumns();
+ CFMetaData cfMetaData = columnFamily.metadata();
+
+ AbstractType comparator = columnFamily.getComparator();
- if (cf.isSuper())
+ // key is represented as String according to current Partitioner
+ outs.print(" " + asKey(bytesToHex(row.getKey().key)));
+
+ if (columnFamily.isSuper())
{
outs.print("{ ");
- Iterator<IColumn> iter = cf.getSortedColumns().iterator();
+ Iterator<IColumn> iter =
columnFamily.getSortedColumns().iterator();
while (iter.hasNext())
{
IColumn column = iter.next();
- outs.print(asKey(bytesToHex(column.name())));
+
+ // header of the row
+ outs.print(asKey(comparator.getString(column.name())));
outs.print("{");
outs.print(asKey("deletedAt"));
outs.print(column.getMarkedForDeleteAt());
outs.print(", ");
outs.print(asKey("subColumns"));
- serializeColumns(outs, column.getSubColumns(), comparator);
+
+ // columns
+ serializeColumns(outs, column.getSubColumns(),
columnFamily.getSubComparator(), cfMetaData);
+
outs.print("}");
+
if (iter.hasNext())
+ {
outs.print(", ");
+ }
}
outs.print("}");
}
else
{
- serializeColumns(outs, cf.getSortedColumns(), comparator);
+ serializeColumns(outs, columnFamily.getSortedColumns(),
comparator, cfMetaData);
}
}
@@ -179,10 +202,11 @@ public class SSTableExport
* @param ssTableFile the SSTable to export the rows from
* @param outs PrintStream to write the output to
* @param keys the keys corresponding to the rows to export
+ * @param excludes the keys to exclude from export
+ *
* @throws IOException on failure to read/write input/output
*/
- public static void export(String ssTableFile, PrintStream outs, String[]
keys, String[] excludes)
- throws IOException
+ public static void export(String ssTableFile, PrintStream outs, String[]
keys, String[] excludes) throws IOException
{
SSTableReader reader =
SSTableReader.open(Descriptor.fromFilename(ssTableFile));
SSTableScanner scanner =
reader.getDirectScanner(INPUT_FILE_BUFFER_SIZE);
@@ -202,11 +226,13 @@ public class SSTableExport
{
if (excludeSet.contains(key))
continue;
+
DecoratedKey<?> dk = partitioner.decorateKey(hexToBytes(key));
// validate order of the keys in the sstable
if (lastKey != null && lastKey.compareTo(dk) > 0 )
throw new IOException("Key out of order! " + lastKey + " > " +
dk);
+
lastKey = dk;
scanner.seekTo(dk);
@@ -225,12 +251,10 @@ public class SSTableExport
catch (IOException ioexc)
{
System.err.println("WARNING: Corrupt row " + key + "
(skipping).");
- continue;
}
catch (OutOfMemoryError oom)
{
System.err.println("ERROR: Out of memory deserializing row
" + key);
- continue;
}
}
}
@@ -274,12 +298,10 @@ public class SSTableExport
catch (IOException ioexcep)
{
System.err.println("WARNING: Corrupt row " +
bytesToHex(row.getKey().key) + " (skipping).");
- continue;
}
catch (OutOfMemoryError oom)
{
System.err.println("ERROR: Out of memory deserializing row " +
bytesToHex(row.getKey().key));
- continue;
}
}
@@ -292,6 +314,8 @@ public class SSTableExport
*
* @param ssTableFile the SSTable to export
* @param outs PrintStream to write the output to
+ * @param excludes the keys to exclude from export
+ *
* @throws IOException on failure to read/write input/output
*/
public static void export(String ssTableFile, PrintStream outs, String[]
excludes) throws IOException
@@ -304,6 +328,8 @@ public class SSTableExport
* Export an SSTable and write the resulting JSON to standard out.
*
* @param ssTableFile SSTable to export
+ * @param excludes the keys to exclude from export
+ *
* @throws IOException on failure to read/write SSTable/standard out
*/
public static void export(String ssTableFile, String[] excludes) throws
IOException
@@ -317,6 +343,7 @@ public class SSTableExport
*
* @param args command lines arguments
* @throws IOException on failure to open/read/write files or output
streams
+ * @throws ConfigurationException if configuration is invalid
*/
public static void main(String[] args) throws IOException,
ConfigurationException
{
Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java?rev=1062896&r1=1062895&r2=1062896&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java Mon
Jan 24 17:41:14 2011
@@ -23,6 +23,8 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.BytesType;
import org.apache.commons.cli.*;
import org.apache.cassandra.config.CFMetaData;
@@ -74,7 +76,7 @@ public class SSTableImport
options.addOption(new Option(KEY_COUNT_OPTION, true, "Number of keys
to import (Optional)."));
options.addOption(new Option(IS_SORTED_OPTION, false, "Assume JSON
file as already sorted (e.g. created by sstable2json tool) (Optional)."));
}
-
+
private static class JsonColumn<T>
{
private ByteBuffer name;
@@ -84,19 +86,23 @@ public class SSTableImport
private int ttl;
private int localExpirationTime;
- public JsonColumn(T json)
+ public JsonColumn(T json, CFMetaData meta, boolean isSubColumn)
{
+ AbstractType comparator = (isSubColumn) ? meta.subcolumnComparator
: meta.comparator;
+
if (json instanceof List)
{
List fields = (List<?>) json;
assert fields.size() == 4 || fields.size() == 6 : "Column
definition should have 4 or 6 fields.";
- name = hexToBytes((String) fields.get(0));
- value = hexToBytes((String) fields.get(1));
+ name = stringAsType((String) fields.get(0), comparator);
+ value = stringAsType((String) fields.get(1),
meta.getValueValidator(name.duplicate()));
+
timestamp = (Long) fields.get(2);
isDeleted = (Boolean) fields.get(3);
+
if (fields.size() == 6)
{
ttl = (Integer) fields.get(4);
@@ -135,7 +141,7 @@ public class SSTableImport
for (Object c : row)
{
- JsonColumn col = new JsonColumn<List>((List) c);
+ JsonColumn col = new JsonColumn<List>((List) c, cfm, (superName !=
null));
QueryPath path = new QueryPath(cfm.cfName, superName,
col.getName());
if (col.ttl > 0)
@@ -164,13 +170,14 @@ public class SSTableImport
CFMetaData metaData = cfamily.metadata();
assert metaData != null;
+ AbstractType comparator = metaData.comparator;
+
// Super columns
for (Map.Entry<?, ?> entry : row.entrySet())
{
- ByteBuffer superName = hexToBytes((String) entry.getKey());
Map<?, ?> data = (Map<?, ?>) entry.getValue();
- addColumnsToCF((List<?>) data.get("subColumns"), superName,
cfamily);
+ addColumnsToCF((List<?>) data.get("subColumns"),
stringAsType((String) entry.getKey(), comparator), cfamily);
// *WARNING* markForDeleteAt has been DEPRECATED at Cassandra side
//BigInteger deletedAt = (BigInteger) data.get("deletedAt");
@@ -452,4 +459,15 @@ public class SSTableImport
keyCountToImport = keyCount;
}
+ /**
+ * Convert a string to bytes (ByteBuffer) according to type
+ * @param content string to convert
+ * @param type type to use for conversion
+ * @return byte buffer representation of the given string
+ */
+ private static ByteBuffer stringAsType(String content, AbstractType type)
+ {
+ return (type == BytesType.instance) ? hexToBytes(content) :
type.fromString(content);
+ }
+
}
Modified:
cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableImportTest.java?rev=1062896&r1=1062895&r2=1062896&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
(original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
Mon Jan 24 17:41:14 2011
@@ -20,7 +20,6 @@ package org.apache.cassandra.tools;
import java.io.File;
import java.io.IOException;
-import java.nio.ByteBuffer;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -34,13 +33,17 @@ import org.apache.cassandra.db.columnite
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableReader;
+
import org.apache.cassandra.utils.ByteBufferUtil;
+
import static org.apache.cassandra.utils.ByteBufferUtil.hexToBytes;
+
import static org.apache.cassandra.io.sstable.SSTableUtils.tempSSTableFile;
import static org.junit.Assert.assertEquals;
import org.apache.cassandra.Util;
+import org.apache.cassandra.utils.FBUtilities;
import org.json.simple.parser.ParseException;
import org.junit.Test;
@@ -82,7 +85,7 @@ public class SSTableImportTest extends S
IColumn superCol = cf.getColumn(ByteBufferUtil.bytes("superA"));
assert superCol != null;
assert superCol.getSubColumns().size() > 0;
- IColumn subColumn =
superCol.getSubColumn(ByteBufferUtil.bytes("colAA"));
+ IColumn subColumn =
superCol.getSubColumn(ByteBufferUtil.bytes("636f6c4141"));
assert subColumn.value().equals(hexToBytes("76616c75654141"));
}
