Author: eevans
Date: Mon Nov 30 03:25:01 2009
New Revision: 885324
URL: http://svn.apache.org/viewvc?rev=885324&view=rev
Log:
export/import tombstone and timestamp information
Modified:
incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
Modified:
incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java?rev=885324&r1=885323&r2=885324&view=diff
==============================================================================
---
incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
(original)
+++
incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
Mon Nov 30 03:25:01 2009
@@ -22,7 +22,6 @@
import java.io.PrintStream;
import java.util.Collection;
import java.util.Iterator;
-
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamily;
import org.apache.cassandra.db.DecoratedKey;
@@ -33,12 +32,7 @@
import org.apache.cassandra.io.SSTableReader;
import org.apache.cassandra.io.SSTableScanner;
import org.apache.cassandra.utils.FBUtilities;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.PosixParser;
+import org.apache.commons.cli.*;
/**
* Export SSTables to JSON format.
@@ -76,19 +70,26 @@
private static String serializeColumns(Collection<IColumn> cols,
AbstractType comp)
{
- StringBuilder json = new StringBuilder("{");
+ StringBuilder json = new StringBuilder("[");
Iterator<IColumn> iter = cols.iterator();
while (iter.hasNext())
{
+ json.append("[");
IColumn column = iter.next();
- json.append(asKey(comp.getString(column.name())));
+ json.append(quote(comp.getString(column.name())));
+ json.append(", ");
json.append(quote(FBUtilities.bytesToHex(column.value())));
+ json.append(", ");
+ json.append(column.timestamp());
+ json.append(", ");
+ json.append(column.isMarkedForDelete());
+ json.append("]");
if (iter.hasNext())
json.append(", ");
}
- json.append(" }");
+ json.append("]");
return json.toString();
}
@@ -108,12 +109,18 @@
{
IColumn column = iter.next();
json.append(asKey(comparator.getString(column.name())));
+ json.append("{");
+ json.append(asKey("deletedAt"));
+ json.append(column.getMarkedForDeleteAt());
+ json.append(", ");
+ json.append(asKey("subColumns"));
json.append(serializeColumns(column.getSubColumns(),
comparator));
+ json.append("}");
if (iter.hasNext())
json.append(", ");
}
- json.append(" }");
+ json.append("}");
}
else
{
@@ -189,16 +196,10 @@
export(ssTableFile, outs, keys);
}
- /**
- * Export an SSTable and write the resulting JSON to a PrintStream.
- *
- * @param ssTableFile the SSTable to export
- * @param outs PrintStream to write the output to
- * @throws IOException on failure to read/write input/output
- */
- public static void export(String ssTableFile, PrintStream outs) throws
IOException
+ // This is necessary to accommodate the test suite since you cannot open a
Reader more
+ // than once from within the same process.
+ static void export(SSTableReader reader, PrintStream outs) throws
IOException
{
- SSTableReader reader = SSTableReader.open(ssTableFile);
SSTableScanner scanner = reader.getScanner();
outs.println("{");
@@ -232,6 +233,19 @@
}
/**
+ * Export an SSTable and write the resulting JSON to a PrintStream.
+ *
+ * @param ssTableFile the SSTable to export
+ * @param outs PrintStream to write the output to
+ * @throws IOException on failure to read/write input/output
+ */
+ public static void export(String ssTableFile, PrintStream outs) throws
IOException
+ {
+ SSTableReader reader = SSTableReader.open(ssTableFile);
+ export(reader, outs);
+ }
+
+ /**
* Export an SSTable and write the resulting JSON to a file.
*
* @param ssTableFile SSTable to export
Modified:
incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java?rev=885324&r1=885323&r2=885324&view=diff
==============================================================================
---
incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
(original)
+++
incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
Mon Nov 30 03:25:01 2009
@@ -24,16 +24,14 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamily;
import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.SuperColumn;
import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.DataOutputBuffer;
import org.apache.cassandra.io.SSTableWriter;
import org.apache.cassandra.utils.FBUtilities;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.PosixParser;
+import org.apache.commons.cli.*;
+import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.json.simple.parser.ParseException;
@@ -58,6 +56,24 @@
optColfamily.setRequired(true);
options.addOption(optColfamily);
}
+
+ private static class JsonColumn
+ {
+ private String name;
+ private String value;
+ private long timestamp;
+ private boolean isDeleted;
+
+ private JsonColumn(Object obj) throws ClassCastException
+ {
+ JSONArray colSpec = (JSONArray)obj;
+ assert colSpec.size() == 4;
+ name = (String)colSpec.get(0);
+ value = (String)colSpec.get(1);
+ timestamp = (Long)colSpec.get(2);
+ isDeleted = (Boolean)colSpec.get(3);
+ }
+ }
/**
* Add columns to a column family.
@@ -65,13 +81,13 @@
* @param row the columns associated with a row
* @param cfamily the column family to add columns to
*/
- private static void addToStandardCF(JSONObject row, ColumnFamily cfamily)
+ private static void addToStandardCF(JSONArray row, ColumnFamily cfamily)
{
- for (Map.Entry<String, String> col : (Set<Map.Entry<String, String>>)
row.entrySet())
+ for (Object c : row)
{
- QueryPath path = new QueryPath(cfamily.name(), null,
col.getKey().getBytes());
- byte[] value = FBUtilities.hexToBytes(col.getValue());
- cfamily.addColumn(path, value, System.currentTimeMillis());
+ JsonColumn col = new JsonColumn(c);
+ QueryPath path = new QueryPath(cfamily.name(), null,
col.name.getBytes());
+ cfamily.addColumn(path, FBUtilities.hexToBytes(col.value),
col.timestamp, col.isDeleted);
}
}
@@ -87,14 +103,19 @@
for (Map.Entry<String, JSONObject> entry : (Set<Map.Entry<String,
JSONObject>>)row.entrySet())
{
byte[] superName = entry.getKey().getBytes();
+ long deletedAt = (Long)entry.getValue().get("deletedAt");
+ JSONArray subColumns =
(JSONArray)entry.getValue().get("subColumns");
- // Sub-columns
- for (Map.Entry<String, String> col : (Set<Map.Entry<String,
String>>)entry.getValue().entrySet())
+ // Add sub-columns
+ for (Object c : subColumns)
{
- QueryPath path = new QueryPath(cfamily.name(), superName,
col.getKey().getBytes());
- byte[] value = FBUtilities.hexToBytes(col.getValue());
- cfamily.addColumn(path, value, System.currentTimeMillis());
+ JsonColumn col = new JsonColumn(c);
+ QueryPath path = new QueryPath(cfamily.name(), superName,
col.name.getBytes());
+ cfamily.addColumn(path, FBUtilities.hexToBytes(col.value),
col.timestamp, col.isDeleted);
}
+
+ SuperColumn superColumn =
(SuperColumn)cfamily.getColumn(superName);
+ superColumn.markForDeleteAt((int)System.currentTimeMillis(),
deletedAt);
}
}
@@ -108,7 +129,7 @@
* @throws IOException for errors reading/writing input/output
* @throws ParseException for errors encountered parsing JSON input
*/
- private static void importJson(String jsonFile, String keyspace, String
cf, String ssTablePath)
+ public static void importJson(String jsonFile, String keyspace, String cf,
String ssTablePath)
throws IOException, ParseException
{
ColumnFamily cfamily = ColumnFamily.create(keyspace, cf);
@@ -119,6 +140,11 @@
try
{
JSONObject json = (JSONObject)JSONValue.parse(new
FileReader(jsonFile));
+
+ // FIXME: see
http://code.google.com/p/json-simple/issues/detail?id=13
+ if (json == null)
+ throw new RuntimeException("Error parsing JSON input!");
+
SSTableWriter writer = new SSTableWriter(ssTablePath, json.size(),
partitioner);
List<DecoratedKey<?>> decoratedKeys = new
ArrayList<DecoratedKey<?>>();
@@ -128,12 +154,10 @@
for (DecoratedKey<?> rowKey : decoratedKeys)
{
- JSONObject value = (JSONObject)json.get(rowKey.key);
-
if (cfType.equals("Super"))
- addToSuperCF(value, cfamily);
+ addToSuperCF((JSONObject)json.get(rowKey.key), cfamily);
else
- addToStandardCF(value, cfamily);
+ addToStandardCF((JSONArray)json.get(rowKey.key), cfamily);
ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
writer.append(rowKey, dob);
@@ -145,8 +169,7 @@
}
catch (ClassCastException cce)
{
- //throw cce;
- throw new RuntimeException("Invalid JSON input, or incorrect
column family");
+ throw new RuntimeException("Invalid JSON input, or incorrect
column family.", cce);
}
}
