Author: eevans
Date: Mon Nov 30 03:24:46 2009
New Revision: 885322
URL: http://svn.apache.org/viewvc?rev=885322&view=rev
Log:
CASSANDRA-499 JSON to SSTable converter (importer)
Added:
incubator/cassandra/trunk/bin/json2sstable (with props)
incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
(with props)
Added: incubator/cassandra/trunk/bin/json2sstable
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/bin/json2sstable?rev=885322&view=auto
==============================================================================
--- incubator/cassandra/trunk/bin/json2sstable (added)
+++ incubator/cassandra/trunk/bin/json2sstable Mon Nov 30 03:24:46 2009
@@ -0,0 +1,42 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
+ for include in /usr/share/cassandra/cassandra.in.sh \
+ /usr/local/share/cassandra/cassandra.in.sh \
+ /opt/cassandra/cassandra.in.sh \
+ ~/.cassandra.in.sh \
+ `dirname $0`/cassandra.in.sh; do
+ if [ -r $include ]; then
+ . $include
+ break
+ fi
+ done
+elif [ -r $CASSANDRA_INCLUDE ]; then
+ . $CASSANDRA_INCLUDE
+fi
+
+if [ -z $CLASSPATH ]; then
+ echo "You must set the CLASSPATH var" >&2
+ exit 1
+fi
+
+java -cp $CLASSPATH -Dstorage-config=$CASSANDRA_CONF \
+ org.apache.cassandra.tools.SSTableImport "$@"
+
+# vi:ai sw=4 ts=4 tw=0 et
Propchange: incubator/cassandra/trunk/bin/json2sstable
------------------------------------------------------------------------------
svn:executable = *
Added:
incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java?rev=885322&view=auto
==============================================================================
---
incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
(added)
+++
incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
Mon Nov 30 03:24:46 2009
@@ -0,0 +1,193 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools;
+
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.*;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamily;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.filter.QueryPath;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.DataOutputBuffer;
+import org.apache.cassandra.io.SSTableWriter;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.PosixParser;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+import org.json.simple.parser.ParseException;
+
+/**
+ * Create SSTables from JSON input
+ */
+public class SSTableImport
+{
+ private static final String KEYSPACE_OPTION = "K";
+ private static final String COLFAM_OPTION = "c";
+ private static Options options;
+ private static CommandLine cmd;
+
+ static
+ {
+ options = new Options();
+ Option optKeyspace = new Option(KEYSPACE_OPTION, true, "Keyspace
name");
+ optKeyspace.setRequired(true);
+ options.addOption(optKeyspace);
+ Option optColfamily = new Option(COLFAM_OPTION, true, "Column family");
+ optColfamily.setRequired(true);
+ options.addOption(optColfamily);
+ }
+
+ /**
+ * Add columns to a column family.
+ *
+ * @param row the columns associated with a row
+ * @param cfamily the column family to add columns to
+ */
+ private static void addToStandardCF(JSONObject row, ColumnFamily cfamily)
+ {
+ for (Map.Entry<String, String> col : (Set<Map.Entry<String, String>>)
row.entrySet())
+ {
+ QueryPath path = new QueryPath(cfamily.name(), null,
col.getKey().getBytes());
+ byte[] value = FBUtilities.hexToBytes(col.getValue());
+ cfamily.addColumn(path, value, System.currentTimeMillis());
+ }
+ }
+
+ /**
+ * Add super columns to a column family.
+ *
+ * @param row the super columns associated with a row
+ * @param cfamily the column family to add columns to
+ */
+ private static void addToSuperCF(JSONObject row, ColumnFamily cfamily)
+ {
+ // Super columns
+ for (Map.Entry<String, JSONObject> entry : (Set<Map.Entry<String,
JSONObject>>)row.entrySet())
+ {
+ byte[] superName = entry.getKey().getBytes();
+
+ // Sub-columns
+ for (Map.Entry<String, String> col : (Set<Map.Entry<String,
String>>)entry.getValue().entrySet())
+ {
+ QueryPath path = new QueryPath(cfamily.name(), superName,
col.getKey().getBytes());
+ byte[] value = FBUtilities.hexToBytes(col.getValue());
+ cfamily.addColumn(path, value, System.currentTimeMillis());
+ }
+ }
+ }
+
+ /**
+ * Convert a JSON formatted file to an SSTable.
+ *
+ * @param jsonFile the file containing JSON formatted data
+ * @param keyspace keyspace the data belongs to
+ * @param cf column family the data belongs to
+ * @param ssTablePath file to write the SSTable to
+ * @throws IOException for errors reading/writing input/output
+ * @throws ParseException for errors encountered parsing JSON input
+ */
+ private static void importJson(String jsonFile, String keyspace, String
cf, String ssTablePath)
+ throws IOException, ParseException
+ {
+ ColumnFamily cfamily = ColumnFamily.create(keyspace, cf);
+ String cfType = cfamily.type(); // Super or Standard
+ IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
+ DataOutputBuffer dob = new DataOutputBuffer();
+
+ try
+ {
+ JSONObject json = (JSONObject)JSONValue.parse(new
FileReader(jsonFile));
+ SSTableWriter writer = new SSTableWriter(ssTablePath, json.size(),
partitioner);
+ List<DecoratedKey<?>> decoratedKeys = new
ArrayList<DecoratedKey<?>>();
+
+ for (String key : (Set<String>)json.keySet())
+ decoratedKeys.add(partitioner.decorateKey(key));
+ Collections.sort(decoratedKeys);
+
+ for (DecoratedKey<?> rowKey : decoratedKeys)
+ {
+ JSONObject value = (JSONObject)json.get(rowKey.key);
+
+ if (cfType.equals("Super"))
+ addToSuperCF(value, cfamily);
+ else
+ addToStandardCF(value, cfamily);
+
+ ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
+ writer.append(rowKey, dob);
+ dob.reset();
+ cfamily.clear();
+ }
+
+ writer.closeAndOpenReader(0);
+ }
+ catch (ClassCastException cce)
+ {
+ //throw cce;
+ throw new RuntimeException("Invalid JSON input, or incorrect
column family");
+ }
+ }
+
+ /**
+ * Converts JSON to an SSTable file. JSON input can either be a file
specified
+ * using an optional command line argument, or supplied on standard in.
+ *
+ * @param args command line arguments
+ * @throws IOException on failure to open/read/write files or output
streams
+ * @throws ParseException on failure to parse JSON input
+ */
+ public static void main(String[] args) throws IOException, ParseException
+ {
+ String usage = String.format("Usage: %s -K keyspace -c column_family
<json> <sstable>%n",
+ SSTableImport.class.getName());
+
+ CommandLineParser parser = new PosixParser();
+ try
+ {
+ cmd = parser.parse(options, args);
+ } catch (org.apache.commons.cli.ParseException e1)
+ {
+ System.err.println(e1.getMessage());
+ System.err.println(usage);
+ System.exit(1);
+ }
+
+ if (cmd.getArgs().length != 2)
+ {
+ System.err.println(usage);
+ System.exit(1);
+ }
+
+ String json = cmd.getArgs()[0];
+ String ssTable = cmd.getArgs()[1];
+ String keyspace = cmd.getOptionValue(KEYSPACE_OPTION);
+ String cfamily = cmd.getOptionValue(COLFAM_OPTION);
+
+ importJson(json, keyspace, cfamily, ssTable);
+
+ System.exit(0);
+ }
+
+}
\ No newline at end of file
Propchange:
incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
------------------------------------------------------------------------------
svn:eol-style = native
