Author: jbellis
Date: Mon May 23 14:43:48 2011
New Revision: 1126520
URL: http://svn.apache.org/viewvc?rev=1126520&view=rev
Log:
merge from 0.8.0
Removed:
cassandra/branches/cassandra-0.8/examples/hadoop_streaming_output/
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/hadoop/streaming/
Modified:
cassandra/branches/cassandra-0.8/ (props changed)
cassandra/branches/cassandra-0.8/CHANGES.txt
cassandra/branches/cassandra-0.8/NEWS.txt
cassandra/branches/cassandra-0.8/contrib/ (props changed)
cassandra/branches/cassandra-0.8/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
(props changed)
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
(props changed)
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
(props changed)
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
(props changed)
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
(props changed)
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/hadoop/ColumnFamilyOutputFormat.java
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java
Propchange: cassandra/branches/cassandra-0.8/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 14:43:48 2011
@@ -2,7 +2,7 @@
/cassandra/branches/cassandra-0.7:1026516-1126504
/cassandra/branches/cassandra-0.7.0:1053690-1055654
/cassandra/branches/cassandra-0.8:1090934-1125013,1125041
-/cassandra/branches/cassandra-0.8.0:1125021-1126507
+/cassandra/branches/cassandra-0.8.0:1125021-1126519
/cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1:1102511-1125020
/incubator/cassandra/branches/cassandra-0.3:774578-796573
Modified: cassandra/branches/cassandra-0.8/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/CHANGES.txt?rev=1126520&r1=1126519&r2=1126520&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.8/CHANGES.txt Mon May 23 14:43:48 2011
@@ -28,6 +28,7 @@
* update CQL consistency levels (CASSANDRA-2566)
* debian packaging fixes (CASSANDRA-2481, 2647)
* fix UUIDType, IntegerType for direct buffers (CASSANDRA-2682, 2684)
+ * switch to native Thrift for Hadoop map/reduce (CASSANDRA-2667)
0.8.0-rc1
Modified: cassandra/branches/cassandra-0.8/NEWS.txt
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/NEWS.txt?rev=1126520&r1=1126519&r2=1126520&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/NEWS.txt (original)
+++ cassandra/branches/cassandra-0.8/NEWS.txt Mon May 23 14:43:48 2011
@@ -13,8 +13,10 @@ Upgrading
- 0.8 is fully API-compatible with 0.7. You can continue
to use your 0.7 clients.
- Avro record classes used in map/reduce and Hadoop streaming code have
- moved from org.apache.cassandra.avro to org.apache.cassandra.hadoop.avro,
- applications using these classes will need to be updated accordingly.
+ been removed. Map/reduce can be switched to Thrift by changing
+ org.apache.cassandra.avro in import statements to
+ org.apache.cassandra.thrift (no class names change). Streaming support
+ has been removed for the time being.
- The loadbalance command has been removed from nodetool. For similar
behavior, decommission then rebootstrap with empty initial_token.
- Thrift unframed mode has been removed.
Propchange: cassandra/branches/cassandra-0.8/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 14:43:48 2011
@@ -2,7 +2,7 @@
/cassandra/branches/cassandra-0.7/contrib:1026516-1126504
/cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125041
-/cassandra/branches/cassandra-0.8.0/contrib:1125021-1126507
+/cassandra/branches/cassandra-0.8.0/contrib:1125021-1126519
/cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/contrib:1102511-1125020
/incubator/cassandra/branches/cassandra-0.3/contrib:774578-796573
Modified:
cassandra/branches/cassandra-0.8/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java?rev=1126520&r1=1126519&r2=1126520&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.8/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
(original)
+++
cassandra/branches/cassandra-0.8/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
Mon May 23 14:43:48 2011
@@ -22,6 +22,7 @@ import java.util.*;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.TypeParser;
import org.apache.cassandra.thrift.*;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.commons.logging.Log;
@@ -32,9 +33,9 @@ import org.apache.cassandra.db.IColumn;
import org.apache.cassandra.db.SuperColumn;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.hadoop.*;
-import org.apache.cassandra.hadoop.avro.Mutation;
-import org.apache.cassandra.hadoop.avro.Deletion;
-import org.apache.cassandra.hadoop.avro.ColumnOrSuperColumn;
+import org.apache.cassandra.thrift.Mutation;
+import org.apache.cassandra.thrift.Deletion;
+import org.apache.cassandra.thrift.ColumnOrSuperColumn;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.hadoop.conf.Configuration;
@@ -158,7 +159,7 @@ public class CassandraStorage extends Lo
// super
ArrayList<Tuple> subcols = new ArrayList<Tuple>();
- for (IColumn subcol : ((SuperColumn)col).getSubColumns())
+ for (IColumn subcol : col.getSubColumns())
subcols.add(columnToTuple(subcol.name(), subcol, cfDef));
pair.set(1, new DefaultDataBag(subcols));
@@ -179,8 +180,8 @@ public class CassandraStorage extends Lo
AbstractType default_validator = null;
try
{
- comparator = FBUtilities.getComparator(cfDef.comparator_type);
- default_validator =
FBUtilities.getComparator(cfDef.default_validation_class);
+ comparator = TypeParser.parse(cfDef.comparator_type);
+ default_validator =
TypeParser.parse(cfDef.default_validation_class);
}
catch (ConfigurationException e)
{
@@ -202,7 +203,7 @@ public class CassandraStorage extends Lo
AbstractType validator = null;
try
{
- validator =
FBUtilities.getComparator(cd.getValidation_class());
+ validator = TypeParser.parse(cd.getValidation_class());
validators.put(cd.name, validator);
}
catch (ConfigurationException e)
@@ -385,13 +386,13 @@ public class CassandraStorage extends Lo
if (pair.get(1) == null)
{
mutation.deletion = new Deletion();
- mutation.deletion.predicate = new
org.apache.cassandra.hadoop.avro.SlicePredicate();
+ mutation.deletion.predicate = new
org.apache.cassandra.thrift.SlicePredicate();
mutation.deletion.predicate.column_names =
Arrays.asList(objToBB(pair.get(0)));
mutation.deletion.timestamp =
System.currentTimeMillis() * 1000;
}
else
{
- org.apache.cassandra.hadoop.avro.Column column = new
org.apache.cassandra.hadoop.avro.Column();
+ org.apache.cassandra.thrift.Column column = new
org.apache.cassandra.thrift.Column();
column.name =
marshallers.get(0).decompose((pair.get(0)));
if (validators.get(column.name) == null)
// Have to special case BytesType to convert
DataByteArray into ByteBuffer
Propchange:
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 14:43:48 2011
@@ -2,7 +2,7 @@
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1126504
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125041
-/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1126507
+/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1126519
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1102511-1125020
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573
Propchange:
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 14:43:48 2011
@@ -2,7 +2,7 @@
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1126504
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125041
-/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1126507
+/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1126519
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1102511-1125020
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198
Propchange:
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 14:43:48 2011
@@ -2,7 +2,7 @@
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1126504
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125041
-/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1126507
+/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1126519
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1102511-1125020
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573
Propchange:
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 14:43:48 2011
@@ -2,7 +2,7 @@
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1126504
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125041
-/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1126507
+/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1126519
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1102511-1125020
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573
Propchange:
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 14:43:48 2011
@@ -2,7 +2,7 @@
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1126504
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125041
-/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1126507
+/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1126519
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1102511-1125020
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198
Modified:
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/hadoop/ColumnFamilyOutputFormat.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/hadoop/ColumnFamilyOutputFormat.java?rev=1126520&r1=1126519&r2=1126520&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/hadoop/ColumnFamilyOutputFormat.java
(original)
+++
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/hadoop/ColumnFamilyOutputFormat.java
Mon May 23 14:43:48 2011
@@ -31,7 +31,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.auth.SimpleAuthenticator;
-import org.apache.cassandra.hadoop.avro.Mutation;
import org.apache.cassandra.thrift.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.*;
Modified:
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java?rev=1126520&r1=1126519&r2=1126520&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java
(original)
+++
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java
Mon May 23 14:43:48 2011
@@ -59,8 +59,8 @@ import org.apache.cassandra.utils.ByteBu
* @see OutputFormat
*
*/
-final class ColumnFamilyRecordWriter extends
RecordWriter<ByteBuffer,List<org.apache.cassandra.hadoop.avro.Mutation>>
-implements
org.apache.hadoop.mapred.RecordWriter<ByteBuffer,List<org.apache.cassandra.hadoop.avro.Mutation>>
+final class ColumnFamilyRecordWriter extends
RecordWriter<ByteBuffer,List<Mutation>>
+implements org.apache.hadoop.mapred.RecordWriter<ByteBuffer,List<Mutation>>
{
// The configuration this writer is associated with.
private final Configuration conf;
@@ -122,7 +122,7 @@ implements org.apache.hadoop.mapred.Reco
* @throws IOException
*/
@Override
- public void write(ByteBuffer keybuff,
List<org.apache.cassandra.hadoop.avro.Mutation> value) throws IOException
+ public void write(ByteBuffer keybuff, List<Mutation> value) throws
IOException
{
Range range = ringCache.getRange(keybuff);
@@ -136,77 +136,8 @@ implements org.apache.hadoop.mapred.Reco
clients.put(range, client);
}
- for (org.apache.cassandra.hadoop.avro.Mutation amut : value)
- client.put(new Pair<ByteBuffer,Mutation>(keybuff,
avroToThrift(amut)));
- }
-
- /**
- * Deep copies the given Avro mutation into a new Thrift mutation.
- */
- private Mutation avroToThrift(org.apache.cassandra.hadoop.avro.Mutation
amut)
- {
- Mutation mutation = new Mutation();
- org.apache.cassandra.hadoop.avro.ColumnOrSuperColumn acosc =
amut.column_or_supercolumn;
- if (acosc == null)
- {
- // deletion
- assert amut.deletion != null;
- Deletion deletion = new
Deletion().setTimestamp(amut.deletion.timestamp);
- mutation.setDeletion(deletion);
-
- org.apache.cassandra.hadoop.avro.SlicePredicate apred =
amut.deletion.predicate;
- if (apred == null && amut.deletion.super_column == null)
- {
- // leave Deletion alone to delete entire row
- }
- else if (amut.deletion.super_column != null)
- {
- // super column
-
deletion.setSuper_column(ByteBufferUtil.getArray(amut.deletion.super_column));
- }
- else if (apred.column_names != null)
- {
- // column names
- List<ByteBuffer> names = new
ArrayList<ByteBuffer>(apred.column_names.size());
- for (ByteBuffer name : apred.column_names)
- names.add(name);
- deletion.setPredicate(new
SlicePredicate().setColumn_names(names));
- }
- else
- {
- // range
- deletion.setPredicate(new
SlicePredicate().setSlice_range(avroToThrift(apred.slice_range)));
- }
- }
- else
- {
- // creation
- ColumnOrSuperColumn cosc = new ColumnOrSuperColumn();
- mutation.setColumn_or_supercolumn(cosc);
- if (acosc.column != null)
- // standard column
- cosc.setColumn(avroToThrift(acosc.column));
- else
- {
- // super column
- ByteBuffer scolname = acosc.super_column.name;
- List<Column> scolcols = new
ArrayList<Column>(acosc.super_column.columns.size());
- for (org.apache.cassandra.hadoop.avro.Column acol :
acosc.super_column.columns)
- scolcols.add(avroToThrift(acol));
- cosc.setSuper_column(new SuperColumn(scolname, scolcols));
- }
- }
- return mutation;
- }
-
- private SliceRange
avroToThrift(org.apache.cassandra.hadoop.avro.SliceRange asr)
- {
- return new SliceRange(asr.start, asr.finish, asr.reversed, asr.count);
- }
-
- private Column avroToThrift(org.apache.cassandra.hadoop.avro.Column acol)
- {
- return new
Column(acol.name).setValue(acol.value).setTimestamp(acol.timestamp);
+ for (Mutation amut : value)
+ client.put(new Pair<ByteBuffer,Mutation>(keybuff, amut));
}
/**
