Author: jbellis
Date: Tue Apr 19 00:05:56 2011
New Revision: 1094822
URL: http://svn.apache.org/viewvc?rev=1094822&view=rev
Log:
merge from 0.7
Modified:
cassandra/branches/cassandra-0.8/ (props changed)
cassandra/branches/cassandra-0.8/CHANGES.txt
cassandra/branches/cassandra-0.8/contrib/ (props changed)
cassandra/branches/cassandra-0.8/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
(props changed)
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
(props changed)
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
(props changed)
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
(props changed)
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
(props changed)
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
Propchange: cassandra/branches/cassandra-0.8/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 19 00:05:56 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1081914,1083000
-/cassandra/branches/cassandra-0.7:1026516-1091087,1091503,1091542,1091654,1091911,1094195,1094604,1094647,1094796,1094809,1094818
+/cassandra/branches/cassandra-0.7:1026516-1094195,1094604,1094647,1094796,1094809,1094818
/cassandra/branches/cassandra-0.7.0:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689
/cassandra/trunk:1090978-1090979
Modified: cassandra/branches/cassandra-0.8/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/CHANGES.txt?rev=1094822&r1=1094821&r2=1094822&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.8/CHANGES.txt Tue Apr 19 00:05:56 2011
@@ -55,6 +55,7 @@
* Try harder to close files after compaction (CASSANDRA-2431)
* re-set bootstrapped flag after move finishes (CASSANDRA-2435)
* use 64KB flush buffer instead of in_memory_compaction_limit (CASSANDRA-2463)
+ * fix duplicate results from CFS.scan (CASSANDRA-2406)
* avoid caching token-only decoratedkeys (CASSANDRA-2416)
Propchange: cassandra/branches/cassandra-0.8/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 19 00:05:56 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/contrib:1026516-1091087,1091503,1091542,1091654,1091911,1094195,1094604,1094647,1094796,1094809,1094818
+/cassandra/branches/cassandra-0.7/contrib:1026516-1094195,1094604,1094647,1094796,1094809,1094818
/cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689
/cassandra/trunk/contrib:1090978-1090979
Modified:
cassandra/branches/cassandra-0.8/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java?rev=1094822&r1=1094821&r2=1094822&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.8/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
(original)
+++
cassandra/branches/cassandra-0.8/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
Tue Apr 19 00:05:56 2011
@@ -68,7 +68,7 @@ public class CassandraStorage extends Lo
public final static String PIG_INITIAL_ADDRESS = "PIG_INITIAL_ADDRESS";
public final static String PIG_PARTITIONER = "PIG_PARTITIONER";
- private static String UDFCONTEXT_SCHEMA_KEY = "cassandra.schema";
+ private static String UDFCONTEXT_SCHEMA_KEY_PREFIX = "cassandra.schema";
private final static ByteBuffer BOUND = ByteBufferUtil.EMPTY_BYTE_BUFFER;
private static final Log logger =
LogFactory.getLog(CassandraStorage.class);
@@ -169,7 +169,7 @@ public class CassandraStorage extends Lo
{
UDFContext context = UDFContext.getUDFContext();
Properties property = context.getUDFProperties(CassandraStorage.class);
- return cfdefFromString(property.getProperty(UDFCONTEXT_SCHEMA_KEY));
+ return cfdefFromString(property.getProperty(getSchemaContextKey()));
}
private List<AbstractType> getDefaultMarshallers(CfDef cfDef) throws
IOException
@@ -226,7 +226,7 @@ public class CassandraStorage extends Lo
this.reader = reader;
}
- private void setLocationFromUri(String location) throws IOException
+ private void setLocationFromUri(String location) throws IOException
{
// parse uri into keyspace and columnfamily
String names[];
@@ -396,7 +396,7 @@ public class CassandraStorage extends Lo
if (validators.get(column.name) == null)
// Have to special case BytesType to convert
DataByteArray into ByteBuffer
if (marshallers.get(1) instanceof BytesType)
- column.value = ByteBuffer.wrap(((DataByteArray)
pair.get(1)).get());
+ column.value = objToBB(pair.get(1));
else
column.value =
marshallers.get(1).decompose(pair.get(1));
else
@@ -434,9 +434,10 @@ public class CassandraStorage extends Lo
{
UDFContext context = UDFContext.getUDFContext();
Properties property = context.getUDFProperties(CassandraStorage.class);
-
+
+ String schemaContextKey = getSchemaContextKey();
// Only get the schema if we haven't already gotten it
- if (!property.containsKey(UDFCONTEXT_SCHEMA_KEY))
+ if (!property.containsKey(schemaContextKey))
{
Cassandra.Client client = null;
try
@@ -454,7 +455,7 @@ public class CassandraStorage extends Lo
break;
}
}
- property.setProperty(UDFCONTEXT_SCHEMA_KEY,
cfdefToString(cfDef));
+ property.setProperty(schemaContextKey, cfdefToString(cfDef));
}
catch (TException e)
{
@@ -520,4 +521,14 @@ public class CassandraStorage extends Lo
}
return cfDef;
}
+
+ private String getSchemaContextKey()
+ {
+ StringBuilder sb = new StringBuilder(UDFCONTEXT_SCHEMA_KEY_PREFIX);
+ sb.append('.');
+ sb.append(keyspace);
+ sb.append('.');
+ sb.append(column_family);
+ return sb.toString();
+ }
}
Propchange:
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 19 00:05:56 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1081914,1083000
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1091087,1091503,1091542,1091654,1091911,1094195,1094604,1094647,1094796,1094809,1094818
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1094195,1094604,1094647,1094796,1094809,1094818
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689
/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090978-1090979
Propchange:
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 19 00:05:56 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1081914,1083000
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1091087,1091503,1091542,1091654,1091911,1094195,1094604,1094647,1094796,1094809,1094818
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1094195,1094604,1094647,1094796,1094809,1094818
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689
/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090978-1090979
Propchange:
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 19 00:05:56 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1081914,1083000
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1091087,1091503,1091542,1091654,1091911,1094195,1094604,1094647,1094796,1094809,1094818
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1094195,1094604,1094647,1094796,1094809,1094818
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689
/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090978-1090979
Propchange:
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 19 00:05:56 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1081914,1083000
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1091087,1091503,1091542,1091654,1091911,1094195,1094604,1094647,1094796,1094809,1094818
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1094195,1094604,1094647,1094796,1094809,1094818
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689
/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090978-1090979
Propchange:
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 19 00:05:56 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1081914,1083000
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1091087,1091503,1091542,1091654,1091911,1094195,1094604,1094647,1094796,1094809,1094818
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1094195,1094604,1094647,1094796,1094809,1094818
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689
/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090978-1090979
Modified:
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=1094822&r1=1094821&r2=1094822&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
(original)
+++
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
Tue Apr 19 00:05:56 2011
@@ -86,7 +86,7 @@ public class DatabaseDescriptor
private static RequestSchedulerOptions requestSchedulerOptions;
public static final UUID INITIAL_VERSION = new UUID(4096, 0); // has type
nibble set to 1, everything else to zero.
- private static UUID defsVersion = INITIAL_VERSION;
+ private static volatile UUID defsVersion = INITIAL_VERSION;
/**
* Inspect the classpath to find storage configuration file
Modified:
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1094822&r1=1094821&r2=1094822&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Tue Apr 19 00:05:56 2011
@@ -1459,6 +1459,10 @@ public class ColumnFamilyStore implement
ByteBuffer startKey = clause.start_key;
QueryPath path = new QueryPath(columnFamily);
+ // we need to store last data key accessed to avoid duplicate results
+ // because in the while loop new iteration we can access the same
column if start_key was not set
+ ByteBuffer lastDataKey = null;
+
// fetch row keys matching the primary expression, fetch the slice
predicate for each
// and filter by remaining expressions. repeat until finished w/
assigned range or index row is exhausted.
outer:
@@ -1490,10 +1494,11 @@ public class ColumnFamilyStore implement
continue;
dataKey = column.name();
n++;
+
DecoratedKey dk = partitioner.decorateKey(dataKey);
if (!range.right.equals(partitioner.getMinimumToken()) &&
range.right.compareTo(dk.token) < 0)
break outer;
- if (!range.contains(dk.token))
+ if (!range.contains(dk.token) || dataKey.equals(lastDataKey))
continue;
// get the row columns requested, and additional columns for
the expressions if necessary
@@ -1534,7 +1539,8 @@ public class ColumnFamilyStore implement
}
if (n < clause.count || startKey.equals(dataKey))
break;
- startKey = dataKey;
+
+ lastDataKey = startKey = dataKey;
}
return rows;
Modified:
cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java?rev=1094822&r1=1094821&r2=1094822&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
(original)
+++
cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
Tue Apr 19 00:05:56 2011
@@ -211,6 +211,35 @@ public class ColumnFamilyStoreTest exten
}
@Test
+ public void testLargeScan() throws IOException
+ {
+ RowMutation rm;
+ for (int i = 0; i < 100; i++)
+ {
+ rm = new RowMutation("Keyspace1", ByteBufferUtil.bytes("key" + i));
+ rm.add(new QueryPath("Indexed1", null,
ByteBufferUtil.bytes("birthdate")), ByteBufferUtil.bytes(34L), 0);
+ rm.add(new QueryPath("Indexed1", null,
ByteBufferUtil.bytes("notbirthdate")), ByteBufferUtil.bytes((long) (i % 2)), 0);
+ rm.applyUnsafe();
+ }
+
+ IndexExpression expr = new
IndexExpression(ByteBufferUtil.bytes("birthdate"), IndexOperator.EQ,
ByteBufferUtil.bytes(34L));
+ IndexExpression expr2 = new
IndexExpression(ByteBufferUtil.bytes("notbirthdate"), IndexOperator.EQ,
ByteBufferUtil.bytes(1L));
+ IndexClause clause = new IndexClause(Arrays.asList(expr, expr2),
ByteBufferUtil.EMPTY_BYTE_BUFFER, 100);
+ IFilter filter = new IdentityQueryFilter();
+ IPartitioner p = StorageService.getPartitioner();
+ Range range = new Range(p.getMinimumToken(), p.getMinimumToken());
+ List<Row> rows =
Table.open("Keyspace1").getColumnFamilyStore("Indexed1").scan(clause, range,
filter);
+
+ assert rows != null;
+ assert rows.size() == 50 : rows.size();
+ Set<DecoratedKey> keys = new HashSet<DecoratedKey>();
+ // extra check that there are no duplicate results -- see
https://issues.apache.org/jira/browse/CASSANDRA-2406
+ for (Row row : rows)
+ keys.add(row.key);
+ assert rows.size() == keys.size();
+ }
+
+ @Test
public void testIndexDeletions() throws IOException
{
ColumnFamilyStore cfs =
Table.open("Keyspace3").getColumnFamilyStore("Indexed1");
