Author: jbellis
Date: Sat Aug 21 11:37:04 2010
New Revision: 987728
URL: http://svn.apache.org/viewvc?rev=987728&view=rev
Log:
allow index expressions against columns that are not part of the SlicePredicate.
patch by jbellis; reviewed by Nate McCall for CASSANDRA-1410
Modified:
cassandra/trunk/CHANGES.txt
cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
Modified: cassandra/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=987728&r1=987727&r2=987728&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Sat Aug 21 11:37:04 2010
@@ -24,6 +24,8 @@ dev
* use JNA, if present, to take snapshots (CASSANDRA-1371)
* truncate hints if starting 0.7 for the first time (CASSANDRA-1414)
* fix FD leak in single-row slicepredicate queries (CASSANDRA-1416)
+ * allow index expressions against columns that are not part of the
+ SlicePredicate (CASSANDRA-1410)
0.7-beta1
Modified:
cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=987728&r1=987727&r2=987728&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Sat
Aug 21 11:37:04 2010
@@ -897,7 +897,12 @@ public class ColumnFamilyStore implement
*/
public ColumnFamily getColumnFamily(QueryFilter filter)
{
- return getColumnFamily(filter, (int) (System.currentTimeMillis() /
1000) - metadata.gcGraceSeconds);
+ return getColumnFamily(filter, gcBefore());
+ }
+
+ private int gcBefore()
+ {
+ return (int) (System.currentTimeMillis() / 1000) -
metadata.gcGraceSeconds;
}
private ColumnFamily cacheRow(DecoratedKey key)
@@ -1136,15 +1141,68 @@ public class ColumnFamilyStore implement
public List<Row> scan(IndexClause clause, AbstractBounds range, IFilter
dataFilter)
{
+ // Start with the most-restrictive indexed clause, then apply
remaining clauses
+ // to each row matching that clause.
// TODO: allow merge join instead of just one index + loop
- IndexExpression first = highestSelectivityPredicate(clause);
- ColumnFamilyStore indexCFS =
getIndexedColumnFamilyStore(first.column_name);
+ IndexExpression primary = highestSelectivityPredicate(clause);
+ ColumnFamilyStore indexCFS =
getIndexedColumnFamilyStore(primary.column_name);
assert indexCFS != null;
- DecoratedKey indexKey = indexCFS.partitioner.decorateKey(first.value);
+ DecoratedKey indexKey =
indexCFS.partitioner.decorateKey(primary.value);
+
+ // if the slicepredicate doesn't contain all the columns for which we
have expressions to evaluate,
+ // it needs to be expanded to include those too
+ IFilter firstFilter = dataFilter;
+ NamesQueryFilter extraFilter = null;
+ if (clause.expressions.size() > 1)
+ {
+ if (dataFilter instanceof SliceQueryFilter)
+ {
+ // if we have a high chance of getting all the columns in a
single index slice, do that.
+ // otherwise, create an extraFilter to fetch by name the
columns referenced by the additional expressions.
+ if (getMaxRowSize() < DatabaseDescriptor.getColumnIndexSize())
+ {
+ firstFilter = new
SliceQueryFilter(ArrayUtils.EMPTY_BYTE_ARRAY,
+
ArrayUtils.EMPTY_BYTE_ARRAY,
+ ((SliceQueryFilter)
dataFilter).reversed,
+ Integer.MAX_VALUE);
+ }
+ else
+ {
+ SortedSet<byte[]> columns = new
TreeSet<byte[]>(getComparator());
+ for (IndexExpression expr : clause.expressions)
+ {
+ if (expr == primary)
+ continue;
+ columns.add(expr.column_name);
+ }
+ extraFilter = new NamesQueryFilter(columns);
+ }
+ }
+ else
+ {
+ // just add in columns that are not part of the resultset
+ assert dataFilter instanceof NamesQueryFilter;
+ SortedSet<byte[]> columns = new
TreeSet<byte[]>(getComparator());
+ for (IndexExpression expr : clause.expressions)
+ {
+ if (expr == primary || ((NamesQueryFilter)
dataFilter).columns.contains(expr.column_name))
+ continue;
+ columns.add(expr.column_name);
+ }
+ if (columns.size() > 0)
+ {
+ columns.addAll(((NamesQueryFilter) dataFilter).columns);
+ firstFilter = new NamesQueryFilter(columns);
+ }
+ }
+ }
List<Row> rows = new ArrayList<Row>();
byte[] startKey = clause.start_key;
-
+ QueryPath path = new QueryPath(columnFamily);
+
+ // fetch row keys matching the primary expression, fetch the slice
predicate for each
+ // and filter by remaining expressions. repeat until finished w/
assigned range or index row is exhausted.
outer:
while (true)
{
@@ -1176,9 +1234,38 @@ public class ColumnFamilyStore implement
break outer;
if (!range.contains(dk.token))
continue;
- ColumnFamily data = getColumnFamily(new QueryFilter(dk, new
QueryPath(columnFamily), dataFilter));
- if (satisfies(data, clause, first))
+
+ // get the row columns requested, and additional columns for
the expressions if necessary
+ ColumnFamily data = getColumnFamily(new QueryFilter(dk, path,
firstFilter));
+ if (extraFilter != null)
+ {
+ // we might have gotten the expression columns in with the
main data slice, but
+ // we can't know for sure until that slice is done. So,
we'll do the extra query
+ // if we go through and any expression columns are not
present.
+ for (IndexExpression expr : clause.expressions)
+ {
+ if (expr != primary &&
data.getColumn(expr.column_name) == null)
+ {
+ data.addAll(getColumnFamily(new QueryFilter(dk,
path, extraFilter)));
+ break;
+ }
+ }
+ }
+
+ if (satisfies(data, clause, primary))
+ {
+ // cut the resultset back to what was requested, if
necessary
+ if (firstFilter != dataFilter)
+ {
+ ColumnFamily expandedData = data;
+ data = expandedData.cloneMeShallow();
+ IColumnIterator iter =
dataFilter.getMemtableColumnIterator(expandedData, dk, getComparator());
+ new QueryFilter(dk, path,
dataFilter).collectCollatedColumns(data, iter, gcBefore());
+ }
+
rows.add(new Row(dk, data));
+ }
+
if (rows.size() == clause.count)
break outer;
}
Modified:
cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java?rev=987728&r1=987727&r2=987728&view=diff
==============================================================================
---
cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
(original)
+++
cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
Sat Aug 21 11:37:04 2010
@@ -179,6 +179,7 @@ public class ColumnFamilyStoreTest exten
rm.add(new QueryPath("Indexed1", null, "birthdate".getBytes("UTF8")),
FBUtilities.toByteArray(3L), new TimestampClock(0));
rm.apply();
+ // basic single-expression query
IndexExpression expr = new
IndexExpression("birthdate".getBytes("UTF8"), IndexOperator.EQ,
FBUtilities.toByteArray(1L));
IndexClause clause = new IndexClause(Arrays.asList(expr),
ArrayUtils.EMPTY_BYTE_ARRAY, 100);
IFilter filter = new IdentityQueryFilter();
@@ -193,12 +194,28 @@ public class ColumnFamilyStoreTest exten
assert Arrays.equals(FBUtilities.toByteArray(1L),
rows.get(0).cf.getColumn("birthdate".getBytes("UTF8")).value());
assert Arrays.equals(FBUtilities.toByteArray(1L),
rows.get(1).cf.getColumn("birthdate".getBytes("UTF8")).value());
+ // add a second expression
IndexExpression expr2 = new
IndexExpression("notbirthdate".getBytes("UTF8"), IndexOperator.GTE,
FBUtilities.toByteArray(2L));
clause = new IndexClause(Arrays.asList(expr, expr2),
ArrayUtils.EMPTY_BYTE_ARRAY, 100);
rows =
Table.open("Keyspace1").getColumnFamilyStore("Indexed1").scan(clause, range,
filter);
assert rows.size() == 1 : StringUtils.join(rows, ",");
assert Arrays.equals("k3".getBytes(), rows.get(0).key.key);
+
+ // same query again, but with resultset not including the subordinate
expression
+ rows =
Table.open("Keyspace1").getColumnFamilyStore("Indexed1").scan(clause, range,
new NamesQueryFilter("birthdate".getBytes("UTF8")));
+
+ assert rows.size() == 1 : StringUtils.join(rows, ",");
+ assert Arrays.equals("k3".getBytes(), rows.get(0).key.key);
+ assert rows.get(0).cf.getColumnCount() == 1 : rows.get(0).cf;
+
+ // once more, this time with a slice rowset that needs to be expanded
+ SliceQueryFilter sqf = new
SliceQueryFilter(ArrayUtils.EMPTY_BYTE_ARRAY, ArrayUtils.EMPTY_BYTE_ARRAY,
false, 0);
+ rows =
Table.open("Keyspace1").getColumnFamilyStore("Indexed1").scan(clause, range,
sqf);
+
+ assert rows.size() == 1 : StringUtils.join(rows, ",");
+ assert Arrays.equals("k3".getBytes(), rows.get(0).key.key);
+ assert rows.get(0).cf.getColumnCount() == 0;
}
@Test
