This is an automated email from the ASF dual-hosted git repository. dcapwell pushed a commit to branch cassandra-5.0 in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 8125a694800df01f1d1117626981d48dbe52eb6d Merge: 3cdd540bfe 4da92af589 Author: David Capwell <[email protected]> AuthorDate: Tue Jan 7 17:45:06 2025 -0800 Merge branch 'cassandra-4.1' into cassandra-5.0 CHANGES.txt | 1 + .../org/apache/cassandra/db/filter/RowFilter.java | 32 ++++++++++++---------- .../cql3/validation/operations/SelectTest.java | 30 ++++++++++++++++++++ .../cassandra/index/sai/plan/OperationTest.java | 2 +- .../cassandra/index/sasi/plan/OperationTest.java | 2 +- 5 files changed, 50 insertions(+), 17 deletions(-) diff --cc CHANGES.txt index b76e5bed8a,3e5597558d..8eb1fba300 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,16 -1,10 +1,17 @@@ -4.1.8 +5.0.3 + * Avoid memory allocation in offheap_object's NativeCell.valueSize() and NativeClustering.dataSize() (CASSANDRA-20162) + * Add flag to avoid invalidating key cache on sstable deletions (CASSANDRA-20068) + * Interpret inet, bigint, varint, and decimal as non-reversed types for query construction and post-filtering (CASSANDRA-20100) + * Fix delayed gossip shutdown messages clobbering startup states that leave restarted nodes appearing down (CASSANDRA-20033) + * Streamline the serialized format for index status gossip messages (CASSANDRA-20058) + * Batch clusterings into single SAI partition post-filtering reads (CASSANDRA-19497) + * Ban the usage of "var" instead of full types in the production code (CASSANDRA-20038) + * Suppress CVE-2024-45772 from lucene-core-9.7.0.jar (CASSANDRA-20024) +Merged from 4.1: * Add nodetool checktokenmetadata command that checks TokenMetadata is insync with Gossip endpointState (CASSANDRA-18758) - * Backport Java 11 support for Simulator (CASSANDRA-17178/CASSANDRA-19935) * Equality check for Paxos.Electorate should not depend on collection types (CASSANDRA-19935) - * Fix race condition in DecayingEstimatedHistogramReservoir during rescale (CASSANDRA-19365) Merged from 4.0: + * IndexOutOfBoundsException when accessing partition where the column was deleted (CASSANDRA-20108) * Enhance CQLSSTableWriter to notify clients on sstable production (CASSANDRA-19800) * Fix gossip issue with gossip-only and bootstrapping nodes missing DC/Rack/Host ID endpoint state (CASSANDRA-19983) * Change the resolution of AbstractCommitLogService#lastSyncedAt to nanos to be aligned with later comparisons (CASSANDRA-20074) diff --cc src/java/org/apache/cassandra/db/filter/RowFilter.java index beb3edfcd7,4730656263..0742f4ee9f --- a/src/java/org/apache/cassandra/db/filter/RowFilter.java +++ b/src/java/org/apache/cassandra/db/filter/RowFilter.java @@@ -191,74 -130,7 +191,74 @@@ public class RowFilter implements Itera return false; } - protected abstract Transformation<BaseRowIterator<?>> filter(TableMetadata metadata, int nowInSec); + /** + * Note that the application of this transformation does not yet take {@link #isStrict()} into account. This means + * that even when strict filtering is not safe, expressions will be applied as intersections rather than unions. + * The filter will always be evaluated strictly in conjunction with replica filtering protection at the + * coordinator, however, even after CASSANDRA-19007 is addressed. + * + * @see <a href="https://issues.apache.org/jira/browse/CASSANDRA-190007";>CASSANDRA-19007</a> + */ + protected Transformation<BaseRowIterator<?>> filter(TableMetadata metadata, long nowInSec) + { + List<Expression> partitionLevelExpressions = new ArrayList<>(); + List<Expression> rowLevelExpressions = new ArrayList<>(); + for (Expression e: expressions) + { + if (e.column.isStatic() || e.column.isPartitionKey()) + partitionLevelExpressions.add(e); + else + rowLevelExpressions.add(e); + } + + long numberOfRegularColumnExpressions = rowLevelExpressions.size(); + final boolean filterNonStaticColumns = numberOfRegularColumnExpressions > 0; + + return new Transformation<>() + { + DecoratedKey pk; + + @Override + protected BaseRowIterator<?> applyToPartition(BaseRowIterator<?> partition) + { + pk = partition.partitionKey(); + + // Short-circuit all partitions that won't match based on static and partition keys + for (Expression e : partitionLevelExpressions) - if (!e.isSatisfiedBy(metadata, partition.partitionKey(), partition.staticRow())) ++ if (!e.isSatisfiedBy(metadata, partition.partitionKey(), partition.staticRow(), nowInSec)) + { + partition.close(); + return null; + } + + BaseRowIterator<?> iterator = partition instanceof UnfilteredRowIterator + ? Transformation.apply((UnfilteredRowIterator) partition, this) + : Transformation.apply((RowIterator) partition, this); + + if (filterNonStaticColumns && !iterator.hasNext()) + { + iterator.close(); + return null; + } + + return iterator; + } + + @Override + public Row applyToRow(Row row) + { + Row purged = row.purge(DeletionPurger.PURGE_ALL, nowInSec, metadata.enforceStrictLiveness()); + if (purged == null) + return null; + + for (Expression e : rowLevelExpressions) - if (!e.isSatisfiedBy(metadata, pk, purged)) ++ if (!e.isSatisfiedBy(metadata, pk, purged, nowInSec)) + return null; + + return row; + } + }; + } /** * Filters the provided iterator so that only the row satisfying the expression of this filter @@@ -529,9 -470,9 +529,9 @@@ * (i.e. it should come from a RowIterator). * @return whether the row is satisfied by this expression. */ - public abstract boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row); - public abstract boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row, int nowInSec); ++ public abstract boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row, long nowInSec); - protected ByteBuffer getValue(TableMetadata metadata, DecoratedKey partitionKey, Row row) - protected ByteBuffer getValue(TableMetadata metadata, DecoratedKey partitionKey, Row row, int nowInSec) ++ protected ByteBuffer getValue(TableMetadata metadata, DecoratedKey partitionKey, Row row, long nowInSec) { switch (column.kind) { @@@ -704,7 -645,7 +704,7 @@@ super(column, operator, value); } - public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row) - public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row, int nowInSec) ++ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row, long nowInSec) { // We support null conditions for LWT (in ColumnCondition) but not for RowFilter. // TODO: we should try to merge both code someday. @@@ -744,10 -685,9 +744,10 @@@ case LIKE_SUFFIX: case LIKE_CONTAINS: case LIKE_MATCHES: + case ANN: { - assert !column.isComplex() : "Only CONTAINS and CONTAINS_KEY are supported for 'complex' types"; + assert !column.isComplex() : "Only CONTAINS and CONTAINS_KEY are supported for collection types"; - ByteBuffer foundValue = getValue(metadata, partitionKey, row); + ByteBuffer foundValue = getValue(metadata, partitionKey, row, nowInSec); // Note that CQL expression are always of the form 'x < 4', i.e. the tested value is on the left. return foundValue != null && operator.isSatisfiedBy(column.type, foundValue, value); } @@@ -874,7 -814,8 +874,8 @@@ return CompositeType.build(ByteBufferAccessor.instance, key, value); } - public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row) + @Override - public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row, int nowInSec) ++ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row, long nowInSec) { assert key != null; // We support null conditions for LWT (in ColumnCondition) but not for RowFilter. @@@ -993,7 -934,8 +994,8 @@@ } // Filtering by custom expressions isn't supported yet, so just accept any row - public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row) + @Override - public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row, int nowInSec) ++ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row, long nowInSec) { return true; } diff --cc test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java index 639f6a5145,cdedec01a8..4c7a7c1c2a --- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java @@@ -2326,6 -2302,36 +2326,36 @@@ public class SelectTest extends CQLTest }); } + @Test + public void filteringOnDeletedStaticColumnValue() throws Throwable + { + // Create table with int-only columns + createTable("CREATE TABLE %s (pk0 int, pk1 int, ck0 int, ck1 int, s0 tinyint static, v0 int, v1 int, PRIMARY KEY ((pk0, pk1), ck0, ck1))"); + + // Insert rows + execute("INSERT INTO %s (pk0, pk1, s0, ck0, ck1, v0, v1) VALUES (?, ?, ?, ?, ?, ?, ?)", 1000, 2000, (byte) 126, 100, 1, 20, 30); + execute("INSERT INTO %s (pk0, pk1, s0, ck0, ck1, v0, v1) VALUES (?, ?, ?, ?, ?, ?, ?)", 1000, 2000, (byte) 125, 200, 2, 40, 50); + execute("INSERT INTO %s (pk0, pk1, s0, ck0, ck1, v0, v1) VALUES (?, ?, ?, ?, ?, ?, ?)", 1000, 3000, (byte) 122, 300, 3, 60, 70); + execute("DELETE s0,v0,v1 FROM %s WHERE pk0=1000 AND pk1=2000 and ck0=100 and ck1=1"); + + beforeAndAfterFlush(() -> { + // Verify the columns are deleted + assertRows(execute("SELECT pk0, pk1, s0, ck0, ck1, v0, v1 FROM %s WHERE s0=? ALLOW FILTERING", (byte) 122), - row(1000, 3000, (byte) 122, 300, 3, 60, 70)); ++ row(1000, 3000, (byte) 122, 300, 3, 60, 70)); + }); + + execute("DELETE v0 FROM %s WHERE pk0=1000 AND pk1=3000 AND ck0=300 AND ck1=3"); + + beforeAndAfterFlush(() -> { + assertRows(execute("SELECT pk0, pk1, s0, ck0, ck1, v0, v1 FROM %s WHERE s0=? ALLOW FILTERING", (byte) 122), - row(1000, 3000, (byte) 122, 300, 3, null, 70)); ++ row(1000, 3000, (byte) 122, 300, 3, null, 70)); + + assertRows(execute("SELECT pk0, pk1, s0, ck0, ck1, v0, v1 FROM %s WHERE pk0=1000 AND pk1=3000 AND ck0=300 AND ck1=3"), - row(1000, 3000, (byte) 122, 300, 3, null, 70)); ++ row(1000, 3000, (byte) 122, 300, 3, null, 70)); + }); + + } + @Test public void containsFilteringOnNonClusteringColumn() throws Throwable { createTable("CREATE TABLE %s (a int, b int, c int, d list<int>, PRIMARY KEY (a, b, c))"); diff --cc test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java index 8b0acaaf2b,0000000000..81292cbda0 mode 100644,000000..100644 --- a/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java +++ b/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java @@@ -1,549 -1,0 +1,549 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.index.sai.plan; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.Operator; +import org.apache.cassandra.cql3.statements.schema.IndexTarget; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.PartitionRangeReadCommand; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.CollectionType; +import org.apache.cassandra.db.marshal.CompositeType; +import org.apache.cassandra.db.marshal.DoubleType; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.ListType; +import org.apache.cassandra.db.marshal.LongType; +import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.BufferCell; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.index.sai.QueryContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.IndexMetadata; +import org.apache.cassandra.schema.Indexes; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.FBUtilities; + +import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG; +import static org.apache.cassandra.db.marshal.Int32Type.instance; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class OperationTest +{ + private static final String KS_NAME = "sai"; + private static final String CF_NAME = "test_cf"; + private static final String CLUSTERING_CF_NAME = "clustering_test_cf"; + private static final String STATIC_CF_NAME = "static_sai_test_cf"; + + private static ColumnFamilyStore BACKEND; + private static ColumnFamilyStore CLUSTERING_BACKEND; + private static ColumnFamilyStore STATIC_BACKEND; + + private QueryController controller; + private QueryController controllerClustering; + private QueryController controllerStatic; + + @BeforeClass + public static void loadSchema() throws ConfigurationException + { + CASSANDRA_CONFIG.setString("cassandra-murmur.yaml"); + + SchemaLoader.loadSchema(); + + SchemaLoader.createKeyspace(KS_NAME, + KeyspaceParams.simpleTransient(1), + skinnySAITableMetadata(KS_NAME, CF_NAME), + clusteringSAITableMetadata(KS_NAME, CLUSTERING_CF_NAME), + staticSAITableMetadata(KS_NAME, STATIC_CF_NAME)); + + BACKEND = Keyspace.open(KS_NAME).getColumnFamilyStore(CF_NAME); + CLUSTERING_BACKEND = Keyspace.open(KS_NAME).getColumnFamilyStore(CLUSTERING_CF_NAME); + STATIC_BACKEND = Keyspace.open(KS_NAME).getColumnFamilyStore(STATIC_CF_NAME); + } + + @Before + public void beforeTest() + { + ReadCommand command = PartitionRangeReadCommand.allDataRead(BACKEND.metadata(), FBUtilities.nowInSeconds()); + controller = new QueryController(BACKEND, command, null, contextWithUnrepairedMatches(command)); + + command = PartitionRangeReadCommand.allDataRead(CLUSTERING_BACKEND.metadata(), FBUtilities.nowInSeconds()); + controllerClustering = new QueryController(CLUSTERING_BACKEND, command, null, contextWithUnrepairedMatches(command)); + + command = PartitionRangeReadCommand.allDataRead(STATIC_BACKEND.metadata(), FBUtilities.nowInSeconds()); + controllerStatic = new QueryController(STATIC_BACKEND, command, null, contextWithUnrepairedMatches(command)); + } + + private static QueryContext contextWithUnrepairedMatches(ReadCommand command) + { + QueryContext context = new QueryContext(command, DatabaseDescriptor.getRangeRpcTimeout(TimeUnit.MILLISECONDS)); + context.hasUnrepairedMatches = true; + return context; + } + + @Test + public void testAnalyze() + { + final ColumnMetadata age = getColumn(UTF8Type.instance.decompose("age")); + + // age > 1 AND age < 7 + Map<Expression.IndexOperator, Expression> expressions = convert(Operation.buildIndexExpressions(controller, + Arrays.asList(new SimpleExpression(age, Operator.GT, Int32Type.instance.decompose(1)), + new SimpleExpression(age, Operator.LT, Int32Type.instance.decompose(7))))); + + assertEquals(1, expressions.size()); + + Expression rangeExpression = expressions.get(Expression.IndexOperator.RANGE); + + assertExpression(rangeExpression, Expression.IndexOperator.RANGE, Int32Type.instance.decompose(1), false, Int32Type.instance.decompose(7), false); + } + + @Test + public void testSatisfiedBy() + { + final ColumnMetadata timestamp = getColumn(UTF8Type.instance.decompose("timestamp")); + final ColumnMetadata age = getColumn(UTF8Type.instance.decompose("age")); + + Operation.Node node = new Operation.ExpressionNode(new SimpleExpression(age, Operator.EQ, Int32Type.instance.decompose(5))); + FilterTree filterTree = node.buildFilter(controller, true); + + DecoratedKey key = buildKey("0"); + Unfiltered row = buildRow(buildCell(age, instance.decompose(6), System.currentTimeMillis())); + Row staticRow = buildRow(Clustering.STATIC_CLUSTERING); + + assertFalse(filterTree.isSatisfiedBy(key, (Row) row, staticRow)); + + row = buildRow(buildCell(age, instance.decompose(5), System.currentTimeMillis())); + + assertTrue(filterTree.isSatisfiedBy(key, (Row) row, staticRow)); + + row = buildRow(buildCell(age, instance.decompose(6), System.currentTimeMillis())); + + assertFalse(filterTree.isSatisfiedBy(key, (Row) row, staticRow)); + + // range with exclusions - age > 1 AND age <= 10 + node = new Operation.AndNode(); + node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.GT, Int32Type.instance.decompose(1)))); + node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.LTE, Int32Type.instance.decompose(10)))); + filterTree = node.buildFilter(controller, true); + + Set<Integer> exclusions = Sets.newHashSet(0, 1, 11); + for (int i = 0; i <= 11; i++) + { + row = buildRow(buildCell(age, instance.decompose(i), System.currentTimeMillis())); + + boolean result = filterTree.isSatisfiedBy(key, (Row) row, staticRow); + assertTrue(exclusions.contains(i) != result); + } + + // now let's test aggregated AND commands + node = new Operation.AndNode(); + + node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.GTE, Int32Type.instance.decompose(0)))); + node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.LT, Int32Type.instance.decompose(10)))); + + filterTree = node.buildFilter(controller, true); + + for (int i = 0; i < 10; i++) + { + row = buildRow(buildCell(age, instance.decompose(i), System.currentTimeMillis())); + + boolean result = filterTree.isSatisfiedBy(key, (Row) row, staticRow); + assertTrue(result); + } + + // multiple analyzed expressions in the Operation timestamp >= 10 AND age = 5 + node = new Operation.AndNode(); + node.add(new Operation.ExpressionNode(new SimpleExpression(timestamp, Operator.GTE, LongType.instance.decompose(10L)))); + node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.EQ, Int32Type.instance.decompose(5)))); + + FilterTree filterTreeStrict = node.buildFilter(controller, true); + FilterTree filterTreeNonStrict = node.buildFilter(controller, false); + + long startTime = System.currentTimeMillis(); + row = buildRow(buildCell(age, instance.decompose(6), startTime), + buildCell(timestamp, LongType.instance.decompose(11L), startTime + 1)); + + assertFalse(filterTreeStrict.isSatisfiedBy(key, (Row) row, staticRow)); + assertTrue(filterTreeNonStrict.isSatisfiedBy(key, (Row) row, staticRow)); // matches on timestamp >= 10 + + row = buildRow(buildCell(age, instance.decompose(5), startTime + 2), + buildCell(timestamp, LongType.instance.decompose(22L), startTime + 3)); + + assertTrue(filterTreeStrict.isSatisfiedBy(key, (Row) row, staticRow)); + assertTrue(filterTreeNonStrict.isSatisfiedBy(key, (Row) row, staticRow)); + + row = buildRow(buildCell(age, instance.decompose(5), startTime + 4), + buildCell(timestamp, LongType.instance.decompose(9L), startTime + 5)); + + assertFalse(filterTreeStrict.isSatisfiedBy(key, (Row) row, staticRow)); + assertTrue(filterTreeNonStrict.isSatisfiedBy(key, (Row) row, staticRow)); // matches on age = 5 + } + + @Test + public void testAnalyzeNotIndexedButDefinedColumn() + { + final ColumnMetadata firstName = getColumn(UTF8Type.instance.decompose("first_name")); + final ColumnMetadata height = getColumn(UTF8Type.instance.decompose("height")); + + // first_name = 'a' AND height > 5 + Map<Expression.IndexOperator, Expression> expressions; + expressions = convert(Operation.buildIndexExpressions(controller, + Arrays.asList(new SimpleExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("a")), + new SimpleExpression(height, Operator.GT, Int32Type.instance.decompose(5))))); + + assertEquals(2, expressions.size()); + + expressions = convert(Operation.buildIndexExpressions(controller, + Arrays.asList(new SimpleExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("a")), + new SimpleExpression(height, Operator.GT, Int32Type.instance.decompose(0)), + new SimpleExpression(height, Operator.EQ, Int32Type.instance.decompose(5))))); + + assertEquals(2, expressions.size()); + + Expression rangeExpression = expressions.get(Expression.IndexOperator.RANGE); + + assertExpression(rangeExpression, Expression.IndexOperator.RANGE, Int32Type.instance.decompose(0), false, Int32Type.instance.decompose(5), true); + + expressions = convert(Operation.buildIndexExpressions(controller, + Arrays.asList(new SimpleExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("a")), + new SimpleExpression(height, Operator.GTE, Int32Type.instance.decompose(0)), + new SimpleExpression(height, Operator.LT, Int32Type.instance.decompose(10))))); + + assertEquals(2, expressions.size()); + + rangeExpression = expressions.get(Expression.IndexOperator.RANGE); + + assertExpression(rangeExpression, Expression.IndexOperator.RANGE, Int32Type.instance.decompose(0), true, Int32Type.instance.decompose(10), false); + } + + @Test + public void testSatisfiedByWithClustering() + { + ColumnMetadata location = getColumn(CLUSTERING_BACKEND, UTF8Type.instance.decompose("location")); + ColumnMetadata age = getColumn(CLUSTERING_BACKEND, UTF8Type.instance.decompose("age")); + ColumnMetadata height = getColumn(CLUSTERING_BACKEND, UTF8Type.instance.decompose("height")); + ColumnMetadata score = getColumn(CLUSTERING_BACKEND, UTF8Type.instance.decompose("score")); + + DecoratedKey key = buildKey(CLUSTERING_BACKEND, "0"); + Row row = buildRow(Clustering.make(UTF8Type.instance.fromString("US"), Int32Type.instance.decompose(27)), + buildCell(height, instance.decompose(182), System.currentTimeMillis()), + buildCell(score, DoubleType.instance.decompose(1.0d), System.currentTimeMillis())); + Row staticRow = buildRow(Clustering.STATIC_CLUSTERING); + + Operation.Node node = new Operation.AndNode(); + node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.EQ, Int32Type.instance.decompose(27)))); + node.add(new Operation.ExpressionNode(new SimpleExpression(height, Operator.EQ, Int32Type.instance.decompose(182)))); + + assertTrue(node.buildFilter(controllerClustering, true).isSatisfiedBy(key, row, staticRow)); + + node = new Operation.AndNode(); + + node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.EQ, Int32Type.instance.decompose(28)))); + node.add(new Operation.ExpressionNode(new SimpleExpression(height, Operator.EQ, Int32Type.instance.decompose(182)))); + + assertFalse(node.buildFilter(controllerClustering, true).isSatisfiedBy(key, row, staticRow)); + + node = new Operation.AndNode(); + node.add(new Operation.ExpressionNode(new SimpleExpression(location, Operator.EQ, UTF8Type.instance.decompose("US")))); + node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.GTE, Int32Type.instance.decompose(27)))); + + assertTrue(node.buildFilter(controllerClustering, true).isSatisfiedBy(key, row, staticRow)); + + node = new Operation.AndNode(); + node.add(new Operation.ExpressionNode(new SimpleExpression(location, Operator.EQ, UTF8Type.instance.decompose("BY")))); + node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.GTE, Int32Type.instance.decompose(28)))); + + assertFalse(node.buildFilter(controllerClustering, true).isSatisfiedBy(key, row, staticRow)); + + node = new Operation.AndNode(); + node.add(new Operation.ExpressionNode(new SimpleExpression(location, Operator.EQ, UTF8Type.instance.decompose("US")))); + node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.LTE, Int32Type.instance.decompose(27)))); + node.add(new Operation.ExpressionNode(new SimpleExpression(height, Operator.GTE, Int32Type.instance.decompose(182)))); + + assertTrue(node.buildFilter(controllerClustering, true).isSatisfiedBy(key, row, staticRow)); + + node = new Operation.AndNode(); + node.add(new Operation.ExpressionNode(new SimpleExpression(location, Operator.EQ, UTF8Type.instance.decompose("US")))); + node.add(new Operation.ExpressionNode(new SimpleExpression(height, Operator.GTE, Int32Type.instance.decompose(182)))); + node.add(new Operation.ExpressionNode(new SimpleExpression(score, Operator.EQ, DoubleType.instance.decompose(1.0d)))); + + assertTrue(node.buildFilter(controllerClustering, true).isSatisfiedBy(key, row, staticRow)); + + node = new Operation.AndNode(); + node.add(new Operation.ExpressionNode(new SimpleExpression(height, Operator.GTE, Int32Type.instance.decompose(182)))); + node.add(new Operation.ExpressionNode(new SimpleExpression(score, Operator.EQ, DoubleType.instance.decompose(1.0d)))); + + assertTrue(node.buildFilter(controllerClustering, true).isSatisfiedBy(key, row, staticRow)); + } + + private Map<Expression.IndexOperator, Expression> convert(Multimap<ColumnMetadata, Expression> expressions) + { + Map<Expression.IndexOperator, Expression> converted = new EnumMap<>(Expression.IndexOperator.class); + for (Expression expression : expressions.values()) + { + Expression column = converted.get(expression.getIndexOperator()); + assert column == null; // sanity check + converted.put(expression.getIndexOperator(), expression); + } + + return converted; + } + + @Test + public void testSatisfiedByWithStatic() + { + final ColumnMetadata sensorType = getColumn(STATIC_BACKEND, UTF8Type.instance.decompose("sensor_type")); + final ColumnMetadata value = getColumn(STATIC_BACKEND, UTF8Type.instance.decompose("value")); + + DecoratedKey key = buildKey(STATIC_BACKEND, 0); + Row row = buildRow(Clustering.make(UTF8Type.instance.fromString("date"), LongType.instance.decompose(20160401L)), + buildCell(value, DoubleType.instance.decompose(24.56), System.currentTimeMillis())); + Row staticRow = buildRow(Clustering.STATIC_CLUSTERING, + buildCell(sensorType, UTF8Type.instance.decompose("TEMPERATURE"), System.currentTimeMillis())); + + // sensor_type ='TEMPERATURE' AND value = 24.56 + Operation.Node node = new Operation.AndNode(); + node.add(new Operation.ExpressionNode(new SimpleExpression(sensorType, Operator.EQ, UTF8Type.instance.decompose("TEMPERATURE")))); + node.add(new Operation.ExpressionNode(new SimpleExpression(value, Operator.EQ, DoubleType.instance.decompose(24.56)))); + + assertTrue(node.buildFilter(controllerStatic, true).isSatisfiedBy(key, row, staticRow)); + + // sensor_type ='TEMPERATURE' AND value = 30 + node = new Operation.AndNode(); + node.add(new Operation.ExpressionNode(new SimpleExpression(sensorType, Operator.EQ, UTF8Type.instance.decompose("TEMPERATURE")))); + node.add(new Operation.ExpressionNode(new SimpleExpression(value, Operator.EQ, DoubleType.instance.decompose(30.00)))); + + assertFalse(node.buildFilter(controllerStatic, true).isSatisfiedBy(key, row, staticRow)); + } + + public static TableMetadata.Builder skinnySAITableMetadata(String keyspace, String table) + { + TableMetadata.Builder builder = + TableMetadata.builder(keyspace, table) + .addPartitionKeyColumn("id", UTF8Type.instance) + .addRegularColumn("first_name", UTF8Type.instance) + .addRegularColumn("last_name", UTF8Type.instance) + .addRegularColumn("age", Int32Type.instance) + .addRegularColumn("height", Int32Type.instance) + .addRegularColumn("timestamp", LongType.instance) + .addRegularColumn("address", UTF8Type.instance) + .addRegularColumn("score", DoubleType.instance); + + Indexes.Builder indexes = Indexes.builder(); + addIndex(indexes, table, "first_name"); + addIndex(indexes, table, "last_name"); + addIndex(indexes, table, "age"); + addIndex(indexes, table, "timestamp"); + addIndex(indexes, table, "address"); + addIndex(indexes, table, "score"); + + return builder.indexes(indexes.build()); + } + + public static TableMetadata.Builder clusteringSAITableMetadata(String keyspace, String table) + { + return clusteringSAITableMetadata(keyspace, table, "location", "age", "height", "score"); + } + + public static TableMetadata.Builder clusteringSAITableMetadata(String keyspace, String table, String...indexedColumns) + { + Indexes.Builder indexes = Indexes.builder(); + for (String indexedColumn : indexedColumns) + { + addIndex(indexes, table, indexedColumn); + } + + return TableMetadata.builder(keyspace, table) + .addPartitionKeyColumn("name", UTF8Type.instance) + .addClusteringColumn("location", UTF8Type.instance) + .addClusteringColumn("age", Int32Type.instance) + .addRegularColumn("height", Int32Type.instance) + .addRegularColumn("score", DoubleType.instance) + .indexes(indexes.build()); + } + + public static TableMetadata.Builder staticSAITableMetadata(String keyspace, String table) + { + TableMetadata.Builder builder = + TableMetadata.builder(keyspace, table) + .addPartitionKeyColumn("sensor_id", Int32Type.instance) + .addStaticColumn("sensor_type", UTF8Type.instance) + .addClusteringColumn("date", LongType.instance) + .addRegularColumn("value", DoubleType.instance) + .addRegularColumn("variance", Int32Type.instance); + + Indexes.Builder indexes = Indexes.builder(); + + addIndex(indexes, table, "sensor_type"); + addIndex(indexes, table, "value"); + addIndex(indexes, table, "variance"); + + return builder.indexes(indexes.build()); + } + + private void assertExpression(Expression expression, Expression.IndexOperator indexOperator, ByteBuffer lower, + boolean lowerInclusive, ByteBuffer upper, boolean upperInclusive) + { + assertEquals(indexOperator, expression.getIndexOperator()); + assertEquals(lower, expression.lower().value.raw); + assertEquals(lowerInclusive, expression.lower().inclusive); + assertEquals(upper, expression.upper().value.raw); + assertEquals(upperInclusive, expression.upper().inclusive); + } + + private static void addIndex(Indexes.Builder indexes, String table, String column) + { + String indexName = table + '_' + column; + indexes.add(IndexMetadata.fromSchemaMetadata(indexName, IndexMetadata.Kind.CUSTOM, new HashMap<String, String>() + {{ + put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName()); + put(IndexTarget.TARGET_OPTION_NAME, column); + }})); + } + + private static DecoratedKey buildKey(Object... key) + { + return buildKey(BACKEND, key); + } + + private static DecoratedKey buildKey(ColumnFamilyStore cfs, Object... key) + { + AbstractType<?> type = cfs.metadata().partitionKeyType; + ByteBuffer decomposed; + if(type instanceof CompositeType) + { + Preconditions.checkArgument(key.length == type.subTypes().size()); + decomposed = ((CompositeType) type).decompose(key); + } + else + { + Preconditions.checkArgument(key.length == 1); + decomposed = ((AbstractType) type).decompose(key[0]); + } + return Murmur3Partitioner.instance.decorateKey(decomposed); + } + + private static Unfiltered buildRow(Cell<?>... cells) + { + return buildRow(Clustering.EMPTY, cells); + } + + private static Row buildRow(Clustering<?> clustering, Cell<?>... cells) + { + Row.Builder rowBuilder = BTreeRow.sortedBuilder(); + rowBuilder.newRow(clustering); + for (Cell<?> c : cells) + rowBuilder.addCell(c); + + return rowBuilder.build(); + } + + private static Cell<?> buildCell(ColumnMetadata column, ByteBuffer value, long timestamp) + { + return BufferCell.live(column, timestamp, value); + } + + private static ColumnMetadata getColumn(ByteBuffer name) + { + return getColumn(BACKEND, name); + } + + private static ColumnMetadata getColumn(ColumnFamilyStore cfs, ByteBuffer name) + { + return cfs.metadata().getColumn(name); + } + + private static class SimpleExpression extends RowFilter.Expression + { + SimpleExpression(ColumnMetadata column, Operator operator, ByteBuffer value) + { + super(column, operator, value); + } + + @Override + public Kind kind() + { + return Kind.SIMPLE; + } + + @Override - public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row) ++ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row, long nowInSec) + { + throw new UnsupportedOperationException(); + } + + @Override + protected String toString(boolean cql) + { + AbstractType<?> type = column.type; + switch (operator) + { + case CONTAINS: + assert type instanceof CollectionType; + CollectionType<?> ct = (CollectionType<?>)type; + type = ct.kind == CollectionType.Kind.SET ? ct.nameComparator() : ct.valueComparator(); + break; + case CONTAINS_KEY: + assert type instanceof MapType; + type = ((MapType<?, ?>)type).nameComparator(); + break; + case IN: + type = ListType.getInstance(type, false); + break; + default: + break; + } + return cql + ? String.format("%s %s %s", column.name.toCQLString(), operator, type.toCQLString(value) ) + : String.format("%s %s %s", column.name.toString(), operator, type.getString(value)); + } + } +} diff --cc test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java index 0cef902be0,1d732aa59b..58a8243a4c --- a/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java +++ b/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java @@@ -650,7 -648,7 +650,7 @@@ public class OperationTest extends Sche } @Override - public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row) - public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row, int nowInSec) ++ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row, long nowInSec) { throw new UnsupportedOperationException(); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
