This is an automated email from the ASF dual-hosted git repository. adelapena pushed a commit to branch cassandra-3.11 in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 0541c5199690757294c40d9adfd1b86419158675 Merge: a311109 865b67b Author: Andrés de la Peña <[email protected]> AuthorDate: Mon Mar 1 12:33:02 2021 +0000 Merge branch 'cassandra-3.0' into cassandra-3.11 CHANGES.txt | 1 + build.xml | 2 +- .../apache/cassandra/db/filter/ColumnFilter.java | 84 ++-- src/java/org/apache/cassandra/gms/Gossiper.java | 6 + .../test/ReadDigestConsistencyTest.java | 113 +++++ .../distributed/upgrade/MixedModeReadTest.java | 65 +-- .../cassandra/db/filter/ColumnFilterTest.java | 521 ++++++++++++++++----- 7 files changed, 578 insertions(+), 214 deletions(-) diff --cc CHANGES.txt index ba1b42c,c174cd3..6bd40ad --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,5 -1,5 +1,6 @@@ -3.0.25: +3.11.11 +Merged from 3.0: + * Fix ColumnFilter behaviour to prevent digest mitmatches during upgrades (CASSANDRA-16415) * Update debian packaging for python3 (CASSANDRA-16396) * Avoid pushing schema mutations when setting up distributed system keyspaces locally (CASSANDRA-16387) Merged from 2.2: diff --cc build.xml index 9e0bb73,939ccd8..326a1af --- a/build.xml +++ b/build.xml @@@ -422,7 -396,7 +422,7 @@@ <dependency groupId="org.apache.thrift" artifactId="libthrift" version="0.9.2"> <exclusion groupId="commons-logging" artifactId="commons-logging"/> </dependency> -- <dependency groupId="junit" artifactId="junit" version="4.6" /> ++ <dependency groupId="junit" artifactId="junit" version="4.12" /> <dependency groupId="org.mockito" artifactId="mockito-core" version="3.2.4" /> <dependency groupId="org.apache.cassandra" artifactId="dtest-api" version="0.0.7" /> <dependency groupId="org.reflections" artifactId="reflections" version="0.9.12" /> diff --cc src/java/org/apache/cassandra/db/filter/ColumnFilter.java index e18717c,520c43c..f405431 --- a/src/java/org/apache/cassandra/db/filter/ColumnFilter.java +++ b/src/java/org/apache/cassandra/db/filter/ColumnFilter.java @@@ -23,6 -23,6 +23,9 @@@ import java.util.* import com.google.common.collect.SortedSetMultimap; import com.google.common.collect.TreeMultimap; ++import org.slf4j.Logger; ++import org.slf4j.LoggerFactory; ++ import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.db.*; @@@ -62,16 -49,20 +65,18 @@@ import org.apache.cassandra.net.Messagi */ public class ColumnFilter { ++ private final static Logger logger = LoggerFactory.getLogger(ColumnFilter.class); ++ public static final Serializer serializer = new Serializer(); - // Distinguish between the 2 cases described above: if 'isFetchAll' is true, then all columns will be retrieved - // by the query, but the values for column/cells not selected by 'queried' and 'subSelections' will be skipped. - // Otherwise, only the column/cells returned by 'queried' and 'subSelections' will be returned at all. + // True if _fetched_ is all the columns, in which case metadata must not be null. If false, + // then _fetched_ == _queried_ and we only store _queried_. private final boolean isFetchAll; - private final PartitionColumns queried; // can be null if isFetchAll and we don't want to skip any value private final PartitionColumns fetched; + private final PartitionColumns queried; // can be null if isFetchAll and _fetched_ == _queried_ private final SortedSetMultimap<ColumnIdentifier, ColumnSubselection> subSelections; // can be null - /** - * Used on replica for deserialisation - */ private ColumnFilter(boolean isFetchAll, PartitionColumns fetched, PartitionColumns queried, @@@ -111,7 -102,7 +116,19 @@@ */ public static ColumnFilter selection(CFMetaData metadata, PartitionColumns queried) { -- return new ColumnFilter(true, metadata.partitionColumns(), queried, null); ++ // When fetchAll is enabled on pre CASSANDRA-10657 (3.4-), queried columns are not considered at all, and it ++ // is assumed that all columns are queried. CASSANDRA-10657 (3.4+) brings back skipping values of columns ++ // which are not in queried set when fetchAll is enabled. That makes exactly the same filter being ++ // interpreted in a different way on 3.4- and 3.4+. ++ // ++ // Moreover, there is no way to convert the filter with fetchAll and queried != null so that it is ++ // interpreted the same way on 3.4- because that Cassandra version does not support such filtering. ++ // ++ // In order to avoid inconsitencies in data read by 3.4- and 3.4+ we need to avoid creation of incompatible ++ // filters when the cluster contains 3.4- nodes. We do that by forcibly setting queried to null. ++ // ++ // see CASSANDRA-10657, CASSANDRA-15833, CASSANDRA-16415 ++ return new ColumnFilter(true, metadata.partitionColumns(), Gossiper.instance.isAnyNodeOn30() ? null : queried, null); } /** @@@ -357,11 -280,7 +374,14 @@@ s.put(subSelection.column().name, subSelection); } - // see CASSANDRA-15833 - if (isFetchAll && Gossiper.instance.isAnyNodeOn30()) - return new ColumnFilter(isFetchAll, isFetchAll ? metadata.partitionColumns() : selectedColumns, selectedColumns, s); ++ // See the comment in {@link ColumnFilter#selection(CFMetaData, PartitionColumns)} ++ if (isFetchAll && queried != null && Gossiper.instance.isAnyNodeOn30()) ++ { ++ logger.trace("Column filter will be automatically converted to query all columns because 3.0 nodes are present in the cluster"); + queried = null; ++ } + + return new ColumnFilter(isFetchAll, isFetchAll ? metadata.partitionColumns() : null, queried, s); } } @@@ -385,44 -305,27 +405,29 @@@ @Override public String toString() { + String prefix = ""; ++ if (isFetchAll && queried == null) ++ return "*/*"; + if (isFetchAll) - return "*"; - return "*/*"; ++ prefix = "*/"; if (queried.isEmpty()) - return ""; - - Iterator<ColumnDefinition> defs = queried.selectOrderIterator(); - if (!defs.hasNext()) - return "<none>"; - - StringBuilder sb = new StringBuilder(); - while (defs.hasNext()) - { - appendColumnDef(sb, defs.next()); - if (defs.hasNext()) - sb.append(", "); - } - return sb.toString(); - } + return prefix + "[]"; - private void appendColumnDef(StringBuilder sb, ColumnDefinition column) - { - if (subSelections == null) + StringJoiner joiner = new StringJoiner(", ", "[", "]"); + Iterator<ColumnDefinition> it = queried.selectOrderIterator(); + while (it.hasNext()) { - sb.append(column.name); - return; - } + ColumnDefinition column = it.next(); + SortedSet<ColumnSubselection> s = subSelections != null ? subSelections.get(column.name) : Collections.emptySortedSet(); - SortedSet<ColumnSubselection> s = subSelections.get(column.name); - if (s.isEmpty()) - { - sb.append(column.name); - return; + if (s.isEmpty()) + joiner.add(String.valueOf(column.name)); + else - s.forEach(subSel -> joiner.add(String.format("%s%s", column.name, subSel))); ++ s.forEach(subSel -> joiner.add(String.format("%s%s", column.name , subSel))); } - - int i = 0; - for (ColumnSubselection subSel : s) - sb.append(i++ == 0 ? "" : ", ").append(column.name).append(subSel); + return prefix + joiner.toString(); } public static class Serializer @@@ -505,11 -408,7 +510,18 @@@ } } - // See CASSANDRA-15833 - if (version <= MessagingService.VERSION_3014 && isFetchAll) - return new ColumnFilter(isFetchAll, fetched, selection, subSelections); ++ // Since nodes with and without CASSANDRA-10657 are not distinguishable by messaging version, we need to ++ // check whether there are any nodes running pre CASSANDRA-10657 Cassandra and apply conversion to the ++ // column filter so that it is interpreted in the same way as on the nodes without CASSANDRA-10657. ++ // ++ // See the comment in {@link ColumnFilter#selection(CFMetaData, PartitionColumns)} ++ if (isFetchAll && queried != null && Gossiper.instance.isAnyNodeOn30()) ++ { ++ logger.trace("Deserialized column filter will be automatically converted to query all columns because 3.0 nodes are present in the cluster"); + queried = null; ++ } + + return new ColumnFilter(isFetchAll, fetched, queried, subSelections); } public long serializedSize(ColumnFilter selection, int version) diff --cc src/java/org/apache/cassandra/gms/Gossiper.java index 0e46767,b09d9e1..819078d --- a/src/java/org/apache/cassandra/gms/Gossiper.java +++ b/src/java/org/apache/cassandra/gms/Gossiper.java @@@ -1833,4 -1673,4 +1833,10 @@@ public class Gossiper implements IFailu stop(); ExecutorUtils.shutdownAndWait(timeout, unit, executor); } ++ ++ @VisibleForTesting ++ public void setAnyNodeOn30(boolean anyNodeOn30) ++ { ++ this.anyNodeOn30 = anyNodeOn30; ++ } } diff --cc test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadTest.java index 249e1b8,0000000..d908cd5 mode 100644,000000..100644 --- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadTest.java +++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadTest.java @@@ -1,102 -1,0 +1,65 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.upgrade; + - import java.util.UUID; - - import org.junit.Assert; +import org.junit.Test; + - import org.apache.cassandra.distributed.UpgradeableCluster; - import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.Feature; - import org.apache.cassandra.distributed.impl.DelegatingInvokableInstance; - import org.apache.cassandra.distributed.shared.DistributedTestBase; ++import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.shared.Versions; +import org.apache.cassandra.gms.Gossiper; + ++import static org.apache.cassandra.distributed.test.ReadDigestConsistencyTest.CREATE_TABLE; ++import static org.apache.cassandra.distributed.test.ReadDigestConsistencyTest.insertData; ++import static org.apache.cassandra.distributed.test.ReadDigestConsistencyTest.testDigestConsistency; ++ +public class MixedModeReadTest extends UpgradeTestBase +{ - public static final String TABLE_NAME = "tbl"; - public static final String CREATE_TABLE = String.format( - "CREATE TABLE %s.%s (key int, c1 text, c2 text, c3 text, PRIMARY KEY (key))", - DistributedTestBase.KEYSPACE, TABLE_NAME); - - public static final String INSERT = String.format( - "INSERT INTO %s.%s (key, c1, c2, c3) VALUES (?, ?, ?, ?)", - DistributedTestBase.KEYSPACE, TABLE_NAME); - - public static final String SELECT_C1 = String.format("SELECT key, c1 FROM %s.%s WHERE key = ?", - DistributedTestBase.KEYSPACE, TABLE_NAME); - public static final String SELECT_TRACE = "SELECT activity FROM system_traces.events where session_id = ? and source = ? ALLOW FILTERING;"; - + @Test + public void mixedModeReadColumnSubsetDigestCheck() throws Throwable + { + new TestCase() + .nodes(2) + .nodesToUpgrade(1) + .upgrade(Versions.Major.v30, Versions.Major.v3X) + .withConfig(config -> config.with(Feature.GOSSIP, Feature.NETWORK)) + .setup(cluster -> { + cluster.schemaChange(CREATE_TABLE); - cluster.coordinator(1).execute(INSERT, ConsistencyLevel.ALL, 1, "foo", "bar", "baz"); - cluster.coordinator(1).execute(INSERT, ConsistencyLevel.ALL, 2, "foo", "bar", "baz"); - - // baseline to show no digest mismatches before upgrade - checkTraceForDigestMismatch(cluster, 1); - checkTraceForDigestMismatch(cluster, 2); ++ insertData(cluster.coordinator(1)); ++ testDigestConsistency(cluster.coordinator(1)); ++ testDigestConsistency(cluster.coordinator(2)); + }) - .runAfterNodeUpgrade((cluster, node) -> { - if (node != 1) - return; // shouldn't happen but guard for future test changes - - ++ .runAfterClusterUpgrade(cluster -> { + // we need to let gossip settle or the test will fail + int attempts = 1; - while (!((DelegatingInvokableInstance) (cluster.get(1))).delegate().callOnInstance(() -> Gossiper.instance.isAnyNodeOn30())) ++ //noinspection Convert2MethodRef ++ while (!((IInvokableInstance) (cluster.get(1))).callOnInstance(() -> Gossiper.instance.isAnyNodeOn30())) + { - if (attempts > 30) ++ if (attempts++ > 30) + throw new RuntimeException("Gossiper.instance.isAnyNodeOn30() continually returns false despite expecting to be true"); + Thread.sleep(1000); + } + + // should not cause a disgest mismatch in mixed mode - checkTraceForDigestMismatch(cluster, 1); - checkTraceForDigestMismatch(cluster, 2); ++ testDigestConsistency(cluster.coordinator(1)); ++ testDigestConsistency(cluster.coordinator(2)); + }) + .run(); + } - - private void checkTraceForDigestMismatch(UpgradeableCluster cluster, int coordinatorNode) - { - UUID sessionId = UUID.randomUUID(); - cluster.coordinator(coordinatorNode).executeWithTracing(sessionId, SELECT_C1, ConsistencyLevel.ALL, 1); - Object[][] results = cluster.coordinator(coordinatorNode) - .execute(SELECT_TRACE, ConsistencyLevel.ALL, - sessionId, cluster.get(coordinatorNode).broadcastAddress().getAddress()); - for (Object[] result : results) - { - String activity = (String) result[0]; - Assert.assertFalse("Found Digest Mismatch", activity.toLowerCase().contains("mismatch for key")); - } - } - - +} diff --cc test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java index 5bbd2f5,80c1e42..1ddf039 --- a/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java +++ b/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java @@@ -18,16 -18,20 +18,28 @@@ package org.apache.cassandra.db.filter; + import java.io.IOException; ++import java.util.Arrays; ++import java.util.Collection; + import java.util.function.Consumer; + + import com.google.common.base.Throwables; + import org.junit.Assert; ++import org.junit.Before; ++import org.junit.BeforeClass; import org.junit.Test; ++import org.junit.runner.RunWith; ++import org.junit.runners.Parameterized; - import junit.framework.Assert; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; ++import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.PartitionColumns; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.SetType; import org.apache.cassandra.db.rows.CellPath; import org.apache.cassandra.dht.Murmur3Partitioner; ++import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputBuffer; @@@ -36,143 -40,337 +48,400 @@@ import org.apache.cassandra.utils.ByteB import static org.junit.Assert.assertEquals; ++@RunWith(Parameterized.class) public class ColumnFilterTest { private static final ColumnFilter.Serializer serializer = new ColumnFilter.Serializer(); + private final CFMetaData metadata = CFMetaData.Builder.create("ks", "table") + .withPartitioner(Murmur3Partitioner.instance) + .addPartitionKey("pk", Int32Type.instance) + .addClusteringColumn("ck", Int32Type.instance) + .addStaticColumn("s1", Int32Type.instance) + .addStaticColumn("s2", SetType.getInstance(Int32Type.instance, true)) + .addRegularColumn("v1", Int32Type.instance) + .addRegularColumn("v2", SetType.getInstance(Int32Type.instance, true)) + .build(); + + private final ColumnDefinition s1 = metadata.getColumnDefinition(ByteBufferUtil.bytes("s1")); + private final ColumnDefinition s2 = metadata.getColumnDefinition(ByteBufferUtil.bytes("s2")); + private final ColumnDefinition v1 = metadata.getColumnDefinition(ByteBufferUtil.bytes("v1")); + private final ColumnDefinition v2 = metadata.getColumnDefinition(ByteBufferUtil.bytes("v2")); + private final CellPath path0 = CellPath.create(ByteBufferUtil.bytes(0)); + private final CellPath path1 = CellPath.create(ByteBufferUtil.bytes(1)); + private final CellPath path2 = CellPath.create(ByteBufferUtil.bytes(2)); + private final CellPath path3 = CellPath.create(ByteBufferUtil.bytes(3)); + private final CellPath path4 = CellPath.create(ByteBufferUtil.bytes(4)); + ++ @Parameterized.Parameter ++ public boolean anyNodeOn30; ++ ++ @Parameterized.Parameters(name = "{index}: anyNodeOn30={0}") ++ public static Collection<Object[]> data() ++ { ++ return Arrays.asList(new Object[]{ true }, new Object[]{ false }); ++ } ++ ++ @BeforeClass ++ public static void beforeClass() ++ { ++ DatabaseDescriptor.clientInitialization(); ++ } ++ ++ @Before ++ public void before() ++ { ++ Gossiper.instance.setAnyNodeOn30(anyNodeOn30); ++ } ++ + // Select all + @Test - public void columnFilterSerialisationRoundTrip() throws Exception - { - CFMetaData metadata = CFMetaData.Builder.create("ks", "table") - .withPartitioner(Murmur3Partitioner.instance) - .addPartitionKey("pk", Int32Type.instance) - .addClusteringColumn("ck", Int32Type.instance) - .addRegularColumn("v1", Int32Type.instance) - .addRegularColumn("v2", Int32Type.instance) - .addRegularColumn("v3", Int32Type.instance) - .build(); - - ColumnDefinition v1 = metadata.getColumnDefinition(ByteBufferUtil.bytes("v1")); - - testRoundTrip(ColumnFilter.all(metadata), metadata, MessagingService.VERSION_30); - testRoundTrip(ColumnFilter.all(metadata), metadata, MessagingService.VERSION_3014); - - testRoundTrip(ColumnFilter.selection(metadata.partitionColumns().without(v1)), metadata, MessagingService.VERSION_30); - testRoundTrip(ColumnFilter.selection(metadata.partitionColumns().without(v1)), metadata, MessagingService.VERSION_3014); - } - - private static void testRoundTrip(ColumnFilter columnFilter, CFMetaData metadata, int version) throws Exception - { - DataOutputBuffer output = new DataOutputBuffer(); - serializer.serialize(columnFilter, output, version); - Assert.assertEquals(serializer.serializedSize(columnFilter, version), output.position()); - DataInputPlus input = new DataInputBuffer(output.buffer(), false); - Assert.assertEquals(serializer.deserialize(input, version, metadata), columnFilter); - } - - /** - * Tests whether a filter fetches and/or queries columns and cells. - */ - @Test - public void testFetchedQueried() - { - CFMetaData metadata = CFMetaData.Builder.create("ks", "table") - .withPartitioner(Murmur3Partitioner.instance) - .addPartitionKey("k", Int32Type.instance) - .addRegularColumn("simple", Int32Type.instance) - .addRegularColumn("complex", SetType.getInstance(Int32Type.instance, true)) - .build(); - - ColumnDefinition simple = metadata.getColumnDefinition(ByteBufferUtil.bytes("simple")); - ColumnDefinition complex = metadata.getColumnDefinition(ByteBufferUtil.bytes("complex")); - CellPath path1 = CellPath.create(ByteBufferUtil.bytes(1)); - CellPath path2 = CellPath.create(ByteBufferUtil.bytes(2)); - ColumnFilter filter; - - // select only the simple column, without table metadata - filter = ColumnFilter.selection(PartitionColumns.builder().add(simple).build()); - assertFetchedQueried(true, true, filter, simple); - assertFetchedQueried(false, false, filter, complex); - assertFetchedQueried(false, false, filter, complex, path1); - assertFetchedQueried(false, false, filter, complex, path2); - - // select only the complex column, without table metadata - filter = ColumnFilter.selection(PartitionColumns.builder().add(complex).build()); - assertFetchedQueried(false, false, filter, simple); - assertFetchedQueried(true, true, filter, complex); - assertFetchedQueried(true, true, filter, complex, path1); - assertFetchedQueried(true, true, filter, complex, path2); - - // select both the simple and complex columns, without table metadata - filter = ColumnFilter.selection(PartitionColumns.builder().add(simple).add(complex).build()); - assertFetchedQueried(true, true, filter, simple); - assertFetchedQueried(true, true, filter, complex); - assertFetchedQueried(true, true, filter, complex, path1); - assertFetchedQueried(true, true, filter, complex, path2); - - // select only the simple column, with table metadata - filter = ColumnFilter.selection(metadata, PartitionColumns.builder().add(simple).build()); - assertFetchedQueried(true, true, filter, simple); - assertFetchedQueried(true, false, filter, complex); - assertFetchedQueried(true, false, filter, complex, path1); - assertFetchedQueried(true, false, filter, complex, path2); - - // select only the complex column, with table metadata - filter = ColumnFilter.selection(metadata, PartitionColumns.builder().add(complex).build()); - assertFetchedQueried(true, false, filter, simple); - assertFetchedQueried(true, true, filter, complex); - assertFetchedQueried(true, true, filter, complex, path1); - assertFetchedQueried(true, true, filter, complex, path2); - - // select both the simple and complex columns, with table metadata - filter = ColumnFilter.selection(metadata, PartitionColumns.builder().add(simple).add(complex).build()); - assertFetchedQueried(true, true, filter, simple); - assertFetchedQueried(true, true, filter, complex); - assertFetchedQueried(true, true, filter, complex, path1); - assertFetchedQueried(true, true, filter, complex, path2); - - // select only the simple column, with selection builder - filter = ColumnFilter.selectionBuilder().add(simple).build(); - assertFetchedQueried(true, true, filter, simple); - assertFetchedQueried(false, false, filter, complex); - assertFetchedQueried(false, false, filter, complex, path1); - assertFetchedQueried(false, false, filter, complex, path2); - - // select only a cell of the complex column, with selection builder - filter = ColumnFilter.selectionBuilder().select(complex, path1).build(); - assertFetchedQueried(false, false, filter, simple); - assertFetchedQueried(true, true, filter, complex); - assertFetchedQueried(true, true, filter, complex, path1); - assertFetchedQueried(true, false, filter, complex, path2); - - // select both the simple column and a cell of the complex column, with selection builder - filter = ColumnFilter.selectionBuilder().add(simple).select(complex, path1).build(); - assertFetchedQueried(true, true, filter, simple); - assertFetchedQueried(true, true, filter, complex); - assertFetchedQueried(true, true, filter, complex, path1); - assertFetchedQueried(true, false, filter, complex, path2); + public void testSelectAll() + { + Consumer<ColumnFilter> check = filter -> { + testRoundTrips(filter); + assertEquals("*/*", filter.toString()); + assertFetchedQueried(true, true, filter, v1, v2, s1, s2); + assertCellFetchedQueried(true, true, filter, v2, path0, path1, path2, path3, path4); + assertCellFetchedQueried(true, true, filter, s2, path0, path1, path2, path3, path4); + }; + + check.accept(ColumnFilter.all(metadata)); + check.accept(ColumnFilter.allColumnsBuilder(metadata).build()); } - private static void assertFetchedQueried(boolean expectedFetched, - boolean expectedQueried, - ColumnFilter filter, - ColumnDefinition column) + // Selections + + @Test + public void testSelectNothing() { - assert !expectedQueried || expectedFetched; - boolean actualFetched = filter.fetches(column); - assertEquals(expectedFetched, actualFetched); - assertEquals(expectedQueried, actualFetched && filter.fetchedColumnIsQueried(column)); + Consumer<ColumnFilter> check = filter -> { + testRoundTrips(filter); + assertEquals("[]", filter.toString()); + assertFetchedQueried(false, false, filter, v1, v2, s1, s2); + assertCellFetchedQueried(false, false, filter, v2, path0, path1, path2, path3, path4); + assertCellFetchedQueried(false, false, filter, s2, path0, path1, path2, path3, path4); + }; + + check.accept(ColumnFilter.selection(PartitionColumns.NONE)); + check.accept(ColumnFilter.selectionBuilder().build()); + } + + @Test + public void testSelectSimpleColumn() + { + Consumer<ColumnFilter> check = filter -> { + testRoundTrips(filter); + assertEquals("[v1]", filter.toString()); + assertFetchedQueried(true, true, filter, v1); + assertFetchedQueried(false, false, filter, v2, s1, s2); + assertCellFetchedQueried(false, false, filter, v2, path0, path1, path2, path3, path4); + assertCellFetchedQueried(false, false, filter, s2, path0, path1, path2, path3, path4); + }; + + check.accept(ColumnFilter.selection(PartitionColumns.builder().add(v1).build())); + check.accept(ColumnFilter.selectionBuilder().add(v1).build()); + } + + @Test + public void testSelectComplexColumn() + { + Consumer<ColumnFilter> check = filter -> { + testRoundTrips(filter); + assertEquals("[v2]", filter.toString()); + assertFetchedQueried(true, true, filter, v2); + assertFetchedQueried(false, false, filter, v1, s1, s2); + assertCellFetchedQueried(true, true, filter, v2, path0, path1, path2, path3, path4); + assertCellFetchedQueried(false, false, filter, s2, path0, path1, path2, path3, path4); + }; + + check.accept(ColumnFilter.selection(PartitionColumns.builder().add(v2).build())); + check.accept(ColumnFilter.selectionBuilder().add(v2).build()); + } + + @Test + public void testSelectStaticColumn() + { + Consumer<ColumnFilter> check = filter -> { + testRoundTrips(filter); + assertEquals("[s1]", filter.toString()); + assertFetchedQueried(true, true, filter, s1); + assertFetchedQueried(false, false, filter, v1, v2, s2); + assertCellFetchedQueried(false, false, filter, v2, path0, path1, path2, path3, path4); + assertCellFetchedQueried(false, false, filter, s2, path0, path1, path2, path3, path4); + }; + + check.accept(ColumnFilter.selection(PartitionColumns.builder().add(s1).build())); + check.accept(ColumnFilter.selectionBuilder().add(s1).build()); + } + + @Test + public void testSelectStaticComplexColumn() + { + Consumer<ColumnFilter> check = filter -> { + testRoundTrips(filter); + assertEquals("[s2]", filter.toString()); + assertFetchedQueried(true, true, filter, s2); + assertFetchedQueried(false, false, filter, v1, v2, s1); + assertCellFetchedQueried(false, false, filter, v2, path0, path1, path2, path3, path4); + assertCellFetchedQueried(true, true, filter, s2, path0, path1, path2, path3, path4); + }; + + check.accept(ColumnFilter.selection(PartitionColumns.builder().add(s2).build())); + check.accept(ColumnFilter.selectionBuilder().add(s2).build()); + } + + @Test + public void testSelectColumns() + { + Consumer<ColumnFilter> check = filter -> { + testRoundTrips(filter); + assertEquals("[s1, s2, v1, v2]", filter.toString()); + assertFetchedQueried(true, true, filter, v1, v2, s1, s2); + assertCellFetchedQueried(true, true, filter, v2, path0, path1, path2, path3, path4); + assertCellFetchedQueried(true, true, filter, s2, path0, path1, path2, path3, path4); + }; + + check.accept(ColumnFilter.selection(PartitionColumns.builder().add(v1).add(v2).add(s1).add(s2).build())); + check.accept(ColumnFilter.selectionBuilder().add(v1).add(v2).add(s1).add(s2).build()); + } + + @Test + public void testSelectIndividualCells() + { + ColumnFilter filter = ColumnFilter.selectionBuilder().select(v2, path1).select(v2, path3).build(); + testRoundTrips(filter); + assertEquals("[v2[1], v2[3]]", filter.toString()); + assertFetchedQueried(true, true, filter, v2); + assertFetchedQueried(false, false, filter, v1, s1, s2); + assertCellFetchedQueried(true, true, filter, v2, path1, path3); + assertCellFetchedQueried(false, false, filter, v2, path0, path2, path4); + assertCellFetchedQueried(false, false, filter, s2, path0, path1, path2, path3, path4); + } + + @Test + public void testSelectIndividualCellsFromStatic() + { + ColumnFilter filter = ColumnFilter.selectionBuilder().select(s2, path1).select(s2, path3).build(); + testRoundTrips(filter); + assertEquals("[s2[1], s2[3]]", filter.toString()); + assertFetchedQueried(true, true, filter, s2); + assertFetchedQueried(false, false, filter, v1, v2, s1); + assertCellFetchedQueried(false, false, filter, v2, path0, path1, path2, path3, path4); + assertCellFetchedQueried(true, true, filter, s2, path1, path3); + assertCellFetchedQueried(false, false, filter, s2, path0, path2, path4); + } + + @Test + public void testSelectCellSlice() + { + ColumnFilter filter = ColumnFilter.selectionBuilder().slice(v2, path1, path3).build(); + testRoundTrips(filter); + assertEquals("[v2[1:3]]", filter.toString()); + assertFetchedQueried(true, true, filter, v2); + assertFetchedQueried(false, false, filter, v1, s1, s2); + assertCellFetchedQueried(true, true, filter, v2, path1, path2, path3); + assertCellFetchedQueried(false, false, filter, v2, path0, path4); + assertCellFetchedQueried(false, false, filter, s2, path0, path1, path2, path3, path4); + } + + @Test + public void testSelectCellSliceFromStatic() + { + ColumnFilter filter = ColumnFilter.selectionBuilder().slice(s2, path1, path3).build(); + testRoundTrips(filter); + assertEquals("[s2[1:3]]", filter.toString()); + assertFetchedQueried(true, true, filter, s2); + assertFetchedQueried(false, false, filter, v1, v2, s1); + assertCellFetchedQueried(false, false, filter, v2, path0, path1, path2, path3, path4); + assertCellFetchedQueried(true, true, filter, s2, path1, path2, path3); + assertCellFetchedQueried(false, false, filter, s2, path0, path4); + } + + @Test + public void testSelectColumnsWithCellsAndSlices() + { + ColumnFilter filter = ColumnFilter.selectionBuilder() + .add(v1) + .add(s1) + .slice(v2, path0, path2) + .select(v2, path4) + .select(s2, path0) + .slice(s2, path2, path4) + .build(); + testRoundTrips(filter); + assertEquals("[s1, s2[0], s2[2:4], v1, v2[0:2], v2[4]]", filter.toString()); + assertFetchedQueried(true, true, filter, v1, v2, s1, s2); + assertCellFetchedQueried(true, true, filter, v2, path0, path1, path2, path4); + assertCellFetchedQueried(false, false, filter, v2, path3); + assertCellFetchedQueried(true, true, filter, s2, path0, path2, path3, path4); + assertCellFetchedQueried(false, false, filter, s2, path1); + } + + // select with metadata + + @Test + public void testSelectSimpleColumnWithMetadata() + { + Consumer<ColumnFilter> check = filter -> { + testRoundTrips(filter); + assertFetchedQueried(true, true, filter, v1); - - assertEquals("*/*", filter.toString()); - assertFetchedQueried(true, true, filter, s1, s2, v2); - assertCellFetchedQueried(true, true, filter, v2, path0, path1, path2, path3, path4); - assertCellFetchedQueried(true, true, filter, s2, path0, path1, path2, path3, path4); ++ if (anyNodeOn30) ++ { ++ assertEquals("*/*", filter.toString()); ++ assertFetchedQueried(true, true, filter, s1, s2, v2); ++ assertCellFetchedQueried(true, true, filter, v2, path0, path1, path2, path3, path4); ++ assertCellFetchedQueried(true, true, filter, s2, path0, path1, path2, path3, path4); ++ } ++ else ++ { ++ assertEquals("*/[v1]", filter.toString()); ++ assertFetchedQueried(true, false, filter, s1, s2, v2); ++ assertCellFetchedQueried(true, false, filter, v2, path0, path1, path2, path3, path4); ++ assertCellFetchedQueried(true, false, filter, s2, path0, path1, path2, path3, path4); ++ } + }; + + check.accept(ColumnFilter.selection(metadata, PartitionColumns.builder().add(v1).build())); + check.accept(ColumnFilter.allColumnsBuilder(metadata).add(v1).build()); + } + + @Test + public void testSelectStaticColumnWithMetadata() + { + Consumer<ColumnFilter> check = filter -> { + testRoundTrips(filter); + assertFetchedQueried(true, true, filter, s1); - - assertEquals("*/*", filter.toString()); - assertFetchedQueried(true, true, filter, v1, v2, s2); - assertCellFetchedQueried(true, true, filter, v2, path0, path1, path2, path3, path4); - assertCellFetchedQueried(true, true, filter, s2, path0, path1, path2, path3, path4); ++ if (anyNodeOn30) ++ { ++ assertEquals("*/*", filter.toString()); ++ assertFetchedQueried(true, true, filter, v1, v2, s2); ++ assertCellFetchedQueried(true, true, filter, v2, path0, path1, path2, path3, path4); ++ assertCellFetchedQueried(true, true, filter, s2, path0, path1, path2, path3, path4); ++ } ++ else ++ { ++ assertEquals("*/[s1]", filter.toString()); ++ assertFetchedQueried(true, false, filter, v1, v2, s2); ++ assertCellFetchedQueried(true, false, filter, v2, path0, path1, path2, path3, path4); ++ assertCellFetchedQueried(false, false, filter, s2, path0, path1, path2, path3, path4); ++ } + }; + + check.accept(ColumnFilter.selection(metadata, PartitionColumns.builder().add(s1).build())); + check.accept(ColumnFilter.allColumnsBuilder(metadata).add(s1).build()); + } + + @Test + public void testSelectCellWithMetadata() + { + ColumnFilter filter = ColumnFilter.allColumnsBuilder(metadata).select(v2, path1).build(); + testRoundTrips(filter); + assertFetchedQueried(true, true, filter, v2); - - assertEquals("*/*", filter.toString()); - assertFetchedQueried(true, true, filter, s1, s2, v1); - assertCellFetchedQueried(true, true, filter, v2, path1); - assertCellFetchedQueried(true, false, filter, v2, path0, path2, path3, path4); - assertCellFetchedQueried(true, true, filter, s2, path0, path1, path2, path3, path4); ++ if (anyNodeOn30) ++ { ++ assertEquals("*/*", filter.toString()); ++ assertFetchedQueried(true, true, filter, s1, s2, v1); ++ assertCellFetchedQueried(true, true, filter, v2, path1); ++ assertCellFetchedQueried(true, false, filter, v2, path0, path2, path3, path4); ++ assertCellFetchedQueried(true, true, filter, s2, path0, path1, path2, path3, path4); ++ } ++ else ++ { ++ assertEquals("*/[v2[1]]", filter.toString()); ++ assertFetchedQueried(true, false, filter, s1, s2, v1); ++ assertCellFetchedQueried(true, true, filter, v2, path1); ++ assertCellFetchedQueried(true, false, filter, v2, path0, path2, path3, path4); ++ assertCellFetchedQueried(true, false, filter, s2, path0, path1, path2, path3, path4); ++ } + } + + @Test + public void testSelectStaticColumnCellWithMetadata() + { + ColumnFilter filter = ColumnFilter.allColumnsBuilder(metadata).select(s2, path1).build(); + testRoundTrips(filter); + assertFetchedQueried(true, true, filter, s2); - - assertEquals("*/*", filter.toString()); - assertFetchedQueried(true, true, filter, v1, v2, s1); - assertCellFetchedQueried(true, true, filter, v2, path0, path1, path2, path3, path4); - assertCellFetchedQueried(true, true, filter, s2, path1); - assertCellFetchedQueried(true, false, filter, s2, path0, path2, path3, path4); ++ if (anyNodeOn30) ++ { ++ assertEquals("*/*", filter.toString()); ++ assertFetchedQueried(true, true, filter, v1, v2, s1); ++ assertCellFetchedQueried(true, true, filter, v2, path0, path1, path2, path3, path4); ++ assertCellFetchedQueried(true, true, filter, s2, path1); ++ assertCellFetchedQueried(true, false, filter, s2, path0, path2, path3, path4); ++ } ++ else ++ { ++ assertEquals("*/[s2[1]]", filter.toString()); ++ assertFetchedQueried(true, false, filter, v1, v2, s1); ++ assertCellFetchedQueried(true, false, filter, v2, path0, path1, path2, path3, path4); ++ assertCellFetchedQueried(true, true, filter, s2, path1); ++ assertCellFetchedQueried(true, false, filter, s2, path0, path2, path3, path4); ++ } + } + + private void testRoundTrips(ColumnFilter cf) + { + testRoundTrip(cf, MessagingService.VERSION_30); + testRoundTrip(cf, MessagingService.VERSION_3014); + } + + private void testRoundTrip(ColumnFilter columnFilter, int version) + { + try + { + DataOutputBuffer output = new DataOutputBuffer(); + serializer.serialize(columnFilter, output, version); + Assert.assertEquals(serializer.serializedSize(columnFilter, version), output.position()); + DataInputPlus input = new DataInputBuffer(output.buffer(), false); + ColumnFilter deserialized = serializer.deserialize(input, version, metadata); + Assert.assertEquals(deserialized, columnFilter); + } + catch (IOException e) + { + throw Throwables.propagate(e); + } } - private static void assertFetchedQueried(boolean expectedIncluded, - boolean expectedNotSkipped, + private static void assertFetchedQueried(boolean expectedFetched, + boolean expectedQueried, ColumnFilter filter, - ColumnDefinition column, - CellPath path) + ColumnDefinition... columns) + { + for (ColumnDefinition column : columns) + { - assertEquals(String.format("Expected includes(%s) to be %s", column.name, expectedIncluded), - expectedIncluded, filter.includes(column)); - if (expectedIncluded) - assertEquals(String.format("Expected canSkipValue(%s) to be %s", column.name, !expectedNotSkipped), - !expectedNotSkipped, filter.canSkipValue(column)); ++ assertEquals(String.format("Expected fetches(%s) to be %s", column.name, expectedFetched), ++ expectedFetched, filter.fetches(column)); ++ if (expectedFetched) ++ assertEquals(String.format("Expected fetchedColumnIsQueried(%s) to be %s", column.name, expectedQueried), ++ expectedQueried, filter.fetchedColumnIsQueried(column)); + } + } + - private static void assertCellFetchedQueried(boolean expectedIncluded, - boolean expectedNotSkipped, ++ private static void assertCellFetchedQueried(boolean expectedFetched, ++ boolean expectedQueried, + ColumnFilter filter, + ColumnDefinition column, + CellPath... paths) { - assert !expectedQueried || expectedFetched; - boolean actualFetched = filter.fetches(column); - assertEquals(expectedFetched, actualFetched); - assertEquals(expectedQueried, actualFetched && filter.fetchedCellIsQueried(column, path)); + ColumnFilter.Tester tester = filter.newTester(column); + + for (CellPath path : paths) + { + int p = ByteBufferUtil.toInt(path.get(0)); ++ if (expectedFetched) ++ assertEquals(String.format("Expected fetchedCellIsQueried(%s:%s) to be %s", column.name, p, expectedQueried), ++ expectedQueried, filter.fetchedCellIsQueried(column, path)); + + if (tester != null) + { - assertEquals(String.format("Expected tester.includes(%s:%s) to be %s", column.name, p, expectedIncluded), - expectedIncluded, tester.includes(path)); - if (expectedIncluded) - assertEquals(String.format("Expected tester.canSkipValue(%s:%s) to be %s", column.name, p, !expectedNotSkipped), - !expectedNotSkipped, tester.canSkipValue(path)); ++ assertEquals(String.format("Expected tester.fetches(%s:%s) to be %s", column.name, p, expectedFetched), ++ expectedFetched, tester.fetches(path)); ++ if (expectedFetched) ++ assertEquals(String.format("Expected tester.fetchedCellIsQueried(%s:%s) to be %s", column.name, p, expectedQueried), ++ expectedQueried, tester.fetchedCellIsQueried(path)); + } + } } - } + } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
