Merge commit '975c3d81b67e9c1e1dcefdda3f90e8edf6be5efa' into cassandra-3.11
* commit '975c3d81b67e9c1e1dcefdda3f90e8edf6be5efa': Fix sstable reader to support range-tombstone-marker for multi-slices Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/66115139 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/66115139 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/66115139 Branch: refs/heads/trunk Commit: 66115139addfb2bb6e26fa85e4225a1178d2e99c Parents: 594f1c1 975c3d8 Author: Sylvain Lebresne <sylv...@datastax.com> Authored: Wed Sep 20 15:11:58 2017 +0200 Committer: Sylvain Lebresne <sylv...@datastax.com> Committed: Wed Sep 20 15:12:40 2017 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../columniterator/AbstractSSTableIterator.java | 7 - .../db/columniterator/SSTableIterator.java | 8 +- .../columniterator/SSTableReversedIterator.java | 2 +- .../org/apache/cassandra/cql3/ViewTest.java | 49 +++++++ .../db/SinglePartitionSliceCommandTest.java | 145 ++++++++++++++++++- 6 files changed, 197 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/66115139/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 01955f6,2d11a3e..39270e5 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,15 -1,5 +1,16 @@@ -3.0.15 +3.11.1 + * AbstractTokenTreeBuilder#serializedSize returns wrong value when there is a single leaf and overflow collisions (CASSANDRA-13869) + * Add a compaction option to TWCS to ignore sstables overlapping checks (CASSANDRA-13418) + * BTree.Builder memory leak (CASSANDRA-13754) + * Revert CASSANDRA-10368 of supporting non-pk column filtering due to correctness (CASSANDRA-13798) + * Fix cassandra-stress hang issues when an error during cluster connection happens (CASSANDRA-12938) + * Better bootstrap failure message when blocked by (potential) range movement (CASSANDRA-13744) + * "ignore" option is ignored in sstableloader (CASSANDRA-13721) + * Deadlock in AbstractCommitLogSegmentManager (CASSANDRA-13652) + * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512) + * Properly evict pstmts from prepared statements cache (CASSANDRA-13641) +Merged from 3.0: + * Fix sstable reader to support range-tombstone-marker for multi-slices (CASSANDRA-13787) * Fix short read protection for tables with no clustering columns (CASSANDRA-13880) * Make isBuilt volatile in PartitionUpdate (CASSANDRA-13619) * Prevent integer overflow of timestamps in CellTest and RowsTest (CASSANDRA-13866) http://git-wip-us.apache.org/repos/asf/cassandra/blob/66115139/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java index b6c60fe,f9e6545..c15416f --- a/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java +++ b/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java @@@ -371,14 -329,7 +371,7 @@@ public abstract class AbstractSSTableIt openMarker = marker.isOpen(false) ? marker.openDeletionTime(false) : null; } - protected DeletionTime getAndClearOpenMarker() - { - DeletionTime toReturn = openMarker; - openMarker = null; - return toReturn; - } - - public boolean hasNext() + public boolean hasNext() { try { http://git-wip-us.apache.org/repos/asf/cassandra/blob/66115139/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/66115139/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java index 88d415b,76d8c4d..cf8798d --- a/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java +++ b/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java @@@ -249,8 -239,8 +249,8 @@@ public class SSTableReversedIterator ex // not breaking ImmutableBTreePartition, we should skip it when returning from the iterator, hence the // skipFirstIteratedItem (this is the last item of the block, but we're iterating in reverse order so it will // be the first returned by the iterator). - RangeTombstone.Bound markerEnd = end == null ? RangeTombstone.Bound.TOP : RangeTombstone.Bound.fromSliceBound(end); + ClusteringBound markerEnd = end == null ? ClusteringBound.TOP : end; - buffer.add(new RangeTombstoneBoundMarker(markerEnd, getAndClearOpenMarker())); + buffer.add(new RangeTombstoneBoundMarker(markerEnd, openMarker)); if (hasPreviousBlock) skipFirstIteratedItem = true; } http://git-wip-us.apache.org/repos/asf/cassandra/blob/66115139/test/unit/org/apache/cassandra/cql3/ViewTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/cql3/ViewTest.java index a4d227a,84b2773..7d9ab71 --- a/test/unit/org/apache/cassandra/cql3/ViewTest.java +++ b/test/unit/org/apache/cassandra/cql3/ViewTest.java @@@ -45,12 -45,12 +45,13 @@@ import org.apache.cassandra.db.ColumnFa import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.utils.FBUtilities; + public class ViewTest extends CQLTester { - int protocolVersion = 4; + ProtocolVersion protocolVersion = ProtocolVersion.V4; private final List<String> views = new ArrayList<>(); @BeforeClass http://git-wip-us.apache.org/repos/asf/cassandra/blob/66115139/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java index b056da1,b1a374f..7ad6198 --- a/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java +++ b/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java @@@ -32,11 -33,10 +33,11 @@@ import org.junit.Test import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.cassandra.SchemaLoader; + import org.apache.cassandra.Util; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.Schema; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.QueryProcessor; @@@ -134,24 -152,136 +155,142 @@@ public class SinglePartitionSliceComman cmd = ReadCommand.legacyReadCommandSerializer.deserialize(in, MessagingService.VERSION_21); logger.debug("ReadCommand: {}", cmd); - UnfilteredPartitionIterator partitionIterator = cmd.executeLocally(ReadOrderGroup.emptyGroup()); - ReadResponse response = ReadResponse.createDataResponse(partitionIterator, cmd); - - logger.debug("creating response: {}", response); - partitionIterator = response.makeIterator(cmd); - assert partitionIterator.hasNext(); - UnfilteredRowIterator partition = partitionIterator.next(); + try (ReadExecutionController controller = cmd.executionController(); + UnfilteredPartitionIterator partitionIterator = cmd.executeLocally(controller)) + { + ReadResponse response = ReadResponse.createDataResponse(partitionIterator, cmd); - LegacyLayout.LegacyUnfilteredPartition rowIter = LegacyLayout.fromUnfilteredRowIterator(cmd, partition); - Assert.assertEquals(Collections.emptyList(), rowIter.cells); + logger.debug("creating response: {}", response); + try (UnfilteredPartitionIterator pIter = response.makeIterator(cmd)) + { + assert pIter.hasNext(); + try (UnfilteredRowIterator partition = pIter.next()) + { + LegacyLayout.LegacyUnfilteredPartition rowIter = LegacyLayout.fromUnfilteredRowIterator(cmd, partition); + Assert.assertEquals(Collections.emptyList(), rowIter.cells); + } + } + } } + @Test + public void testMultiNamesCommandWithFlush() + { + testMultiNamesOrSlicesCommand(true, false); + } + + @Test + public void testMultiNamesCommandWithoutFlush() + { + testMultiNamesOrSlicesCommand(false, false); + } + + @Test + public void testMultiSlicesCommandWithFlush() + { + testMultiNamesOrSlicesCommand(true, true); + } + + @Test + public void testMultiSlicesCommandWithoutFlush() + { + testMultiNamesOrSlicesCommand(false, true); + } + + private AbstractClusteringIndexFilter createClusteringFilter(int uniqueCk1, int uniqueCk2, boolean isSlice) + { + Slices.Builder slicesBuilder = new Slices.Builder(CFM_SLICES.comparator); + BTreeSet.Builder<Clustering> namesBuilder = BTreeSet.builder(CFM_SLICES.comparator); + + for (int ck1 = 0; ck1 < uniqueCk1; ck1++) + { + for (int ck2 = 0; ck2 < uniqueCk2; ck2++) + { + if (isSlice) + slicesBuilder.add(Slice.make(Util.clustering(CFM_SLICES.comparator, ck1, ck2))); + else + namesBuilder.add(Util.clustering(CFM_SLICES.comparator, ck1, ck2)); + } + } + if (isSlice) + return new ClusteringIndexSliceFilter(slicesBuilder.build(), false); + return new ClusteringIndexNamesFilter(namesBuilder.build(), false); + } + + private void testMultiNamesOrSlicesCommand(boolean flush, boolean isSlice) + { + boolean isTombstone = flush || isSlice; + int deletionTime = 5; + int ck1 = 1; + int uniqueCk1 = 2; + int uniqueCk2 = 3; + + DecoratedKey key = CFM_SLICES.decorateKey(ByteBufferUtil.bytes("k")); + QueryProcessor.executeInternal(String.format("DELETE FROM ks.tbl_slices USING TIMESTAMP %d WHERE k='k' AND c1=%d", + deletionTime, + ck1)); + + if (flush) + Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE_SCLICES).forceBlockingFlush(); + + AbstractClusteringIndexFilter clusteringFilter = createClusteringFilter(uniqueCk1, uniqueCk2, isSlice); + ReadCommand cmd = SinglePartitionReadCommand.create(CFM_SLICES, + FBUtilities.nowInSeconds(), + ColumnFilter.all(CFM_SLICES), + RowFilter.NONE, + DataLimits.NONE, + key, + clusteringFilter); + - UnfilteredPartitionIterator partitionIterator = cmd.executeLocally(ReadOrderGroup.emptyGroup()); ++ UnfilteredPartitionIterator partitionIterator = cmd.executeLocally(cmd.executionController()); + assert partitionIterator.hasNext(); + UnfilteredRowIterator partition = partitionIterator.next(); + + int count = 0; + boolean open = true; + while (partition.hasNext()) + { + Unfiltered unfiltered = partition.next(); + if (isTombstone) + { + assertTrue(unfiltered.isRangeTombstoneMarker()); + RangeTombstoneMarker marker = (RangeTombstoneMarker) unfiltered; + + // check if it's open-close pair + assertTrue(marker.isOpen(false) == open); + // check deletion time same as Range Deletion + if (open) + assertEquals(deletionTime, marker.openDeletionTime(false).markedForDeleteAt()); + else + assertEquals(deletionTime, marker.closeDeletionTime(false).markedForDeleteAt()); + + // check clustering values + Clustering clustering = Util.clustering(CFM_SLICES.comparator, ck1, count / 2); + for (int i = 0; i < CFM_SLICES.comparator.size(); i++) + { + int cmp = CFM_SLICES.comparator.compareComponent(i, - clustering.values[i], ++ clustering.getRawValues()[i], + marker.clustering().values[i]); + assertEquals(0, cmp); + } + open = !open; + } + else + { + // deleted row + assertTrue(unfiltered.isRow()); + Row row = (Row) unfiltered; + assertEquals(deletionTime, row.deletion().time().markedForDeleteAt()); + assertEquals(0, row.size()); // no btree + } + count++; + } + if (isTombstone) + assertEquals(uniqueCk2 * 2, count); // open and close range tombstones + else + assertEquals(uniqueCk2, count); + } + private void checkForS(UnfilteredPartitionIterator pi) { Assert.assertTrue(pi.toString(), pi.hasNext()); --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
