Merge branch 'cassandra-3.0' into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a6a9dce1 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a6a9dce1 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a6a9dce1 Branch: refs/heads/cassandra-3.11 Commit: a6a9dce157a4ed14e7d08e854c504423dd199daa Parents: 69f8cc7 d60c783 Author: Blake Eggleston <bdeggles...@gmail.com> Authored: Tue Nov 6 11:17:47 2018 -0800 Committer: Blake Eggleston <bdeggles...@gmail.com> Committed: Tue Nov 6 11:19:04 2018 -0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + src/java/org/apache/cassandra/db/Slice.java | 25 +----- .../cassandra/io/sstable/format/Version.java | 2 + .../io/sstable/format/big/BigFormat.java | 9 ++ .../io/sstable/metadata/MetadataCollector.java | 25 +++--- .../io/sstable/metadata/StatsMetadata.java | 14 ++- .../mc-1-big-CompressionInfo.db | Bin 0 -> 43 bytes .../mc-1-big-Data.db | Bin 0 -> 65 bytes .../mc-1-big-Digest.crc32 | 1 + .../mc-1-big-Filter.db | Bin 0 -> 16 bytes .../mc-1-big-Index.db | Bin 0 -> 8 bytes .../mc-1-big-Statistics.db | Bin 0 -> 4789 bytes .../mc-1-big-Summary.db | Bin 0 -> 56 bytes .../mc-1-big-TOC.txt | 8 ++ .../db/SinglePartitionSliceCommandTest.java | 87 +++++++++++++++++++ .../cassandra/io/sstable/LegacySSTableTest.java | 34 +++++++- 16 files changed, 166 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 03abb5b,0fb1b86..f923fa0 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,5 -1,5 +1,6 @@@ -3.0.18 +3.11.4 +Merged from 3.0: + * Sstable min/max metadata can cause data loss (CASSANDRA-14861) * Dropped columns can cause reverse sstable iteration to return prematurely (CASSANDRA-14838) * Legacy sstables with multi block range tombstones create invalid bound sequences (CASSANDRA-14823) * Expand range tombstone validation checks to multiple interim request stages (CASSANDRA-14824) http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/src/java/org/apache/cassandra/db/Slice.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java index e61f4b3,f48d0a6..a618c96 --- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java @@@ -19,8 -19,8 +19,9 @@@ package org.apache.cassandra.io.sstable import java.nio.ByteBuffer; import java.util.ArrayList; + import java.util.Arrays; import java.util.Collections; +import java.util.EnumMap; import java.util.List; import java.util.Map; @@@ -93,8 -95,8 +97,8 @@@ public class MetadataCollector implemen protected double compressionRatio = NO_COMPRESSION_RATIO; protected StreamingHistogram.StreamingHistogramBuilder estimatedTombstoneDropTime = defaultTombstoneDropTimeHistogramBuilder(); protected int sstableLevel; - protected ByteBuffer[] minClusteringValues; - protected ByteBuffer[] maxClusteringValues; - private ClusteringPrefix minClustering = Slice.Bound.TOP; - private ClusteringPrefix maxClustering = Slice.Bound.BOTTOM; ++ private ClusteringPrefix minClustering = ClusteringBound.TOP; ++ private ClusteringPrefix maxClustering = ClusteringBound.BOTTOM; protected boolean hasLegacyCounterShards = false; protected long totalColumnsSet; protected long totalRows; @@@ -277,7 -271,8 +273,8 @@@ public Map<MetadataType, MetadataComponent> finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, SerializationHeader header) { + Preconditions.checkState(comparator.compare(maxClustering, minClustering) >= 0); - Map<MetadataType, MetadataComponent> components = Maps.newHashMap(); + Map<MetadataType, MetadataComponent> components = new EnumMap<>(MetadataType.class); components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance)); components.put(MetadataType.STATS, new StatsMetadata(estimatedPartitionSize, estimatedCellPerPartitionCount, http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java index 7ad6198,2891687..1bdbcb2 --- a/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java +++ b/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java @@@ -37,11 -43,12 +43,13 @@@ import org.apache.cassandra.SchemaLoade import org.apache.cassandra.Util; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.Schema; import org.apache.cassandra.cql3.ColumnIdentifier; + import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; + import org.apache.cassandra.cql3.statements.SelectStatement; import org.apache.cassandra.db.filter.AbstractClusteringIndexFilter; import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter; import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter; @@@ -387,4 -388,81 +397,81 @@@ public class SinglePartitionSliceComman Assert.assertNotNull(ret); Assert.assertFalse(ret.isEmpty()); } + + + public static List<Unfiltered> getUnfilteredsFromSinglePartition(String q) + { + SelectStatement stmt = (SelectStatement) QueryProcessor.parseStatement(q).prepare(ClientState.forInternalCalls()).statement; + + List<Unfiltered> unfiltereds = new ArrayList<>(); + SinglePartitionReadCommand.Group query = (SinglePartitionReadCommand.Group) stmt.getQuery(QueryOptions.DEFAULT, FBUtilities.nowInSeconds()); + Assert.assertEquals(1, query.commands.size()); + SinglePartitionReadCommand command = Iterables.getOnlyElement(query.commands); - try (ReadOrderGroup group = ReadOrderGroup.forCommand(command); - UnfilteredPartitionIterator partitions = command.executeLocally(group)) ++ try (ReadExecutionController controller = ReadExecutionController.forCommand(command); ++ UnfilteredPartitionIterator partitions = command.executeLocally(controller)) + { + assert partitions.hasNext(); + try (UnfilteredRowIterator partition = partitions.next()) + { + while (partition.hasNext()) + { + Unfiltered next = partition.next(); + unfiltereds.add(next); + } + } + assert !partitions.hasNext(); + } + return unfiltereds; + } + + private static void assertQueryReturnsSingleRT(String query) + { + List<Unfiltered> unfiltereds = getUnfilteredsFromSinglePartition(query); + Assert.assertEquals(2, unfiltereds.size()); + Assert.assertTrue(unfiltereds.get(0).isRangeTombstoneMarker()); + Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(0)).isOpen(false)); + Assert.assertTrue(unfiltereds.get(1).isRangeTombstoneMarker()); + Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(1)).isClose(false)); + } + + private static ByteBuffer bb(int v) + { + return Int32Type.instance.decompose(v); + } + + /** + * tests the bug raised in CASSANDRA-14861, where the sstable min/max can + * exclude range tombstones for clustering ranges not also covered by rows + */ + @Test + public void sstableFiltering() + { + QueryProcessor.executeOnceInternal("CREATE TABLE ks.legacy_mc_inaccurate_min_max (k int, c1 int, c2 int, c3 int, v int, primary key (k, c1, c2, c3))"); + CFMetaData metadata = Schema.instance.getCFMetaData("ks", "legacy_mc_inaccurate_min_max"); + ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.cfId); + + QueryProcessor.executeOnceInternal("INSERT INTO ks.legacy_mc_inaccurate_min_max (k, c1, c2, c3, v) VALUES (100, 2, 2, 2, 2)"); + QueryProcessor.executeOnceInternal("DELETE FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1"); + assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1"); + cfs.forceBlockingFlush(); + assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1"); + + assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1 AND c3=1"); // clustering names + + cfs.truncateBlocking(); + + long nowMillis = System.currentTimeMillis(); - Slice slice = Slice.make(new Clustering(bb(2), bb(3)), new Clustering(bb(10), bb(10))); ++ Slice slice = Slice.make(Clustering.make(bb(2), bb(3)), Clustering.make(bb(10), bb(10))); + RangeTombstone rt = new RangeTombstone(slice, new DeletionTime(TimeUnit.MILLISECONDS.toMicros(nowMillis), + Ints.checkedCast(TimeUnit.MILLISECONDS.toSeconds(nowMillis)))); + PartitionUpdate update = new PartitionUpdate(cfs.metadata, bb(100), cfs.metadata.partitionColumns(), 1); + update.add(rt); + new Mutation(update).apply(); + + assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=3 AND c2=2"); + cfs.forceBlockingFlush(); + assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=3 AND c2=2"); + assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=3 AND c2=2 AND c3=2"); // clustering names + + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index dae7823,5c65b31..bd51c0f --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@@ -35,12 -36,20 +36,21 @@@ import org.slf4j.Logger import org.slf4j.LoggerFactory; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; + import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; + import org.apache.cassandra.cql3.statements.SelectStatement; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.ReadOrderGroup; ++import org.apache.cassandra.db.ReadExecutionController; + import org.apache.cassandra.db.SinglePartitionReadCommand; + import org.apache.cassandra.db.SinglePartitionSliceCommandTest; import org.apache.cassandra.db.compaction.Verifier; + import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; + import org.apache.cassandra.db.rows.RangeTombstoneMarker; + import org.apache.cassandra.db.rows.Unfiltered; + import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@@ -278,7 -268,29 +289,28 @@@ public class LegacySSTableTes } } - @Test + public void testInaccurateSSTableMinMax() throws Exception + { + QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_mc_inaccurate_min_max (k int, c1 int, c2 int, c3 int, v int, primary key (k, c1, c2, c3))"); + loadLegacyTable("legacy_%s_inaccurate_min_max%s", "mc", ""); + + /* + sstable has the following mutations: + INSERT INTO legacy_tables.legacy_mc_inaccurate_min_max (k, c1, c2, c3, v) VALUES (100, 4, 4, 4, 4) + DELETE FROM legacy_tables.legacy_mc_inaccurate_min_max WHERE k=100 AND c1<3 + */ + + String query = "SELECT * FROM legacy_tables.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1"; + List<Unfiltered> unfiltereds = SinglePartitionSliceCommandTest.getUnfilteredsFromSinglePartition(query); + Assert.assertEquals(2, unfiltereds.size()); + Assert.assertTrue(unfiltereds.get(0).isRangeTombstoneMarker()); + Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(0)).isOpen(false)); + Assert.assertTrue(unfiltereds.get(1).isRangeTombstoneMarker()); + Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(1)).isClose(false)); + } + + @Test public void testVerifyOldSSTables() throws Exception { for (String legacyVersion : legacyVersions) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org