Merge branch 'cassandra-2.0' into cassandra-2.1
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9beeba32 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9beeba32 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9beeba32 Branch: refs/heads/trunk Commit: 9beeba3202716d26906b76feeff3bee3e16522d5 Parents: 2e7b088 bed42c2 Author: Aleksey Yeschenko <[email protected]> Authored: Tue May 12 21:01:39 2015 +0300 Committer: Aleksey Yeschenko <[email protected]> Committed: Tue May 12 21:01:39 2015 +0300 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/cassandra/db/ColumnFamilyStore.java | 2 +- .../cassandra/db/filter/ColumnCounter.java | 29 ++-- .../cassandra/db/filter/SliceQueryFilter.java | 34 ++-- .../SliceQueryFilterWithTombstonesTest.java | 166 +++++++++++++++++++ 5 files changed, 204 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/9beeba32/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index aa5f235,d7d01cf..8794509 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,24 -1,5 +1,25 @@@ -2.0.15: +2.1.6 + * Fix commitlog getCompletedTasks to not increment (CASSANDRA-9339) + * Fix for harmless exceptions logged as ERROR (CASSANDRA-8564) + * Delete processed sstables in sstablesplit/sstableupgrade (CASSANDRA-8606) + * Improve sstable exclusion from partition tombstones (CASSANDRA-9298) + * Validate the indexed column rather than the cell's contents for 2i (CASSANDRA-9057) + * Add support for top-k custom 2i queries (CASSANDRA-8717) + * Fix error when dropping table during compaction (CASSANDRA-9251) + * cassandra-stress supports validation operations over user profiles (CASSANDRA-8773) + * Add support for rate limiting log messages (CASSANDRA-9029) + * Log the partition key with tombstone warnings (CASSANDRA-8561) + * Reduce runWithCompactionsDisabled poll interval to 1ms (CASSANDRA-9271) + * Fix PITR commitlog replay (CASSANDRA-9195) + * GCInspector logs very different times (CASSANDRA-9124) + * Fix deleting from an empty list (CASSANDRA-9198) + * Update tuple and collection types that use a user-defined type when that UDT + is modified (CASSANDRA-9148, CASSANDRA-9192) + * Use higher timeout for prepair and snapshot in repair (CASSANDRA-9261) + * Fix anticompaction blocking ANTI_ENTROPY stage (CASSANDRA-9151) + * Repair waits for anticompaction to finish (CASSANDRA-9097) +Merged from 2.0: + * Fix counting of tombstones for TombstoneOverwhelmingException (CASSANDRA-9299) * Fix ReconnectableSnitch reconnecting to peers during upgrade (CASSANDRA-6702) * Include keyspace and table name in error log for collections over the size limit (CASSANDRA-9286) http://git-wip-us.apache.org/repos/asf/cassandra/blob/9beeba32/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/9beeba32/src/java/org/apache/cassandra/db/filter/ColumnCounter.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/filter/ColumnCounter.java index 86cfc40,2d0df1f..8be26e1 --- a/src/java/org/apache/cassandra/db/filter/ColumnCounter.java +++ b/src/java/org/apache/cassandra/db/filter/ColumnCounter.java @@@ -37,17 -39,17 +37,17 @@@ public class ColumnCounte this.timestamp = timestamp; } - public void count(Column column, DeletionInfo.InOrderTester tester) + public void count(Cell cell, DeletionInfo.InOrderTester tester) { - if (!isLive(cell, tester, timestamp)) - ignored++; - else - live++; - } + // The cell is shadowed by a higher-level deletion, and won't be retained. + // For the purposes of this counter, we don't care if it's a tombstone or not. - if (tester.isDeleted(column)) ++ if (tester.isDeleted(cell)) + return; - protected static boolean isLive(Cell cell, DeletionInfo.InOrderTester tester, long timestamp) - { - return cell.isLive(timestamp) && !tester.isDeleted(cell); - if (column.isLive(timestamp)) ++ if (cell.isLive(timestamp)) + live++; + else + tombstones++; } public int live() @@@ -96,11 -99,14 +96,14 @@@ assert toGroup == 0 || type != null; } - public void count(Column column, DeletionInfo.InOrderTester tester) + public void count(Cell cell, DeletionInfo.InOrderTester tester) { - if (!isLive(cell, tester, timestamp)) - if (tester.isDeleted(column)) ++ if (tester.isDeleted(cell)) + return; + - if (!column.isLive(timestamp)) ++ if (!cell.isLive(timestamp)) { - ignored++; + tombstones++; return; } http://git-wip-us.apache.org/repos/asf/cassandra/blob/9beeba32/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java index 38947bf,6e6ab6b..1195d4c --- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java +++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java @@@ -202,36 -190,36 +202,43 @@@ public class SliceQueryFilter implement while (reducedColumns.hasNext()) { - Column column = reducedColumns.next(); + Cell cell = reducedColumns.next(); ++ if (logger.isTraceEnabled()) -- logger.trace(String.format("collecting %s of %s: %s", - columnCounter.live(), count, cell.getString(container.getComparator()))); - columnCounter.live(), count, column.getString(container.getComparator()))); ++ logger.trace("collecting {} of {}: {}", columnCounter.live(), count, cell.getString(container.getComparator())); ++ ++ // An expired tombstone will be immediately discarded in memory, and needn't be counted. ++ if (cell.getLocalDeletionTime() < gcBefore) ++ continue; - columnCounter.count(column, tester); + columnCounter.count(cell, tester); if (columnCounter.live() > count) break; - if (respectTombstoneThresholds() && columnCounter.ignored() > DatabaseDescriptor.getTombstoneFailureThreshold()) + if (respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneFailureThreshold()) { -- Tracing.trace("Scanned over {} tombstones; query aborted (see tombstone_failure_threshold)", DatabaseDescriptor.getTombstoneFailureThreshold()); ++ Tracing.trace("Scanned over {} tombstones; query aborted (see tombstone_failure_threshold)", ++ DatabaseDescriptor.getTombstoneFailureThreshold()); logger.error("Scanned over {} tombstones in {}.{}; query aborted (see tombstone_failure_threshold)", -- DatabaseDescriptor.getTombstoneFailureThreshold(), container.metadata().ksName, container.metadata().cfName); ++ DatabaseDescriptor.getTombstoneFailureThreshold(), ++ container.metadata().ksName, ++ container.metadata().cfName); throw new TombstoneOverwhelmingException(); } - container.addIfRelevant(column, tester, gcBefore); + container.maybeAppendColumn(cell, tester, gcBefore); } - Tracing.trace("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored()); - if (logger.isWarnEnabled() && respectTombstoneThresholds() && columnCounter.ignored() > DatabaseDescriptor.getTombstoneWarnThreshold()) + Tracing.trace("Read {} live and {} tombstone cells", columnCounter.live(), columnCounter.tombstones()); - if (respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneWarnThreshold()) ++ if (logger.isWarnEnabled() && respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneWarnThreshold()) { StringBuilder sb = new StringBuilder(); - AbstractType<?> type = container.metadata().comparator; + CellNameType type = container.metadata().comparator; + for (ColumnSlice sl : slices) { - if (sl == null) - continue; + assert sl != null; sb.append('['); sb.append(type.getString(sl.start)); @@@ -240,15 -228,13 +247,15 @@@ sb.append(']'); } - String msg = String.format("Read %d live and %d tombstoned cells in %s.%s for key: %1.512s (see tombstone_warn_threshold). %d columns were requested, slices=%1.512s", - logger.warn("Read {} live and {} tombstone cells in {}.{} (see tombstone_warn_threshold). {} columns was requested, slices={}", - columnCounter.live(), - columnCounter.tombstones(), - container.metadata().ksName, - container.metadata().cfName, - count, - sb); ++ String msg = String.format("Read %d live and %d tombstone cells in %s.%s for key: %1.512s (see tombstone_warn_threshold). %d columns were requested, slices=%1.512s", + columnCounter.live(), - columnCounter.ignored(), ++ columnCounter.tombstones(), + container.metadata().ksName, + container.metadata().cfName, + container.metadata().getKeyValidator().getString(key.getKey()), + count, + sb); + logger.warn(msg); } } @@@ -435,11 -376,11 +442,10 @@@ ColumnSlice[] slices; slices = new ColumnSlice[in.readInt()]; for (int i = 0; i < slices.length; i++) - slices[i] = ColumnSlice.serializer.deserialize(in, version); + slices[i] = type.sliceSerializer().deserialize(in, version); boolean reversed = in.readBoolean(); int count = in.readInt(); -- int compositesToGroup = -1; -- compositesToGroup = in.readInt(); ++ int compositesToGroup = in.readInt(); return new SliceQueryFilter(slices, reversed, count, compositesToGroup); } @@@ -459,43 -400,4 +465,43 @@@ return size; } } + + public Iterator<RangeTombstone> getRangeTombstoneIterator(final ColumnFamily source) + { + final DeletionInfo delInfo = source.deletionInfo(); + if (!delInfo.hasRanges() || slices.length == 0) - return Iterators.<RangeTombstone>emptyIterator(); ++ return Iterators.emptyIterator(); + + return new AbstractIterator<RangeTombstone>() + { + private int sliceIdx = 0; + private Iterator<RangeTombstone> sliceIter = currentRangeIter(); + + protected RangeTombstone computeNext() + { + while (true) + { + if (sliceIter.hasNext()) + return sliceIter.next(); + + if (!nextSlice()) + return endOfData(); + + sliceIter = currentRangeIter(); + } + } + + private Iterator<RangeTombstone> currentRangeIter() + { + ColumnSlice slice = slices[reversed ? (slices.length - 1 - sliceIdx) : sliceIdx]; + return reversed ? delInfo.rangeIterator(slice.finish, slice.start) + : delInfo.rangeIterator(slice.start, slice.finish); + } + + private boolean nextSlice() + { + return ++sliceIdx < slices.length; + } + }; + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/9beeba32/test/unit/org/apache/cassandra/cql3/SliceQueryFilterWithTombstonesTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/cql3/SliceQueryFilterWithTombstonesTest.java index 0000000,4440782..0cb9819 mode 000000,100644..100644 --- a/test/unit/org/apache/cassandra/cql3/SliceQueryFilterWithTombstonesTest.java +++ b/test/unit/org/apache/cassandra/cql3/SliceQueryFilterWithTombstonesTest.java @@@ -1,0 -1,150 +1,166 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.cassandra.cql3; + ++import java.util.concurrent.TimeUnit; ++ + import org.junit.AfterClass; + import org.junit.BeforeClass; + import org.junit.Test; + -import org.apache.cassandra.SchemaLoader; + import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.ConsistencyLevel; + import org.apache.cassandra.db.filter.TombstoneOverwhelmingException; -import org.apache.cassandra.gms.Gossiper; + + import static junit.framework.Assert.assertTrue; + import static junit.framework.Assert.fail; + -import static org.apache.cassandra.cql3.QueryProcessor.process; -import static org.apache.cassandra.cql3.QueryProcessor.processInternal; - + /** + * Test that TombstoneOverwhelmingException gets thrown when it should be and doesn't when it shouldn't be. + */ -public class SliceQueryFilterWithTombstonesTest ++public class SliceQueryFilterWithTombstonesTest extends CQLTester + { - static final String KEYSPACE = "tombstone_overwhelming_exception_test"; - static final String TABLE = "overwhelmed"; - + static final int ORIGINAL_THRESHOLD = DatabaseDescriptor.getTombstoneFailureThreshold(); + static final int THRESHOLD = 100; + + @BeforeClass + public static void setUp() throws Throwable + { - SchemaLoader.loadSchema(); - - process(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", - KEYSPACE), - ConsistencyLevel.ONE); - - process(String.format("CREATE TABLE IF NOT EXISTS %s.%s (a text, b text, c text, PRIMARY KEY (a, b));", - KEYSPACE, - TABLE), - ConsistencyLevel.ONE); - + DatabaseDescriptor.setTombstoneFailureThreshold(THRESHOLD); + } + + @AfterClass + public static void tearDown() + { - Gossiper.instance.stop(); - + DatabaseDescriptor.setTombstoneFailureThreshold(ORIGINAL_THRESHOLD); + } + - private static UntypedResultSet execute(String query) - { - return processInternal(String.format(query, KEYSPACE, TABLE)); - } - + @Test - public void testBelowThresholdSelect() ++ public void testBelowThresholdSelect() throws Throwable + { ++ createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));"); ++ + // insert exactly the amount of tombstones that shouldn't trigger an exception + for (int i = 0; i < THRESHOLD; i++) - execute("INSERT INTO %s.%s (a, b, c) VALUES ('key1', 'column" + i + "', null);"); ++ execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);"); + + try + { - execute("SELECT * FROM %s.%s WHERE a = 'key1';"); ++ execute("SELECT * FROM %s WHERE a = 'key';"); + } + catch (Throwable e) + { + fail("SELECT with tombstones below the threshold should not have failed, but has: " + e); + } + } + + @Test - public void testBeyondThresholdSelect() ++ public void testBeyondThresholdSelect() throws Throwable + { ++ createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));"); ++ + // insert exactly the amount of tombstones that *SHOULD* trigger an exception + for (int i = 0; i < THRESHOLD + 1; i++) - execute("INSERT INTO %s.%s (a, b, c) VALUES ('key2', 'column" + i + "', null);"); ++ execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);"); + + try + { - execute("SELECT * FROM %s.%s WHERE a = 'key2';"); ++ execute("SELECT * FROM %s WHERE a = 'key';"); + fail("SELECT with tombstones beyond the threshold should have failed, but hasn't"); + } + catch (Throwable e) + { + assertTrue(e instanceof TombstoneOverwhelmingException); + } + } + + @Test - public void testAllShadowedSelect() ++ public void testAllShadowedSelect() throws Throwable + { ++ createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));"); ++ + // insert exactly the amount of tombstones that *SHOULD* normally trigger an exception + for (int i = 0; i < THRESHOLD + 1; i++) - execute("INSERT INTO %s.%s (a, b, c) VALUES ('key3', 'column" + i + "', null);"); ++ execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);"); + + // delete all with a partition level tombstone - execute("DELETE FROM %s.%s WHERE a = 'key3'"); ++ execute("DELETE FROM %s WHERE a = 'key'"); + + try + { - execute("SELECT * FROM %s.%s WHERE a = 'key3';"); ++ execute("SELECT * FROM %s WHERE a = 'key';"); + } + catch (Throwable e) + { + fail("SELECT with tombstones shadowed by a partition tombstone should not have failed, but has: " + e); + } + } + + @Test - public void testLiveShadowedCellsSelect() ++ public void testLiveShadowedCellsSelect() throws Throwable + { ++ createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));"); ++ + for (int i = 0; i < THRESHOLD + 1; i++) - execute("INSERT INTO %s.%s (a, b, c) VALUES ('key4', 'column" + i + "', 'column');"); ++ execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', 'column');"); + + // delete all with a partition level tombstone - execute("DELETE FROM %s.%s WHERE a = 'key4'"); ++ execute("DELETE FROM %s WHERE a = 'key'"); + + try + { - execute("SELECT * FROM %s.%s WHERE a = 'key4';"); ++ execute("SELECT * FROM %s WHERE a = 'key';"); + } + catch (Throwable e) + { + fail("SELECT with regular cells shadowed by a partition tombstone should not have failed, but has: " + e); + } + } ++ ++ @Test ++ public void testExpiredTombstones() throws Throwable ++ { ++ createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds = 1;"); ++ ++ for (int i = 0; i < THRESHOLD + 1; i++) ++ execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);"); ++ ++ // not yet past gc grace - must throw a TOE ++ try ++ { ++ execute("SELECT * FROM %s WHERE a = 'key';"); ++ fail("SELECT with tombstones beyond the threshold should have failed, but hasn't"); ++ } ++ catch (Throwable e) ++ { ++ assertTrue(e instanceof TombstoneOverwhelmingException); ++ } ++ ++ // sleep past gc grace ++ TimeUnit.SECONDS.sleep(2); ++ ++ // past gc grace - must not throw a TOE now ++ try ++ { ++ execute("SELECT * FROM %s WHERE a = 'key';"); ++ } ++ catch (Throwable e) ++ { ++ fail("SELECT with expired tombstones beyond the threshold should not have failed, but has: " + e); ++ } ++ } + }
