Repository: cassandra Updated Branches: refs/heads/trunk d0b34d383 -> 0f58f6c65
Make it possible to change neverPurgeTombstones during runtime Patch by marcuse; reviewed by Chris Lohfink for CASSANDRA-14214 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0f58f6c6 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0f58f6c6 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0f58f6c6 Branch: refs/heads/trunk Commit: 0f58f6c6501518e57ba021cb959b288fd533f472 Parents: d0b34d3 Author: Marcus Eriksson <marc...@apache.org> Authored: Mon Feb 5 09:27:52 2018 +0100 Committer: Marcus Eriksson <marc...@apache.org> Committed: Thu Feb 8 08:41:39 2018 +0100 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/cassandra/db/ColumnFamilyStore.java | 19 +++++ .../cassandra/db/ColumnFamilyStoreMBean.java | 4 + .../compaction/AbstractCompactionStrategy.java | 2 +- .../db/compaction/CompactionController.java | 16 ++-- .../db/compaction/CompactionsCQLTest.java | 83 ++++++++++++++++++++ 6 files changed, 119 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f58f6c6/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index cadd4c8..191caaf 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0 + * Make it possible to change neverPurgeTombstones during runtime (CASSANDRA-14214) * Remove GossipDigestSynVerbHandler#doSort() (CASSANDRA-14174) * Add nodetool clientlist (CASSANDRA-13665) * Revert ProtocolVersion changes from CASSANDRA-7544 (CASSANDRA-14211) http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f58f6c6/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index c12b474..8e7b220 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -218,6 +218,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean @VisibleForTesting final DiskBoundaryManager diskBoundaryManager = new DiskBoundaryManager(); + private volatile boolean neverPurgeTombstones = false; + public static void shutdownPostFlushExecutor() throws InterruptedException { postFlushExecutor.shutdown(); @@ -2636,4 +2638,21 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean { diskBoundaryManager.invalidate(); } + + @Override + public void setNeverPurgeTombstones(boolean value) + { + if (neverPurgeTombstones != value) + logger.info("Changing neverPurgeTombstones for {}.{} from {} to {}", keyspace.getName(), getTableName(), neverPurgeTombstones, value); + else + logger.info("Not changing neverPurgeTombstones for {}.{}, it is {}", keyspace.getName(), getTableName(), neverPurgeTombstones); + + neverPurgeTombstones = value; + } + + @Override + public boolean getNeverPurgeTombstones() + { + return neverPurgeTombstones; + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f58f6c6/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java index e361ffe..bdb842d 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java @@ -195,4 +195,8 @@ public interface ColumnFamilyStoreMBean Enable/Disable compaction space check */ public void compactionDiskSpaceCheck(boolean enable); + + public void setNeverPurgeTombstones(boolean value); + + public boolean getNeverPurgeTombstones(); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f58f6c6/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java index e88524f..a43761f 100644 --- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java @@ -350,7 +350,7 @@ public abstract class AbstractCompactionStrategy */ protected boolean worthDroppingTombstones(SSTableReader sstable, int gcBefore) { - if (disableTombstoneCompactions || CompactionController.NEVER_PURGE_TOMBSTONES) + if (disableTombstoneCompactions || CompactionController.NEVER_PURGE_TOMBSTONES || cfs.getNeverPurgeTombstones()) return false; // since we use estimations to calculate, there is a chance that compaction will not drop tombstones actually. // if that happens we will end up in infinite compaction loop, so first we check enough if enough time has http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f58f6c6/src/java/org/apache/cassandra/db/compaction/CompactionController.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionController.java b/src/java/org/apache/cassandra/db/compaction/CompactionController.java index 32ce67a..59bba0a 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionController.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionController.java @@ -112,6 +112,12 @@ public class CompactionController implements AutoCloseable return; } + if (cfs.getNeverPurgeTombstones()) + { + logger.debug("not refreshing overlaps for {}.{} - neverPurgeTombstones is enabled", cfs.keyspace.getName(), cfs.getTableName()); + return; + } + for (SSTableReader reader : overlappingSSTables) { if (reader.isMarkedCompacted()) @@ -124,7 +130,7 @@ public class CompactionController implements AutoCloseable private void refreshOverlaps() { - if (NEVER_PURGE_TOMBSTONES) + if (NEVER_PURGE_TOMBSTONES || cfs.getNeverPurgeTombstones()) return; if (this.overlappingSSTables != null) @@ -167,8 +173,8 @@ public class CompactionController implements AutoCloseable { logger.trace("Checking droppable sstables in {}", cfStore); - if (NEVER_PURGE_TOMBSTONES || compacting == null) - return Collections.emptySet(); + if (NEVER_PURGE_TOMBSTONES || compacting == null || cfStore.getNeverPurgeTombstones()) + return Collections.<SSTableReader>emptySet(); if (cfStore.getCompactionStrategyManager().onlyPurgeRepairedTombstones() && !Iterables.all(compacting, SSTableReader::isRepaired)) return Collections.emptySet(); @@ -259,7 +265,7 @@ public class CompactionController implements AutoCloseable */ public LongPredicate getPurgeEvaluator(DecoratedKey key) { - if (NEVER_PURGE_TOMBSTONES || !compactingRepaired()) + if (NEVER_PURGE_TOMBSTONES || !compactingRepaired() || cfs.getNeverPurgeTombstones()) return time -> false; overlapIterator.update(key); @@ -321,7 +327,7 @@ public class CompactionController implements AutoCloseable // caller must close iterators public Iterable<UnfilteredRowIterator> shadowSources(DecoratedKey key, boolean tombstoneOnly) { - if (!provideTombstoneSources() || !compactingRepaired() || NEVER_PURGE_TOMBSTONES) + if (!provideTombstoneSources() || !compactingRepaired() || NEVER_PURGE_TOMBSTONES || cfs.getNeverPurgeTombstones()) return null; overlapIterator.update(key); return Iterables.filter(Iterables.transform(overlapIterator.overlaps(), http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f58f6c6/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java index 7873ac9..4d5215e 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java @@ -25,7 +25,14 @@ import org.junit.Test; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -226,6 +233,82 @@ public class CompactionsCQLTest extends CQLTester getCurrentColumnFamilyStore().setCompactionParameters(localOptions); } + @Test + public void testPerCFSNeverPurgeTombstonesCell() throws Throwable + { + testPerCFSNeverPurgeTombstonesHelper(true); + } + + @Test + public void testPerCFSNeverPurgeTombstones() throws Throwable + { + testPerCFSNeverPurgeTombstonesHelper(false); + } + + + public void testPerCFSNeverPurgeTombstonesHelper(boolean deletedCell) throws Throwable + { + createTable("CREATE TABLE %s (id int primary key, b text) with gc_grace_seconds = 0"); + for (int i = 0; i < 100; i++) + { + execute("INSERT INTO %s (id, b) VALUES (?, ?)", i, String.valueOf(i)); + } + getCurrentColumnFamilyStore().forceBlockingFlush(); + + assertTombstones(getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(), false); + if (deletedCell) + execute("UPDATE %s SET b=null WHERE id = ?", 50); + else + execute("DELETE FROM %s WHERE id = ?", 50); + getCurrentColumnFamilyStore().setNeverPurgeTombstones(false); + getCurrentColumnFamilyStore().forceBlockingFlush(); + Thread.sleep(2000); // wait for gcgs to pass + getCurrentColumnFamilyStore().forceMajorCompaction(); + assertTombstones(getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(), false); + if (deletedCell) + execute("UPDATE %s SET b=null WHERE id = ?", 44); + else + execute("DELETE FROM %s WHERE id = ?", 44); + getCurrentColumnFamilyStore().setNeverPurgeTombstones(true); + getCurrentColumnFamilyStore().forceBlockingFlush(); + Thread.sleep(1100); + getCurrentColumnFamilyStore().forceMajorCompaction(); + assertTombstones(getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(), true); + // disable it again and make sure the tombstone is gone: + getCurrentColumnFamilyStore().setNeverPurgeTombstones(false); + getCurrentColumnFamilyStore().forceMajorCompaction(); + assertTombstones(getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(), false); + getCurrentColumnFamilyStore().truncateBlocking(); + } + + private void assertTombstones(SSTableReader sstable, boolean expectTS) + { + boolean foundTombstone = false; + try(ISSTableScanner scanner = sstable.getScanner()) + { + while (scanner.hasNext()) + { + try (UnfilteredRowIterator iter = scanner.next()) + { + if (!iter.partitionLevelDeletion().isLive()) + foundTombstone = true; + while (iter.hasNext()) + { + Unfiltered unfiltered = iter.next(); + assertTrue(unfiltered instanceof Row); + for (Cell c : ((Row)unfiltered).cells()) + { + if (c.isTombstone()) + foundTombstone = true; + } + + } + } + } + } + assertEquals(expectTS, foundTombstone); + } + public boolean verifyStrategies(CompactionStrategyManager manager, Class<? extends AbstractCompactionStrategy> expected) { boolean found = false; --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org