Calculate last compacted keys from higher level patch by Dikang Gu; reviewed by marcuse for CASSANDRA-6216
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6808e755 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6808e755 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6808e755 Branch: refs/heads/trunk Commit: 6808e7555762236b6222f65bf066f90f7597405e Parents: d0fc953 Author: Dikang Gu <[email protected]> Authored: Sat Aug 20 00:55:22 2016 -0700 Committer: Marcus Eriksson <[email protected]> Committed: Tue Aug 23 16:18:39 2016 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../compaction/LeveledCompactionStrategy.java | 7 ++++ .../db/compaction/LeveledManifest.java | 36 ++++++++++++++++++++ 3 files changed, 44 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/6808e755/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index d384fb3..0d461f3 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.0.9 + * Calculate last compacted key on startup (CASSANDRA-6216) * Add schema to snapshot manifest, add USING TIMESTAMP clause to ALTER TABLE statements (CASSANDRA-7190) * Fix clean interval not sent to commit log for empty memtable flush (CASSANDRA-12436) * Fix potential resource leak in RMIServerSocketFactoryImpl (CASSANDRA-12331) http://git-wip-us.apache.org/repos/asf/cassandra/blob/6808e755/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java index cd74620..65fca27 100644 --- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java @@ -85,6 +85,13 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy return manifest.getAllLevelSize(); } + @Override + public void startup() + { + manifest.calculateLastCompactedKeys(); + super.startup(); + } + /** * the only difference between background and maximal in LCS is that maximal is still allowed * (by explicit user request) even when compaction is disabled. http://git-wip-us.apache.org/repos/asf/cassandra/blob/6808e755/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java index 2bfe88f..cf3ae1a 100644 --- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java @@ -30,6 +30,7 @@ import com.google.common.collect.Sets; import com.google.common.primitives.Ints; import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,6 +67,7 @@ public class LeveledManifest @VisibleForTesting protected final List<SSTableReader>[] generations; private final PartitionPosition[] lastCompactedKeys; + private volatile boolean calculatedLastCompactedKeys = false; private final long maxSSTableSizeInBytes; private final SizeTieredCompactionStrategyOptions options; private final int [] compactionCounter; @@ -104,9 +106,38 @@ public class LeveledManifest { manifest.repairOverlappingSSTables(i); } + manifest.calculateLastCompactedKeys(); return manifest; } + /** + * If we want to start compaction in level n, find the newest (by modification time) file in level n+1 + * and use its last token for last compacted key in level n; + */ + public void calculateLastCompactedKeys() + { + for (int i = 0; i < generations.length - 1; i++) + { + // this level is empty + if (generations[i + 1].isEmpty()) + continue; + + SSTableReader sstableWithMaxModificationTime = null; + long maxModificationTime = Long.MIN_VALUE; + for (SSTableReader ssTableReader : generations[i + 1]) + { + long modificationTime = ssTableReader.getCreationTimeFor(Component.DATA); + if (modificationTime >= maxModificationTime) + { + sstableWithMaxModificationTime = ssTableReader; + maxModificationTime = modificationTime; + } + } + + lastCompactedKeys[i] = sstableWithMaxModificationTime.last; + } + } + public synchronized void add(SSTableReader reader) { int level = reader.getSSTableLevel(); @@ -307,6 +338,11 @@ public class LeveledManifest // This isn't a magic wand -- if you are consistently writing too fast for LCS to keep // up, you're still screwed. But if instead you have intermittent bursts of activity, // it can help a lot. + if (!calculatedLastCompactedKeys) + { + this.calculateLastCompactedKeys(); + } + for (int i = generations.length - 1; i > 0; i--) { List<SSTableReader> sstables = getLevel(i);
