Automatic fixing of overlapping leveled sstables patch by jbellis; reviewed by slebresne for CASSANDRA-4644
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6ad7d45a Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6ad7d45a Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6ad7d45a Branch: refs/heads/trunk Commit: 6ad7d45ad56482707ecd541984894e4e2a278cfb Parents: aa7dafa Author: Jonathan Ellis <jbel...@apache.org> Authored: Mon Sep 17 12:37:01 2012 -0500 Committer: Jonathan Ellis <jbel...@apache.org> Committed: Mon Sep 17 12:37:01 2012 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cassandra/db/compaction/LeveledManifest.java | 39 +++++++++++---- .../apache/cassandra/tools/StandaloneScrubber.java | 27 +---------- 3 files changed, 31 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/6ad7d45a/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 7098320..0530134 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,7 @@ access permissions and grant/revoke commands (CASSANDRA-4490) * fix assumption error in CLI when updating/describing keyspace (CASSANDRA-4322) * Adds offline sstablescrub to debian packaging (CASSANDRA-4642) + * Automatic fixing of overlapping leveled sstables (CASSANDRA-4644) 1.1.5 http://git-wip-us.apache.org/repos/asf/cassandra/blob/6ad7d45a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java index b54fe93..493fd9f 100644 --- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java @@ -204,19 +204,38 @@ public class LeveledManifest for (SSTableReader ssTableReader : added) add(ssTableReader, newLevel); + // Fix overlapping sstables from CASSANDRA-4321/4411 if (newLevel != 0) + repairOverlappingSSTables(newLevel); + + serialize(); + } + + public synchronized void repairOverlappingSSTables(int level) + { + SSTableReader previous = null; + Collections.sort(generations[level], SSTable.sstableComparator); + List<SSTableReader> outOfOrderSSTables = new ArrayList<SSTableReader>(); + for (SSTableReader current : generations[level]) { - // Integerity check - DecoratedKey last = null; - Collections.sort(generations[newLevel], SSTable.sstableComparator); - for (SSTableReader sstable : generations[newLevel]) + if (previous != null && current.first.compareTo(previous.last) <= 0) + { + logger.error(String.format("At level %d, %s [%s, %s] overlaps %s [%s, %s]. This is caused by a bug in Cassandra 1.1.0 .. 1.1.3. Sending back to L0. If you have not yet run scrub, you should do so since you may also have rows out-of-order within an sstable", + level, previous, previous.first, previous.last, current, current.first, current.last)); + outOfOrderSSTables.add(current); + } + else { - assert last == null || sstable.first.compareTo(last) > 0; - last = sstable.last; + previous = current; } } - serialize(); + if (!outOfOrderSSTables.isEmpty()) + { + for (SSTableReader sstable : outOfOrderSSTables) + sendBackToL0(sstable); + serialize(); + } } public synchronized void replace(Iterable<SSTableReader> removed, Iterable<SSTableReader> added) @@ -235,12 +254,10 @@ public class LeveledManifest serialize(); } - public synchronized void sendBackToL0(SSTableReader sstable) + private synchronized void sendBackToL0(SSTableReader sstable) { remove(sstable); add(sstable, 0); - - serialize(); } private String toString(Iterable<SSTableReader> sstables) @@ -593,4 +610,6 @@ public class LeveledManifest new Object[] {Arrays.asList(estimated), cfs.table.name, cfs.columnFamily}); return Ints.checkedCast(tasks); } + + } http://git-wip-us.apache.org/repos/asf/cassandra/blob/6ad7d45a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java index 10f9cd8..439ee0d 100644 --- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java +++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java @@ -169,32 +169,7 @@ public class StandaloneScrubber { System.out.println(String.format("Checking leveled manifest")); for (int i = 1; i <= manifest.getLevelCount(); ++i) - { - List<SSTableReader> sstables = new ArrayList<SSTableReader>(manifest.getLevel(i)); - Collections.sort(sstables, SSTable.sstableComparator); - if (sstables.isEmpty()) - continue; - - Iterator<SSTableReader> iter = sstables.iterator(); - SSTableReader previous = iter.next(); - while (iter.hasNext()) - { - SSTableReader current = iter.next(); - - if (previous.last.compareTo(current.first) >= 0) - { - System.err.println(String.format("At level %d, %s [%s, %s] overlaps %s [%s, %s]", i, - previous, previous.first, previous.last, - current, current.first, current.last)); - System.out.println(String.format("Sending %s back to L0 to fix intra-level overlapping", current)); - manifest.sendBackToL0(current); - } - else - { - previous = current; - } - } - } + manifest.repairOverlappingSSTables(i); } private static class Options