calculate exact size required for cleanup operations patch by yukim; reviewed by jbellis for CASSANDRA-1404
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5979bce8 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5979bce8 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5979bce8 Branch: refs/heads/cassandra-1.1 Commit: 5979bce8ec6338f3aaf88ce193da01e10249b12e Parents: 783e5d4 Author: Jonathan Ellis <jbel...@apache.org> Authored: Tue May 15 16:29:54 2012 -0500 Committer: Jonathan Ellis <jbel...@apache.org> Committed: Tue May 15 16:30:04 2012 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/cassandra/db/ColumnFamilyStore.java | 30 ++++++++++++--- .../cassandra/db/compaction/CompactionManager.java | 2 +- .../cassandra/db/compaction/CompactionTask.java | 4 +- 4 files changed, 28 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/5979bce8/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 7f8f7a0..211314f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 1.1.1-dev + * calculate exact size required for cleanup operations (CASSANDRA-1404) * avoid blocking additional writes during flush when the commitlog gets behind temporarily (CASSANDRA-1991) * enable caching on index CFs based on data CF cache setting (CASSANDRA-4197) http://git-wip-us.apache.org/repos/asf/cassandra/blob/5979bce8/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 4a0739e..4cfe041 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -889,17 +889,35 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean CompactionManager.instance.submitBackground(this); } - /* - * Add up all the files sizes this is the worst case file + /** + * Calculate expected file size of SSTable after compaction. + * + * If operation type is {@code CLEANUP}, then we calculate expected file size + * with checking token range to be eliminated. + * Other than that, we just add up all the files' size, which is the worst case file * size for compaction of all the list of files given. + * + * @param sstables SSTables to calculate expected compacted file size + * @param operation Operation type + * @return Expected file size of SSTable after compaction */ - public long getExpectedCompactedFileSize(Iterable<SSTableReader> sstables) + public long getExpectedCompactedFileSize(Iterable<SSTableReader> sstables, OperationType operation) { long expectedFileSize = 0; - for (SSTableReader sstable : sstables) + if (operation == OperationType.CLEANUP) + { + Collection<Range<Token>> ranges = StorageService.instance.getLocalRanges(table.name); + for (SSTableReader sstable : sstables) + { + List<Pair<Long, Long>> positions = sstable.getPositionsForRanges(ranges); + for (Pair<Long, Long> position : positions) + expectedFileSize += position.right - position.left; + } + } + else { - long size = sstable.onDiskLength(); - expectedFileSize = expectedFileSize + size; + for (SSTableReader sstable : sstables) + expectedFileSize += sstable.onDiskLength(); } return expectedFileSize; } http://git-wip-us.apache.org/repos/asf/cassandra/blob/5979bce8/src/java/org/apache/cassandra/db/compaction/CompactionManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java index fd14593..03f1c18 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@ -693,7 +693,7 @@ public class CompactionManager implements CompactionManagerMBean logger.info("Cleaning up " + sstable); // Calculate the expected compacted filesize - long expectedRangeFileSize = cfs.getExpectedCompactedFileSize(Arrays.asList(sstable)) / 2; + long expectedRangeFileSize = cfs.getExpectedCompactedFileSize(Arrays.asList(sstable), OperationType.CLEANUP); File compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(expectedRangeFileSize); if (compactionFileLocation == null) throw new IOException("disk full"); http://git-wip-us.apache.org/repos/asf/cassandra/blob/5979bce8/src/java/org/apache/cassandra/db/compaction/CompactionTask.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java index 6d914e3..02fadb1 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java @@ -74,7 +74,7 @@ public class CompactionTask extends AbstractCompactionTask if (!isCompactionInteresting(toCompact)) return 0; - File compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact)); + File compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact, compactionType)); if (compactionFileLocation == null && partialCompactionsAcceptable()) { // If the compaction file path is null that means we have no space left for this compaction. @@ -85,7 +85,7 @@ public class CompactionTask extends AbstractCompactionTask // Note that we have removed files that are still marked as compacting. // This suboptimal but ok since the caller will unmark all the sstables at the end. toCompact.remove(cfs.getMaxSizeFile(toCompact)); - compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact)); + compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact, compactionType)); } }
