Repository: cassandra Updated Branches: refs/heads/trunk 2d76bd4c6 -> 0f1e838d1
Estimate compressed on disk size when deciding if sstable size limit reached Patch by Tom Petracca; reviewed by marcuse for CASSANDRA-11623 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0f1e838d Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0f1e838d Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0f1e838d Branch: refs/heads/trunk Commit: 0f1e838d17ac90d4d44e3369309b245afa969f33 Parents: 2d76bd4 Author: Tom Petracca <[email protected]> Authored: Mon Apr 18 16:37:00 2016 -0400 Committer: Marcus Eriksson <[email protected]> Committed: Wed May 11 10:43:44 2016 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../compaction/writers/MajorLeveledCompactionWriter.java | 7 +++---- .../db/compaction/writers/MaxSSTableSizeWriter.java | 2 +- .../writers/SplittingSizeTieredCompactionWriter.java | 3 +-- .../io/compress/CompressedSequentialWriter.java | 11 +++++++++++ .../cassandra/io/sstable/format/SSTableWriter.java | 5 +++++ .../cassandra/io/sstable/format/big/BigTableWriter.java | 5 +++++ .../org/apache/cassandra/io/util/SequentialWriter.java | 5 +++++ 8 files changed, 32 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 0b70d77..a9421b6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.8 + * Estimate compressed on disk size when deciding if sstable size limit reached (CASSANDRA-11623) * cassandra-stress profiles should support case sensitive schemas (CASSANDRA-11546) * Remove DatabaseDescriptor dependency from FileUtils (CASSANDRA-11578) * Faster streaming (CASSANDRA-9766) http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java index 91a18da..6cccfcb 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.db.compaction.writers; -import java.util.List; import java.util.Set; import org.apache.cassandra.db.ColumnFamilyStore; @@ -82,12 +81,12 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter @SuppressWarnings("resource") public boolean realAppend(UnfilteredRowIterator partition) { - long posBefore = sstableWriter.currentWriter().getOnDiskFilePointer(); RowIndexEntry rie = sstableWriter.append(partition); - totalWrittenInLevel += sstableWriter.currentWriter().getOnDiskFilePointer() - posBefore; partitionsWritten++; - if (sstableWriter.currentWriter().getOnDiskFilePointer() > maxSSTableSize) + long totalWrittenInCurrentWriter = sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten(); + if (totalWrittenInCurrentWriter > maxSSTableSize) { + totalWrittenInLevel += totalWrittenInCurrentWriter; if (totalWrittenInLevel > LeveledManifest.maxBytesForLevel(currentLevel, maxSSTableSize)) { totalWrittenInLevel = 0; http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java index 8c3ada7..864185e 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java @@ -96,7 +96,7 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter protected boolean realAppend(UnfilteredRowIterator partition) { RowIndexEntry rie = sstableWriter.append(partition); - if (sstableWriter.currentWriter().getOnDiskFilePointer() > maxSSTableSize) + if (sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten() > maxSSTableSize) { switchCompactionLocation(sstableDirectory); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java index 290e88b..46cb891 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java @@ -18,7 +18,6 @@ package org.apache.cassandra.db.compaction.writers; import java.util.Arrays; -import java.util.List; import java.util.Set; import org.slf4j.Logger; @@ -89,7 +88,7 @@ public class SplittingSizeTieredCompactionWriter extends CompactionAwareWriter public boolean realAppend(UnfilteredRowIterator partition) { RowIndexEntry rie = sstableWriter.append(partition); - if (sstableWriter.currentWriter().getOnDiskFilePointer() > currentBytesToWrite && currentRatioIndex < ratios.length - 1) // if we underestimate how many keys we have, the last sstable might get more than we expect + if (sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten() > currentBytesToWrite && currentRatioIndex < ratios.length - 1) // if we underestimate how many keys we have, the last sstable might get more than we expect { currentRatioIndex++; currentBytesToWrite = Math.round(totalSize * ratios[currentRatioIndex]); http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java index 9bd1145..1f33d53 100644 --- a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java +++ b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java @@ -92,6 +92,17 @@ public class CompressedSequentialWriter extends SequentialWriter } } + /** + * Get a quick estimation on how many bytes have been written to disk + * + * It should for the most part be exactly the same as getOnDiskFilePointer() + */ + @Override + public long getEstimatedOnDiskBytesWritten() + { + return chunkOffset; + } + @Override public void flush() { http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java index 6aaf776..9f2e159 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java @@ -212,6 +212,11 @@ public abstract class SSTableWriter extends SSTable implements Transactional public abstract long getOnDiskFilePointer(); + public long getEstimatedOnDiskBytesWritten() + { + return getOnDiskFilePointer(); + } + public abstract void resetAndTruncate(); public SSTableWriter setRepairedAt(long repairedAt) http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java index 39dc889..8645158 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java @@ -394,6 +394,11 @@ public class BigTableWriter extends SSTableWriter return dataFile.getOnDiskFilePointer(); } + public long getEstimatedOnDiskBytesWritten() + { + return dataFile.getEstimatedOnDiskBytesWritten(); + } + /** * Encapsulates writing the index and filter for an SSTable. The state of this object is not valid until it has been closed. */ http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/io/util/SequentialWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/SequentialWriter.java b/src/java/org/apache/cassandra/io/util/SequentialWriter.java index f153b4d..45e4cfa 100644 --- a/src/java/org/apache/cassandra/io/util/SequentialWriter.java +++ b/src/java/org/apache/cassandra/io/util/SequentialWriter.java @@ -276,6 +276,11 @@ public class SequentialWriter extends BufferedDataOutputStreamPlus implements Tr return position(); } + public long getEstimatedOnDiskBytesWritten() + { + return getOnDiskFilePointer(); + } + public long length() { try
