Fix LCS splitting sstables based on uncompressed length patch by slebresne; reviewed by jbellis for CASSANDRA-4419
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/787e0e64 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/787e0e64 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/787e0e64 Branch: refs/heads/trunk Commit: 787e0e648854634d5457dde6792acfb3d97c3b66 Parents: 51a9fd1 Author: Sylvain Lebresne <[email protected]> Authored: Thu Jul 12 11:05:11 2012 +0200 Committer: Sylvain Lebresne <[email protected]> Committed: Thu Jul 12 11:05:11 2012 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cassandra/db/compaction/CompactionTask.java | 4 ++-- .../db/compaction/LeveledCompactionTask.java | 4 ++-- .../io/compress/CompressedSequentialWriter.java | 6 ++++++ .../apache/cassandra/io/sstable/SSTableWriter.java | 5 +++++ .../apache/cassandra/io/util/SequentialWriter.java | 12 ++++++++++++ 6 files changed, 28 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/787e0e64/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index d3cca6c..66a7b9e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -10,6 +10,7 @@ * Oversize integer in CQL throws NumberFormatException (CASSANDRA-4291) * Set gc_grace on index CF to 0 (CASSANDRA-4314) * fix 1.0.x node join to mixed version cluster, other nodes >= 1.1 (CASSANDRA-4195) + * Fix LCS splitting sstable base on uncompressed size (CASSANDRA-4419) 1.0.10 http://git-wip-us.apache.org/repos/asf/cassandra/blob/787e0e64/src/java/org/apache/cassandra/db/compaction/CompactionTask.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java index 2a1b415..f554c7b 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java @@ -169,7 +169,7 @@ public class CompactionTask extends AbstractCompactionTask } } } - if (!nni.hasNext() || newSSTableSegmentThresholdReached(writer, position)) + if (!nni.hasNext() || newSSTableSegmentThresholdReached(writer)) { SSTableReader toIndex = writer.closeAndOpenReader(getMaxDataAge(toCompact)); cachedKeyMap.put(toIndex, cachedKeys); @@ -229,7 +229,7 @@ public class CompactionTask extends AbstractCompactionTask } //extensibility point for other strategies that may want to limit the upper bounds of the sstable segment size - protected boolean newSSTableSegmentThresholdReached(SSTableWriter writer, long position) + protected boolean newSSTableSegmentThresholdReached(SSTableWriter writer) throws IOException { return false; } http://git-wip-us.apache.org/repos/asf/cassandra/blob/787e0e64/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java index b0b605a..d872970 100644 --- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java @@ -69,9 +69,9 @@ public class LeveledCompactionTask extends CompactionTask } @Override - protected boolean newSSTableSegmentThresholdReached(SSTableWriter writer, long position) + protected boolean newSSTableSegmentThresholdReached(SSTableWriter writer) throws IOException { - return position > sstableSizeInMB * 1024L * 1024L; + return writer.getOnDiskFilePointer() > sstableSizeInMB * 1024L * 1024L; } @Override http://git-wip-us.apache.org/repos/asf/cassandra/blob/787e0e64/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java index 545b09e..702a343 100644 --- a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java +++ b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java @@ -69,6 +69,12 @@ public class CompressedSequentialWriter extends SequentialWriter } @Override + public long getOnDiskFilePointer() throws IOException + { + return out.getFilePointer(); + } + + @Override public void sync() throws IOException { throw new UnsupportedOperationException(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/787e0e64/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java index 53980ed..f08bb49 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java @@ -382,6 +382,11 @@ public class SSTableWriter extends SSTable return dataFile.getFilePointer(); } + public long getOnDiskFilePointer() throws IOException + { + return dataFile.getOnDiskFilePointer(); + } + /** * Encapsulates writing the index and filter for an SSTable. The state of this object is not valid until it has been closed. */ http://git-wip-us.apache.org/repos/asf/cassandra/blob/787e0e64/src/java/org/apache/cassandra/io/util/SequentialWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/SequentialWriter.java b/src/java/org/apache/cassandra/io/util/SequentialWriter.java index 79af8c1..dce9bc6 100644 --- a/src/java/org/apache/cassandra/io/util/SequentialWriter.java +++ b/src/java/org/apache/cassandra/io/util/SequentialWriter.java @@ -211,6 +211,18 @@ public class SequentialWriter extends OutputStream return current; } + /** + * Return the current file pointer of the underlying on-disk file. + * Note that since write works by buffering data, the value of this will increase by buffer + * size and not every write to the writer will modify this value. + * Furthermore, for compressed files, this value refers to compressed data, while the + * writer getFilePointer() refers to uncompressedFile + */ + public long getOnDiskFilePointer() throws IOException + { + return getFilePointer(); + } + public long length() throws IOException { return Math.max(Math.max(current, out.length()), bufferOffset + validBufferBytes);
