Updated Branches: refs/heads/trunk 1a2f35731 -> 07ca445e5
Don't keep ancestor information in memory since it is only used on startup. patch by marcuse, reviewed by jbellis for CASSANDRA-5342 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/83b75754 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/83b75754 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/83b75754 Branch: refs/heads/trunk Commit: 83b75754ff143d4d77b01ef76a813da47779c6f4 Parents: cc9ac82 Author: Marcus Eriksson <marc...@spotify.com> Authored: Mon Jul 8 09:55:34 2013 +0200 Committer: Marcus Eriksson <marc...@spotify.com> Committed: Mon Jul 8 09:57:11 2013 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cassandra/io/sstable/SSTableMetadata.java | 39 ++++++++++++++------ .../cassandra/io/sstable/SSTableReader.java | 12 +++++- .../cassandra/io/sstable/SSTableWriter.java | 6 +-- .../cassandra/tools/SSTableMetadataViewer.java | 2 +- .../sstable/SSTableMetadataSerializerTest.java | 13 +++++-- 6 files changed, 52 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/83b75754/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index fa4df36..b08f967 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ (CASSANDRA-5692) * Fix skipping range tombstones with reverse queries (CASSANDRA-5712) * Expire entries out of ThriftSessionManager (CASSANRDA-5719) + * Don't keep ancestor information in memory (CASSANDRA-5342) 1.2.6 * Fix tracing when operation completes before all responses arrive http://git-wip-us.apache.org/repos/asf/cassandra/blob/83b75754/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java b/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java index 865c2ea..de51ea9 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java @@ -20,6 +20,7 @@ package org.apache.cassandra.io.sstable; import java.io.*; import java.util.*; +import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.StreamingHistogram; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,7 +56,6 @@ public class SSTableMetadata public final long maxTimestamp; public final double compressionRatio; public final String partitioner; - public final Set<Integer> ancestors; public final StreamingHistogram estimatedTombstoneDropTime; private SSTableMetadata() @@ -67,7 +67,6 @@ public class SSTableMetadata Long.MAX_VALUE, NO_COMPRESSION_RATIO, null, - Collections.<Integer>emptySet(), defaultTombstoneDropTimeHistogram()); } @@ -78,7 +77,6 @@ public class SSTableMetadata long maxTimestamp, double cr, String partitioner, - Set<Integer> ancestors, StreamingHistogram estimatedTombstoneDropTime) { this.estimatedRowSize = rowSizes; @@ -88,7 +86,6 @@ public class SSTableMetadata this.maxTimestamp = maxTimestamp; this.compressionRatio = cr; this.partitioner = partitioner; - this.ancestors = ancestors; this.estimatedTombstoneDropTime = estimatedTombstoneDropTime; } @@ -193,7 +190,6 @@ public class SSTableMetadata maxTimestamp, compressionRatio, partitioner, - ancestors, estimatedTombstoneDropTime); } @@ -242,7 +238,7 @@ public class SSTableMetadata { private static final Logger logger = LoggerFactory.getLogger(SSTableMetadataSerializer.class); - public void serialize(SSTableMetadata sstableStats, DataOutput dos) throws IOException + public void serialize(SSTableMetadata sstableStats, Set<Integer> ancestors, DataOutput dos) throws IOException { assert sstableStats.partitioner != null; @@ -253,20 +249,31 @@ public class SSTableMetadata dos.writeLong(sstableStats.maxTimestamp); dos.writeDouble(sstableStats.compressionRatio); dos.writeUTF(sstableStats.partitioner); - dos.writeInt(sstableStats.ancestors.size()); - for (Integer g : sstableStats.ancestors) + dos.writeInt(ancestors.size()); + for (Integer g : ancestors) dos.writeInt(g); StreamingHistogram.serializer.serialize(sstableStats.estimatedTombstoneDropTime, dos); } - public SSTableMetadata deserialize(Descriptor descriptor) throws IOException + /** + * deserializes the metadata + * + * returns a pair containing the part of the metadata meant to be kept-in memory and the part + * that should not. + * + * @param descriptor the descriptor + * @return a pair containing data that needs to be in memory and data that is potentially big and is not needed + * in memory + * @throws IOException + */ + public Pair<SSTableMetadata, Set<Integer>> deserialize(Descriptor descriptor) throws IOException { logger.debug("Load metadata for {}", descriptor); File statsFile = new File(descriptor.filenameFor(SSTable.COMPONENT_STATS)); if (!statsFile.exists()) { logger.debug("No sstable stats for {}", descriptor); - return new SSTableMetadata(); + return Pair.create(new SSTableMetadata(), Collections.<Integer>emptySet()); } DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(statsFile))); @@ -280,7 +287,7 @@ public class SSTableMetadata } } - public SSTableMetadata deserialize(DataInputStream dis, Descriptor desc) throws IOException + public Pair<SSTableMetadata, Set<Integer>> deserialize(DataInputStream dis, Descriptor desc) throws IOException { EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(dis); EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(dis); @@ -308,7 +315,15 @@ public class SSTableMetadata StreamingHistogram tombstoneHistogram = desc.version.tracksTombstones ? StreamingHistogram.serializer.deserialize(dis) : defaultTombstoneDropTimeHistogram(); - return new SSTableMetadata(rowSizes, columnCounts, replayPosition, minTimestamp, maxTimestamp, compressionRatio, partitioner, ancestors, tombstoneHistogram); + return Pair.create(new SSTableMetadata(rowSizes, + columnCounts, + replayPosition, + minTimestamp, + maxTimestamp, + compressionRatio, + partitioner, + tombstoneHistogram), + ancestors); } } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/83b75754/src/java/org/apache/cassandra/io/sstable/SSTableReader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java index ea9c451..ab58c11 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java @@ -168,7 +168,7 @@ public class SSTableReader extends SSTable long start = System.currentTimeMillis(); logger.info("Opening {} ({} bytes)", descriptor, new File(descriptor.filenameFor(COMPONENT_DATA)).length()); - SSTableMetadata sstableMetadata = SSTableMetadata.serializer.deserialize(descriptor); + SSTableMetadata sstableMetadata = SSTableMetadata.serializer.deserialize(descriptor).left; // Check if sstable is created using same partitioner. // Partitioner can be null, which indicates older version of sstable or no stats available. @@ -1116,7 +1116,15 @@ public class SSTableReader extends SSTable public Set<Integer> getAncestors() { - return sstableMetadata.ancestors; + try + { + return SSTableMetadata.serializer.deserialize(descriptor).right; + } + catch (IOException e) + { + SSTableReader.logOpenException(descriptor, e); + return Collections.emptySet(); + } } public RandomAccessReader openDataReader(RateLimiter limiter) http://git-wip-us.apache.org/repos/asf/cassandra/blob/83b75754/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java index 51aeaa6..54e154c 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java @@ -328,7 +328,7 @@ public class SSTableWriter extends SSTable dataFile.close(); // write sstable statistics SSTableMetadata sstableMetadata = sstableMetadataCollector.finalizeMetadata(partitioner.getClass().getCanonicalName()); - writeMetadata(descriptor, sstableMetadata); + writeMetadata(descriptor, sstableMetadata, sstableMetadataCollector.ancestors); maybeWriteDigest(); // save the table of components @@ -381,12 +381,12 @@ public class SSTableWriter extends SSTable out.close(); } - private static void writeMetadata(Descriptor desc, SSTableMetadata sstableMetadata) + private static void writeMetadata(Descriptor desc, SSTableMetadata sstableMetadata, Set<Integer> ancestors) { SequentialWriter out = SequentialWriter.open(new File(desc.filenameFor(SSTable.COMPONENT_STATS)), true); try { - SSTableMetadata.serializer.serialize(sstableMetadata, out.stream); + SSTableMetadata.serializer.serialize(sstableMetadata, ancestors, out.stream); } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/83b75754/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java index c1f0332..55f2769 100644 --- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java +++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java @@ -44,7 +44,7 @@ public class SSTableMetadataViewer for (String fname : args) { Descriptor descriptor = Descriptor.fromFilename(fname); - SSTableMetadata metadata = SSTableMetadata.serializer.deserialize(descriptor); + SSTableMetadata metadata = SSTableMetadata.serializer.deserialize(descriptor).left; out.printf("SSTable: %s%n", descriptor); out.printf("Partitioner: %s%n", metadata.partitioner); http://git-wip-us.apache.org/repos/asf/cassandra/blob/83b75754/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java index ab96f89..3e8ef42 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java @@ -24,12 +24,16 @@ import java.io.DataOutputStream; import java.io.DataInputStream; import java.io.File; import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; import org.junit.Test; import org.apache.cassandra.db.commitlog.ReplayPosition; import org.apache.cassandra.dht.RandomPartitioner; import org.apache.cassandra.utils.EstimatedHistogram; +import org.apache.cassandra.utils.Pair; public class SSTableMetadataSerializerTest { @@ -56,13 +60,15 @@ public class SSTableMetadataSerializerTest ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(byteOutput); - - SSTableMetadata.serializer.serialize(originalMetadata, dos); + Set<Integer> ancestors = new HashSet<Integer>(); + ancestors.addAll(Arrays.asList(1,2,3,4)); + SSTableMetadata.serializer.serialize(originalMetadata, ancestors, dos); ByteArrayInputStream byteInput = new ByteArrayInputStream(byteOutput.toByteArray()); DataInputStream dis = new DataInputStream(byteInput); Descriptor desc = new Descriptor(Descriptor.Version.CURRENT, new File("."), "", "", 0, false); - SSTableMetadata stats = SSTableMetadata.serializer.deserialize(dis, desc); + Pair<SSTableMetadata, Set<Integer>> statsPair = SSTableMetadata.serializer.deserialize(dis, desc); + SSTableMetadata stats = statsPair.left; assert stats.estimatedRowSize.equals(originalMetadata.estimatedRowSize); assert stats.estimatedRowSize.equals(rowSizes); @@ -75,5 +81,6 @@ public class SSTableMetadataSerializerTest assert stats.minTimestamp == originalMetadata.minTimestamp; assert stats.maxTimestamp == originalMetadata.maxTimestamp; assert RandomPartitioner.class.getCanonicalName().equals(stats.partitioner); + assert ancestors.equals(statsPair.right); } }
