Improve memory usage of metadata min/max column names patch by Oleg Anastasyev; reviewed by jbellis for CASSANDRA-6077
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7e59db22 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7e59db22 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7e59db22 Branch: refs/heads/trunk Commit: 7e59db22bdba93850a9877e6be021e29be9b9fef Parents: 05952e7 Author: Jonathan Ellis <[email protected]> Authored: Mon Sep 23 09:13:10 2013 -0500 Committer: Jonathan Ellis <[email protected]> Committed: Mon Sep 23 09:13:10 2013 -0500 ---------------------------------------------------------------------- CHANGES.txt | 4 +++ .../cassandra/io/sstable/ColumnNameHelper.java | 26 ++++++++++++++------ .../apache/cassandra/utils/ByteBufferUtil.java | 6 +++++ 3 files changed, 28 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/7e59db22/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index f239b5a..6df5559 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,7 @@ +2.0.2 + * Improve memory usage of metadata min/max column names (CASSANDRA-6077) + + 2.0.1 * Fix bug that could allow reading deleted data temporarily (CASSANDRA-6025) * Improve memory use defaults (CASSANDRA-5069) http://git-wip-us.apache.org/repos/asf/cassandra/blob/7e59db22/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java b/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java index 2ad1cff..aaed765 100644 --- a/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java +++ b/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java @@ -27,6 +27,8 @@ import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.ColumnToCollectionType; import org.apache.cassandra.db.marshal.CompositeType; +import static org.apache.cassandra.utils.ByteBufferUtil.minimalBufferFor; + public class ColumnNameHelper { /** @@ -174,7 +176,7 @@ public class ColumnNameHelper public static List<ByteBuffer> mergeMin(List<ByteBuffer> minColumnNames, List<ByteBuffer> candidates, AbstractType<?> columnNameComparator) { if (minColumnNames.isEmpty()) - return candidates; + return minimalBuffersFor(candidates); if (candidates.isEmpty()) return minColumnNames; @@ -190,18 +192,26 @@ public class ColumnNameHelper List<ByteBuffer> retList = new ArrayList<ByteBuffer>(maxSize); for (int i = 0; i < minSize; i++) - retList.add(min(minColumnNames.get(i), candidates.get(i), ct.types.get(i))); + retList.add(minimalBufferFor(min(minColumnNames.get(i), candidates.get(i), ct.types.get(i)))); for (int i = minSize; i < maxSize; i++) - retList.add(biggest.get(i)); + retList.add(minimalBufferFor(biggest.get(i))); return retList; } else { - return Collections.singletonList(min(minColumnNames.get(0), candidates.get(0), columnNameComparator)); + return Collections.singletonList(minimalBufferFor(min(minColumnNames.get(0), candidates.get(0), columnNameComparator))); } } + private static List<ByteBuffer> minimalBuffersFor(List<ByteBuffer> candidates) + { + List<ByteBuffer> minimalBuffers = new ArrayList<ByteBuffer>(candidates.size()); + for (ByteBuffer byteBuffer : candidates) + minimalBuffers.add(minimalBufferFor(byteBuffer)); + return minimalBuffers; + } + /** * if columnNameComparator is CompositeType the columns are compared by components using the subcomparator * on the same position. @@ -217,7 +227,7 @@ public class ColumnNameHelper public static List<ByteBuffer> mergeMax(List<ByteBuffer> maxColumnNames, List<ByteBuffer> candidates, AbstractType<?> columnNameComparator) { if (maxColumnNames.isEmpty()) - return candidates; + return minimalBuffersFor(candidates); if (candidates.isEmpty()) return maxColumnNames; @@ -232,15 +242,15 @@ public class ColumnNameHelper List<ByteBuffer> retList = new ArrayList<ByteBuffer>(maxSize); for (int i = 0; i < minSize; i++) - retList.add(max(maxColumnNames.get(i), candidates.get(i), ct.types.get(i))); + retList.add(minimalBufferFor(max(maxColumnNames.get(i), candidates.get(i), ct.types.get(i)))); for (int i = minSize; i < maxSize; i++) - retList.add(biggest.get(i)); + retList.add(minimalBufferFor(biggest.get(i))); return retList; } else { - return Collections.singletonList(max(maxColumnNames.get(0), candidates.get(0), columnNameComparator)); + return Collections.singletonList(minimalBufferFor(max(maxColumnNames.get(0), candidates.get(0), columnNameComparator))); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/7e59db22/src/java/org/apache/cassandra/utils/ByteBufferUtil.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java index da8b958..4471fb9 100644 --- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java +++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java @@ -552,4 +552,10 @@ public class ByteBufferUtil int diff = value.remaining() - prefix.remaining(); return prefix.equals(value.duplicate().limit(value.remaining() - diff)); } + + /** trims size of bytebuffer to exactly number of bytes in it, to do not hold too much memory */ + public static ByteBuffer minimalBufferFor(ByteBuffer buf) + { + return buf.capacity() > buf.remaining() ? ByteBuffer.wrap(getArray(buf)) : buf; + } }
