This is an automated email from the ASF dual-hosted git repository.
konstantinov pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new a3a834ad31 Reduce memory footprint of IndexSummaryTest large summary
tests
a3a834ad31 is described below
commit a3a834ad31907f0401489675390656b2dc570425
Author: samlightfoot <[email protected]>
AuthorDate: Thu Feb 12 21:27:38 2026 +0000
Reduce memory footprint of IndexSummaryTest large summary tests
The runtime overflow guard test (testLargeIndexSummary) required ~2 GiB of
off-heap memory to trigger the Integer.MAX_VALUE boundary in
IndexSummaryBuilder.maybeAddEntry. SafeMemoryWriter's 50% buffer growth
strategy caused ~5 GiB peak memory during reallocation (old + new buffer),
exceeding CI medium node limits (3.5-5 GB).
Extract the hardcoded Integer.MAX_VALUE threshold in IndexSummaryBuilder
into a @VisibleForTesting field (maxEntriesSize), allowing the test to
exercise the same guard logic with a 2 MB threshold instead. Rename both
large summary tests to reflect their distinct code paths: runtime truncation
vs constructor-initiated downsampling.
patch by Sam Lightfoot; reviewed by Brandon Williams,Dmitry
Konstantinov,Michael Semb Wever for CASSANDRA-20599
---
.../sstable/indexsummary/IndexSummaryBuilder.java | 15 ++++++----
.../io/sstable/indexsummary/IndexSummaryTest.java | 35 ++++++++++++++--------
2 files changed, 33 insertions(+), 17 deletions(-)
diff --git
a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java
b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java
index 65e8d1b399..6c9699a013 100644
---
a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java
+++
b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java
@@ -22,6 +22,8 @@ import java.nio.ByteOrder;
import java.util.Map;
import java.util.TreeMap;
+import com.google.common.annotations.VisibleForTesting;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,6 +43,9 @@ public class IndexSummaryBuilder implements AutoCloseable
static long defaultExpectedKeySize =
INDEX_SUMMARY_EXPECTED_KEY_SIZE.getLong();
+ @VisibleForTesting
+ static long maxEntriesSize = Integer.MAX_VALUE;
+
// the offset in the keys memory region to look for a given summary
boundary
private final SafeMemoryWriter offsets;
private final SafeMemoryWriter entries;
@@ -105,13 +110,13 @@ public class IndexSummaryBuilder implements AutoCloseable
long expectedEntrySize = getEntrySize(defaultExpectedKeySize);
long maxExpectedEntries = expectedKeys / minIndexInterval;
long maxExpectedEntriesSize = maxExpectedEntries * expectedEntrySize;
- if (maxExpectedEntriesSize > Integer.MAX_VALUE)
+ if (maxExpectedEntriesSize > maxEntriesSize)
{
// that's a _lot_ of keys, and a very low min index interval
- int effectiveMinInterval = (int) Math.ceil((double)(expectedKeys *
expectedEntrySize) / Integer.MAX_VALUE);
+ int effectiveMinInterval = (int) Math.ceil((double)(expectedKeys *
expectedEntrySize) / maxEntriesSize);
maxExpectedEntries = expectedKeys / effectiveMinInterval;
maxExpectedEntriesSize = maxExpectedEntries * expectedEntrySize;
- assert maxExpectedEntriesSize <= Integer.MAX_VALUE :
maxExpectedEntriesSize;
+ assert maxExpectedEntriesSize <= maxEntriesSize :
maxExpectedEntriesSize;
logger.warn("min_index_interval of {} is too low for {} expected
keys of avg size {}; using interval of {} instead",
minIndexInterval, expectedKeys,
defaultExpectedKeySize, effectiveMinInterval);
this.minIndexInterval = effectiveMinInterval;
@@ -198,7 +203,7 @@ public class IndexSummaryBuilder implements AutoCloseable
{
if (keysWritten == nextSamplePosition)
{
- if ((entries.length() + getEntrySize(length)) <= Integer.MAX_VALUE)
+ if ((entries.length() + getEntrySize(length)) <= maxEntriesSize)
{
offsets.writeInt((int) entries.length());
entries.write(keyBytes, offset, length);
@@ -229,7 +234,7 @@ public class IndexSummaryBuilder implements AutoCloseable
{
if (keysWritten == nextSamplePosition)
{
- if ((entries.length() + getEntrySize(decoratedKey)) <=
Integer.MAX_VALUE)
+ if ((entries.length() + getEntrySize(decoratedKey)) <=
maxEntriesSize)
{
offsets.writeInt((int) entries.length());
entries.write(decoratedKey.getKey());
diff --git
a/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java
b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java
index 451c25aaa7..456fc93463 100644
---
a/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java
+++
b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java
@@ -122,20 +122,26 @@ public class IndexSummaryTest
}
/**
- * Test an index summary whose total size is bigger than 2GiB,
- * the index summary builder should log an error but it should still
- * create an index summary, albeit one that does not cover the entire
sstable.
+ * Test that the index summary builder truncates (stops adding entries)
when the entries buffer
+ * would exceed the max capacity at runtime. The builder doesn't predict
the overflow because
+ * defaultExpectedKeySize is smaller than the actual key size, so the
constructor guard doesn't
+ * trigger. Instead, the runtime guard in maybeAddEntry detects the
overflow and stops accepting
+ * entries, producing a partial summary that covers the beginning of the
sstable.
*/
@Test
- public void testLargeIndexSummary() throws IOException
+ public void testLargeIndexSummaryTruncatesWhenOverflowDetectedAtRuntime()
throws IOException
{
- // On Circle CI we normally don't have enough off-heap memory for this
test so ignore it
- Assume.assumeTrue(CIRCLECI.getString() == null);
-
- final int numKeys = 1000000;
+ final int numKeys = 1000;
final int keySize = 3000;
final int minIndexInterval = 1;
+ // Use a small max so the test doesn't need GiBs of off-heap memory.
+ // 1000 keys × 3008 bytes/entry = 3,008,000 > 2,000,000, so the
runtime guard triggers.
+ // defaultExpectedKeySize (64) gives estimated total of 72,000 <
2,000,000, so the
+ // constructor guard does not trigger and minIndexInterval stays at 1.
+ long oldMaxEntriesSize = IndexSummaryBuilder.maxEntriesSize;
+ IndexSummaryBuilder.maxEntriesSize = 2_000_000;
+
try (IndexSummaryBuilder builder = new IndexSummaryBuilder(numKeys,
minIndexInterval, BASE_SAMPLING_LEVEL))
{
for (int i = 0; i < numKeys; i++)
@@ -153,15 +159,20 @@ public class IndexSummaryTest
assertEquals(numKeys + 1, indexSummary.getEstimatedKeyCount());
}
}
+ finally
+ {
+ IndexSummaryBuilder.maxEntriesSize = oldMaxEntriesSize;
+ }
}
/**
- * Test an index summary whose total size is bigger than 2GiB,
- * having updated IndexSummaryBuilder.defaultExpectedKeySize to match the
size,
- * the index summary should be downsampled automatically.
+ * Test that the index summary builder downsamples (increases
minIndexInterval) when the
+ * estimated total entry size exceeds max capacity at construction time.
With defaultExpectedKeySize
+ * matching the actual key size, the constructor detects the overflow
upfront and adjusts
+ * minIndexInterval so the summary covers the entire sstable at a coarser
interval.
*/
@Test
- public void testLargeIndexSummaryWithExpectedSizeMatching() throws
IOException
+ public void
testLargeIndexSummaryDownsamplesWhenOverflowDetectedAtConstruction() throws
IOException
{
// On Circle CI we normally don't have enough off-heap memory for this
test so ignore it
Assume.assumeTrue(CIRCLECI.getString() == null);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
