This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch cassandra-5.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-5.0 by this push:
new e8b7eac7ed Bound and chunk the BTI partition index preload
e8b7eac7ed is described below
commit e8b7eac7ed1e62a8d3cccb014c6c4b3f9f0a33c4
Author: Alexandre Viau <[email protected]>
AuthorDate: Mon Sep 28 09:14:11 2026 -0400
Bound and chunk the BTI partition index preload
Opening a BTI SSTable whose bloom filter is uninformative warms the whole
Partitions.db, one byte per page through a reader that follows
disk_access_mode. That
costs one pread or one readahead-bounded page fault per page: on a 309 GiB
index,
81.1M reads at 50 MiB/s regardless of access mode. It also warms far more
than the
available page cache can hold.
Add cassandra.bti.partition_index_preload_size to bound how much is warmed.
The tail
is warmed because the trie is written bottom-up, so the upper levels
traversed by
every lookup are at the end of the file, while the bulk at the front is
leaf pages
that a lookup reads one of.
The property takes a human-readable size, e.g. 512MiB. 0B skips warming
entirely,
which was not previously possible: preload is enabled whenever the bloom
filter is
uninformative, so the only way to avoid it was to lower
bloom_filter_fp_chance below
1.0. A negative value warms the whole index and is the default, so
behaviour is
unchanged unless the property is set.
The warmed range is read in 1 MiB positioned reads on the channel rather
than one
byte per page, which are unaffected by disk_access_mode; page cache is
shared between
the buffered and mapped views of a file, so a mapped lookup still finds the
data
resident.
The trade-offs: the whole range is copied into userspace rather than one
byte per
page, and under a mapped mode the first lookup takes a minor fault the walk
had
already resolved. Under disk_access_mode: standard the warm no longer
populates the
chunk cache, nor evicts it; lookups fill it on first touch instead.
Assisted-by: Claude Code:claude-opus-5
patch by Alexandre Viau; reviewed by Brandon Williams, Branimir Lambov for
CASSANDRA-21632
---
CHANGES.txt | 1 +
.../config/CassandraRelevantProperties.java | 2 +
.../io/sstable/format/bti/PartitionIndex.java | 77 ++++++++++++++++++----
.../io/sstable/format/bti/PartitionIndexTest.java | 45 +++++++++++++
4 files changed, 114 insertions(+), 11 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 65e27fa29b..1cdbf799c1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
5.0.10
+ * Bound and chunk the BTI partition index preload (CASSANDRA-21632)
* Restore volatile on CommitLogSegment.cdcState dropped in merge
(CASSANDRA-21665)
* Fix compressed scan read-ahead buffer Block leak and chunk-cache pollution
by scans (CASSANDRA-21671)
* A read on a replica that does not know the index yet fails with a
NullPointerException (CASSANDRA-21658)
diff --git
a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
index 8b8fac9270..1a9e2dfa43 100644
--- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
+++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
@@ -74,6 +74,8 @@ public enum CassandraRelevantProperties
*/
BOOTSTRAP_SKIP_SCHEMA_CHECK("cassandra.skip_schema_check"),
BROADCAST_INTERVAL_MS("cassandra.broadcast_interval_ms", "60000"),
+ /** Caps how much of a BTI partition index is preloaded, e.g. "16MiB". 0B
disables preloading; negative (default) preloads it all. */
+
BTI_PARTITION_INDEX_PRELOAD_SIZE("cassandra.bti.partition_index_preload_size",
"-1B"),
BTREE_BRANCH_SHIFT("cassandra.btree.branchshift", "5"),
BTREE_FAN_FACTOR("cassandra.btree.fanfactor"),
/** Represents the maximum size (in bytes) of a serialized mutation that
can be cached **/
diff --git
a/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndex.java
b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndex.java
index cfdfe37eed..a082547607 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndex.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndex.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.io.sstable.format.bti;
+import java.io.EOFException;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.ByteBuffer;
@@ -25,6 +26,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.IPartitioner;
@@ -37,7 +39,7 @@ import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.io.util.PageAware;
+import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.Rebufferer;
import org.apache.cassandra.io.util.SizedInts;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -78,6 +80,13 @@ public class PartitionIndex implements SharedCloseable
public static final int FOOTER_LENGTH = 3 * 8;
private static final int FLAG_HAS_HASH_BYTE = 8;
+ /**
+ * Chunk size used to warm the partition index in {@link #warmIndex}.
Large enough that reads are
+ * device-bound rather than bounded by per-request overhead, small enough
to be a negligible
+ * transient allocation per opening thread.
+ */
+ private static final int PRELOAD_CHUNK_SIZE = (int) FileUtils.ONE_MIB;
+
@VisibleForTesting
public PartitionIndex(FileHandle fh, long trieRoot, long keyCount,
DecoratedKey first, DecoratedKey last)
{
@@ -196,21 +205,67 @@ public class PartitionIndex implements SharedCloseable
DecoratedKey first = partitioner != null ?
partitioner.decorateKey(ByteBufferUtil.readWithShortLength(rdr)) : null;
DecoratedKey last = partitioner != null ?
partitioner.decorateKey(ByteBufferUtil.readWithShortLength(rdr)) : null;
if (preload)
- {
- int csum = 0;
- // force a read of all the pages of the index
- for (long pos = 0; pos < fh.dataLength(); pos +=
PageAware.PAGE_SIZE)
- {
- rdr.seek(pos);
- csum += rdr.readByte();
- }
- logger.trace("Checksum {}", csum); // Note: trace is
required so that reads aren't optimized away.
- }
+ warmIndex(fh, firstPos);
return new PartitionIndex(fh, root, keyCount, first, last);
}
}
+ /**
+ * Warms the partition index so trie lookups don't each fault it in.
+ *
+ * Chunked positioned reads on the channel, not one byte per page through
a {@link FileDataInput},
+ * which follows {@code disk_access_mode} and so costs a pread or a page
fault per page. A
+ * positioned read is unaffected by access mode, and page cache is shared
between the buffered and
+ * mapped views of a file, so a mapped lookup afterwards still finds the
data resident.
+ *
+ * When bounded, warms the trie's tail, since the trie is written
bottom-up and every lookup
+ * traverses the upper levels that end up there. The bound is relative to
{@code firstPos} (the
+ * trie's end), not the file's end, since the file tail past it isn't trie
data.
+ *
+ * @param firstPos offset where the trie ends and the first/last key data
begins
+ * @return the number of bytes read
+ * @see CassandraRelevantProperties#BTI_PARTITION_INDEX_PRELOAD_SIZE
+ */
+ @VisibleForTesting
+ static long warmIndex(FileHandle fh, long firstPos) throws EOFException
+ {
+ long limit =
CassandraRelevantProperties.BTI_PARTITION_INDEX_PRELOAD_SIZE.getSizeInBytes();
+ if (limit == 0)
+ {
+ logger.trace("Partition index warming is disabled, not warming
{}", fh.path());
+ return 0;
+ }
+
+ if (firstPos <= 0)
+ return 0;
+
+ long start = limit > 0 && limit < firstPos ? firstPos - limit : 0;
+
+ ByteBuffer buffer = ByteBuffer.allocateDirect((int)
Math.min(PRELOAD_CHUNK_SIZE, firstPos - start));
+ try
+ {
+ long pos = start;
+ while (pos < firstPos)
+ {
+ buffer.clear();
+ buffer.limit((int) Math.min(buffer.capacity(), firstPos -
pos));
+
+ // ChannelProxy.read may return a short read; advance by what
was actually read.
+ int read = fh.channel.read(buffer, pos);
+ if (read < 0)
+ throw new EOFException("Failed to read " + fh.path() + "
at position " + pos);
+
+ pos += read;
+ }
+ return pos - start;
+ }
+ finally
+ {
+ FileUtils.clean(buffer);
+ }
+ }
+
@Override
public void close()
{
diff --git
a/test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java
b/test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java
index 6ceee331f6..a9d01c95e2 100644
---
a/test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java
+++
b/test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java
@@ -53,9 +53,11 @@ import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.RandomPartitioner;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.io.tries.TrieNode;
import org.apache.cassandra.io.tries.Walker;
import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.io.util.FileHandle;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.PageAware;
@@ -67,6 +69,7 @@ import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import static
org.apache.cassandra.config.CassandraRelevantProperties.BTI_PARTITION_INDEX_PRELOAD_SIZE;
import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -128,6 +131,48 @@ public class PartitionIndexTest
testGetEq(generateSequentialIndex(COUNT));
}
+ @Test
+ public void testWarmIndexHonoursPreloadSize() throws IOException
+ {
+ Pair<List<DecoratedKey>, PartitionIndex> data =
generateRandomIndex(COUNT);
+ try (PartitionIndex index = data.right)
+ {
+ FileHandle fh = index.getFileHandle();
+ long firstPos;
+ try (FileDataInput rdr = fh.createReader(fh.dataLength() -
PartitionIndex.FOOTER_LENGTH))
+ {
+ firstPos = rdr.readLong();
+ }
+ assertTrue("Generated index is too small to bound: " + firstPos,
firstPos > 2 * PageAware.PAGE_SIZE);
+
+ // the default warms the whole trie
+ assertEquals(firstPos, PartitionIndex.warmIndex(fh, firstPos));
+
+ try (WithProperties ignored = new
WithProperties().set(BTI_PARTITION_INDEX_PRELOAD_SIZE, "-1B"))
+ {
+ assertEquals(firstPos, PartitionIndex.warmIndex(fh, firstPos));
+ }
+
+ // a bound at or above the trie length also warms the whole trie
+ try (WithProperties ignored = new
WithProperties().set(BTI_PARTITION_INDEX_PRELOAD_SIZE, (firstPos + 1) + "B"))
+ {
+ assertEquals(firstPos, PartitionIndex.warmIndex(fh, firstPos));
+ }
+
+ // a bound below the trie length warms exactly that much of the
tail
+ try (WithProperties ignored = new
WithProperties().set(BTI_PARTITION_INDEX_PRELOAD_SIZE, (firstPos / 2) + "B"))
+ {
+ assertEquals(firstPos / 2, PartitionIndex.warmIndex(fh,
firstPos));
+ }
+
+ // 0B disables warming entirely
+ try (WithProperties ignored = new
WithProperties().set(BTI_PARTITION_INDEX_PRELOAD_SIZE, "0B"))
+ {
+ assertEquals(0, PartitionIndex.warmIndex(fh, firstPos));
+ }
+ }
+ }
+
@Test
public void testBrokenFile() throws IOException
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
