This is an automated email from the ASF dual-hosted git repository.
KKcorps pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 607576c7a8b Fix endless range index rebuild for MV columns with
rangeIndexVersion 2 (#19936)
607576c7a8b is described below
commit 607576c7a8b6ea19dd7eda2e5614701738d1db1c
Author: Kartik Khare <[email protected]>
AuthorDate: Tue Oct 20 12:51:05 2026 +0530
Fix endless range index rebuild for MV columns with rangeIndexVersion 2
(#19936)
The BitSliced (v2) creator only builds for single-value columns, so MV
columns
always get a v1 index. RangeIndexHandler compared the on-disk version
against the
raw configured version, so needUpdateIndices stayed true and every reload
rebuilt
the v1 index. Compare against the version the creator actually builds
instead.
Co-authored-by: Kartik Khare <[email protected]>
---
.../loader/invertedindex/RangeIndexHandler.java | 22 +++-
.../RangeIndexHandlerMultiValueTest.java | 142 +++++++++++++++++++++
2 files changed, 160 insertions(+), 4 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
index a4c5f042553..a80321d86c8 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
@@ -25,6 +25,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.io.FileUtils;
import
org.apache.pinot.segment.local.segment.creator.impl.inv.BitSlicedRangeIndexCreator;
+import
org.apache.pinot.segment.local.segment.creator.impl.inv.RangeIndexCreator;
import
org.apache.pinot.segment.local.segment.index.dictionary.DictionaryIndexType;
import org.apache.pinot.segment.local.segment.index.loader.BaseIndexHandler;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
@@ -106,16 +107,29 @@ public class RangeIndexHandler extends BaseIndexHandler {
return rangeIndexUpdated;
}
- /// Returns `true` if the on-disk range index version doesn't match the
configured version. Range index v1
- /// (RangeIndexCreator) and v2 (BitSlicedRangeIndexCreator) have
incompatible on-disk layouts and serve
+ /// Returns `true` if the on-disk range index version doesn't match the
version the creator would build. Range index
+ /// v1 (RangeIndexCreator) and v2 (BitSlicedRangeIndexCreator) have
incompatible on-disk layouts and serve
/// different query semantics (v1 is non-exact, v2 is exact), so a version
change requires rebuild.
private boolean existingRangeIndexVersionDiffers(SegmentDirectory.Reader
segmentReader, String column)
throws Exception {
- int configuredVersion =
_fieldIndexConfigs.get(column).getConfig(StandardIndexes.range()).getVersion();
+ int expectedVersion = getEffectiveRangeIndexVersion(column);
// The buffer is owned by SegmentDirectory; don't close it here (mmap
regions are shared).
PinotDataBuffer rangeIndexBuffer = segmentReader.getIndexFor(column,
StandardIndexes.range());
int onDiskVersion = rangeIndexBuffer.getInt(0);
- return onDiskVersion != configuredVersion;
+ return onDiskVersion != expectedVersion;
+ }
+
+ /// Returns the range index version that `RangeIndexType.createIndexCreator`
actually builds for the column. It only
+ /// builds v2 for single-value columns and falls back to v1 for multi-value
ones, so comparing against the raw
+ /// configured version would make every reload rebuild a multi-value
column's v1 index.
+ private int getEffectiveRangeIndexVersion(String column) {
+ int configuredVersion =
_fieldIndexConfigs.get(column).getConfig(StandardIndexes.range()).getVersion();
+ ColumnMetadata columnMetadata =
_segmentDirectory.getSegmentMetadata().getColumnMetadataFor(column);
+ if (configuredVersion == BitSlicedRangeIndexCreator.VERSION &&
columnMetadata != null
+ && !columnMetadata.isSingleValue()) {
+ return RangeIndexCreator.VERSION;
+ }
+ return configuredVersion;
}
@Override
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandlerMultiValueTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandlerMultiValueTest.java
new file mode 100644
index 00000000000..b49dbf8643d
--- /dev/null
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandlerMultiValueTest.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.loader.invertedindex;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.io.FileUtils;
+import
org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import
org.apache.pinot.segment.local.segment.creator.impl.inv.RangeIndexCreator;
+import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
+import org.apache.pinot.segment.local.segment.store.SegmentLocalFSDirectory;
+import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.index.FieldIndexConfigs;
+import org.apache.pinot.segment.spi.index.RangeIndexConfig;
+import org.apache.pinot.segment.spi.index.StandardIndexes;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.readers.GenericRow;
+import org.apache.pinot.spi.utils.ReadMode;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+
+/// Tests [RangeIndexHandler] on a multi-value column configured with range
index version 2.
+///
+/// The BitSliced (v2) creator only supports single-value columns, so a
multi-value column always gets a v1 index.
+/// The handler must treat that v1 index as up to date, otherwise every reload
sees a "version change" and rebuilds it.
+public class RangeIndexHandlerMultiValueTest {
+ private static final File TEMP_DIR =
+ new File(FileUtils.getTempDirectory(),
RangeIndexHandlerMultiValueTest.class.getSimpleName());
+ private static final String SEGMENT_NAME = "testSegment";
+ private static final String MV_COLUMN = "mvIntCol";
+ private static final Map<String, FieldIndexConfigs> RANGE_V2_CONFIGS =
Map.of(MV_COLUMN,
+ new FieldIndexConfigs.Builder().add(StandardIndexes.range(), new
RangeIndexConfig(2)).build());
+
+ @Test
+ public void testNoRebuildAfterReloadForMultiValueColumnWithVersion2()
+ throws Exception {
+ File indexDir = buildSegment();
+
+ assertTrue(needUpdateIndices(indexDir), "Range index is absent on disk, so
the handler is expected to build it");
+ updateIndices(indexDir);
+
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir);
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(indexDir, segmentMetadata, ReadMode.mmap);
+ SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
+ assertEquals(reader.getIndexFor(MV_COLUMN,
StandardIndexes.range()).getInt(0), RangeIndexCreator.VERSION,
+ "Multi-value columns are expected to get a v1 range index even when
v2 is configured");
+ }
+
+ assertFalse(needUpdateIndices(indexDir), "The v1 index is what v2 config
builds for a multi-value column, "
+ + "so a second reload must not rebuild it");
+ }
+
+ private static boolean needUpdateIndices(File indexDir)
+ throws Exception {
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir);
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(indexDir, segmentMetadata, ReadMode.mmap);
+ SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
+ return new RangeIndexHandler(segmentDirectory, RANGE_V2_CONFIGS,
createTableConfig(), createSchema())
+ .needUpdateIndices(reader);
+ }
+ }
+
+ private static void updateIndices(File indexDir)
+ throws Exception {
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir);
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(indexDir, segmentMetadata, ReadMode.mmap);
+ SegmentDirectory.Writer writer = segmentDirectory.createWriter()) {
+ new RangeIndexHandler(segmentDirectory, RANGE_V2_CONFIGS,
createTableConfig(), createSchema())
+ .updateIndices(writer);
+ writer.save();
+ }
+ }
+
+ private static Schema createSchema() {
+ return new Schema.SchemaBuilder().setSchemaName("testSchema")
+ .addMultiValueDimension(MV_COLUMN, DataType.INT)
+ .build();
+ }
+
+ private static TableConfig createTableConfig() {
+ return new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ }
+
+ private static File buildSegment()
+ throws Exception {
+ FileUtils.deleteQuietly(TEMP_DIR);
+ SegmentGeneratorConfig config = new
SegmentGeneratorConfig(createTableConfig(), createSchema());
+ config.setOutDir(TEMP_DIR.getAbsolutePath());
+ config.setSegmentName(SEGMENT_NAME);
+
+ List<GenericRow> rows = new ArrayList<>();
+ int[][] values = {{3, 1}, {7}, {2, 9, 4}, {5, 5}};
+ for (int[] docValues : values) {
+ GenericRow row = new GenericRow();
+ Object[] boxed = new Object[docValues.length];
+ for (int i = 0; i < docValues.length; i++) {
+ boxed[i] = docValues[i];
+ }
+ row.putValue(MV_COLUMN, boxed);
+ rows.add(row);
+ }
+
+ SegmentIndexCreationDriverImpl driver = new
SegmentIndexCreationDriverImpl();
+ driver.init(config, new GenericRowRecordReader(rows));
+ driver.build();
+ return new File(TEMP_DIR, SEGMENT_NAME);
+ }
+
+ @AfterClass
+ public void tearDown() {
+ FileUtils.deleteQuietly(TEMP_DIR);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
