This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 67b3bca2f2f Support star-tree index on RAW forward index with
separated dictionary with predicate queries (#19153)
67b3bca2f2f is described below
commit 67b3bca2f2fb172f33c07cb761582b8c91b12414
Author: Chaitanya Deepthi <[email protected]>
AuthorDate: Fri Aug 7 11:05:19 2026 -0700
Support star-tree index on RAW forward index with separated dictionary with
predicate queries (#19153)
---
.../apache/pinot/core/startree/StarTreeUtils.java | 17 +-
.../pinot/core/startree/StarTreeUtilsTest.java | 243 +++++++++++++++++++++
.../pinot/core/startree/v2/BaseStarTreeV2Test.java | 36 +--
.../v2/RawWithDictionaryStarTreeV2Test.java | 92 ++++++++
4 files changed, 371 insertions(+), 17 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/startree/StarTreeUtils.java
b/pinot-core/src/main/java/org/apache/pinot/core/startree/StarTreeUtils.java
index d870ff6dbd8..e9352c5799f 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/startree/StarTreeUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/startree/StarTreeUtils.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.core.startree;
+import com.google.common.annotations.VisibleForTesting;
import it.unimi.dsi.fastutil.objects.ObjectBooleanPair;
import java.util.ArrayDeque;
import java.util.ArrayList;
@@ -34,6 +35,7 @@ import org.apache.pinot.common.request.context.FilterContext;
import org.apache.pinot.common.request.context.predicate.Predicate;
import org.apache.pinot.core.operator.BaseProjectOperator;
import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator;
+import
org.apache.pinot.core.operator.filter.predicate.PredicateEvaluatorProvider;
import org.apache.pinot.core.query.aggregation.function.AggregationFunction;
import
org.apache.pinot.core.query.aggregation.function.AggregationFunctionUtils;
import org.apache.pinot.core.query.request.context.QueryContext;
@@ -335,12 +337,25 @@ public class StarTreeUtils {
}
for (Pair<Predicate, PredicateEvaluator> pair :
predicatesEvaluatorMapping) {
if (pair.getKey() == predicate) {
- return pair.getValue();
+ return toDictionaryBased(pair.getValue(), predicate, dataSource);
}
}
return null;
}
+ /// Star-tree traversal reads dictionary ids; a raw-value evaluator (built
when the forward index is RAW and no
+ /// dict-consuming scan operator was available) would throw from
`getMatchingDictIds` / `applySV(int)`. Rebuild
+ /// against the segment dictionary when needed.
+ @VisibleForTesting
+ static PredicateEvaluator toDictionaryBased(PredicateEvaluator evaluator,
Predicate predicate,
+ DataSource dataSource) {
+ if (evaluator.isDictionaryBased()) {
+ return evaluator;
+ }
+ return PredicateEvaluatorProvider.getPredicateEvaluator(predicate,
dataSource.getDictionary(),
+ dataSource.getDataSourceMetadata().getDataType(), null);
+ }
+
/// Returns a [BaseProjectOperator] when the filter can be solved with
star-tree, or `null` otherwise.
@Nullable
public static BaseProjectOperator<?>
createStarTreeBasedProjectOperator(IndexSegment indexSegment,
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/startree/StarTreeUtilsTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/startree/StarTreeUtilsTest.java
new file mode 100644
index 00000000000..cd18702ad88
--- /dev/null
+++
b/pinot-core/src/test/java/org/apache/pinot/core/startree/StarTreeUtilsTest.java
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.startree;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.pinot.common.request.context.ExpressionContext;
+import org.apache.pinot.common.request.context.predicate.EqPredicate;
+import org.apache.pinot.core.operator.BaseProjectOperator;
+import org.apache.pinot.core.operator.blocks.ValueBlock;
+import
org.apache.pinot.core.operator.filter.predicate.BaseRawValueBasedPredicateEvaluator;
+import
org.apache.pinot.core.operator.filter.predicate.EqualsPredicateEvaluatorFactory;
+import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator;
+import org.apache.pinot.core.plan.FilterPlanNode;
+import org.apache.pinot.core.query.request.context.QueryContext;
+import
org.apache.pinot.core.query.request.context.utils.QueryContextConverterUtils;
+import
org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
+import
org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
+import org.apache.pinot.segment.local.startree.v2.builder.MultipleTreesBuilder;
+import org.apache.pinot.segment.spi.IndexSegment;
+import org.apache.pinot.segment.spi.SegmentContext;
+import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.datasource.DataSource;
+import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
+import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.apache.pinot.spi.config.table.StarTreeIndexConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.readers.GenericRow;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.ReadMode;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertSame;
+import static org.testng.Assert.assertTrue;
+
+
+/// Tests for [StarTreeUtils] verifying that star-tree can consume filters on
columns configured with a `RAW` forward
+/// index and a separated dictionary.
+///
+/// Before the fix,
[org.apache.pinot.core.startree.operator.StarTreeFilterOperator] invoked
`getMatchingDictIds()` on
+/// a raw-value evaluator during tree traversal and threw
`UnsupportedOperationException`. This test locks in that
+/// the star-tree planner upgrades such evaluators to dictionary-based,
yielding the same query behavior as if the
+/// column were `DICTIONARY`-encoded.
+public class StarTreeUtilsTest {
+
+ private static final String COLUMN = "raw_dim_with_dict";
+
+ //-------------------------------------------------------------------------
+ // Unit tests for the private helper
+ //-------------------------------------------------------------------------
+
+ /// Raw-value evaluator on a column with a dictionary is rebuilt as
dictionary-based.
+ @Test
+ public void testToDictionaryBasedConvertsRawEvaluator() {
+ Dictionary dictionary = mock(Dictionary.class);
+ when(dictionary.length()).thenReturn(2);
+ when(dictionary.getStringValue(0)).thenReturn("v0");
+ when(dictionary.getStringValue(1)).thenReturn("v1");
+ when(dictionary.indexOf("v1")).thenReturn(1);
+
+ DataSourceMetadata metadata = mock(DataSourceMetadata.class);
+ when(metadata.getDataType()).thenReturn(DataType.STRING);
+
+ DataSource dataSource = mock(DataSource.class);
+ when(dataSource.getDictionary()).thenReturn(dictionary);
+ when(dataSource.getDataSourceMetadata()).thenReturn(metadata);
+
+ EqPredicate predicate = new
EqPredicate(ExpressionContext.forIdentifier(COLUMN), "v1");
+ PredicateEvaluator raw =
+ EqualsPredicateEvaluatorFactory.newRawValueBasedEvaluator(predicate,
DataType.STRING);
+ assertFalse(raw.isDictionaryBased(), "Baseline evaluator must be
raw-value-based");
+ assertTrue(raw instanceof BaseRawValueBasedPredicateEvaluator);
+
+ PredicateEvaluator converted = StarTreeUtils.toDictionaryBased(raw,
predicate, dataSource);
+ assertTrue(converted.isDictionaryBased(), "Converted evaluator must be
dictionary-based");
+ assertNotNull(converted.getMatchingDictIds());
+ }
+
+ /// Dictionary-based evaluators flow through unchanged (`instanceof` no-op).
+ @Test
+ public void testToDictionaryBasedIsNoopForDictionaryBased() {
+ Dictionary dictionary = mock(Dictionary.class);
+ when(dictionary.length()).thenReturn(1);
+ when(dictionary.indexOf("v1")).thenReturn(0);
+
+ DataSource dataSource = mock(DataSource.class);
+ when(dataSource.getDictionary()).thenReturn(dictionary);
+
+ EqPredicate predicate = new
EqPredicate(ExpressionContext.forIdentifier(COLUMN), "v1");
+ PredicateEvaluator dictBased =
+ EqualsPredicateEvaluatorFactory.newDictionaryBasedEvaluator(predicate,
dictionary, DataType.STRING);
+ assertTrue(dictBased.isDictionaryBased());
+
+ PredicateEvaluator result = StarTreeUtils.toDictionaryBased(dictBased,
predicate, dataSource);
+ assertSame(result, dictBased, "Dictionary-based evaluator should pass
through unchanged");
+ }
+
+ //-------------------------------------------------------------------------
+ // Segment-level test: RAW forward + separated dictionary + star-tree
+ //-------------------------------------------------------------------------
+
+ private static final File TEMP_DIR = new File(FileUtils.getTempDirectory(),
"StarTreeUtilsTest");
+ private static final String TABLE_NAME = "starTreeUtilsTest";
+ private static final String SEGMENT_NAME = "testSegment";
+ private static final int NUM_ROWS = 1000;
+ private static final int CARDINALITY = 10;
+
+ private IndexSegment _segment;
+
+ @BeforeClass
+ public void setUp()
+ throws Exception {
+ FileUtils.deleteDirectory(TEMP_DIR);
+
+ Schema schema = new Schema.SchemaBuilder()
+ .addSingleValueDimension(COLUMN, DataType.STRING)
+ .build();
+
+ // Explicit indexes JSON:
+ // forward.encodingType = RAW → forward index stores raw values
+ // dictionary.disabled = false → keep the dictionary alongside
+ ObjectNode indexes = JsonUtils.newObjectNode();
+ ObjectNode forwardCfg = JsonUtils.newObjectNode();
+ forwardCfg.put("encodingType", "RAW");
+ indexes.set("forward", forwardCfg);
+ ObjectNode dictCfg = JsonUtils.newObjectNode();
+ dictCfg.put("disabled", false);
+ indexes.set("dictionary", dictCfg);
+
+ FieldConfig rawWithDict = new FieldConfig.Builder(COLUMN)
+ .withEncodingType(FieldConfig.EncodingType.RAW)
+ .withIndexes(indexes)
+ .build();
+
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE)
+ .setTableName(TABLE_NAME)
+ .setFieldConfigList(List.of(rawWithDict))
+ .build();
+
+ List<GenericRow> rows = new ArrayList<>(NUM_ROWS);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ GenericRow row = new GenericRow();
+ row.putValue(COLUMN, "v" + (i % CARDINALITY));
+ rows.add(row);
+ }
+
+ SegmentIndexCreationDriverImpl driver = new
SegmentIndexCreationDriverImpl();
+ SegmentGeneratorConfig cfg = new SegmentGeneratorConfig(tableConfig,
schema);
+ cfg.setOutDir(TEMP_DIR.getPath());
+ cfg.setSegmentName(SEGMENT_NAME);
+ driver.init(cfg, new GenericRowRecordReader(rows));
+ driver.build();
+
+ StarTreeIndexConfig sti =
+ new StarTreeIndexConfig(List.of(COLUMN), null, List.of("COUNT__*"),
null, Integer.MAX_VALUE);
+ File indexDir = new File(TEMP_DIR, SEGMENT_NAME);
+ try (MultipleTreesBuilder builder = new MultipleTreesBuilder(List.of(sti),
false, indexDir,
+ MultipleTreesBuilder.BuildMode.OFF_HEAP)) {
+ builder.build();
+ }
+
+ _segment = ImmutableSegmentLoader.load(indexDir, ReadMode.mmap);
+ }
+
+ @AfterClass
+ public void tearDown()
+ throws Exception {
+ if (_segment != null) {
+ _segment.destroy();
+ }
+ FileUtils.deleteDirectory(TEMP_DIR);
+ }
+
+ /// Sanity check that the column is exactly the RAW-forward +
separated-dictionary configuration the fix targets.
+ @Test
+ public void testColumnHasRawForwardAndDictionary() {
+ DataSource dataSource = _segment.getDataSource(COLUMN);
+ assertNotNull(dataSource.getDictionary(), "Expected dictionary on
RAW-encoded dim column");
+ assertFalse(dataSource.getForwardIndex().isDictionaryEncoded(),
+ "Expected forward index to be RAW-encoded");
+ assertNotNull(_segment.getStarTrees(), "Star-tree must be built");
+ assertEquals(_segment.getStarTrees().size(), 1);
+ }
+
+ /// End-to-end assertion equivalent to `EXPLAIN PLAN` showing a `STAR_TREE`
node:
+ /// a filter query returning `COUNT(*)` from a RAW-forward +
separated-dictionary star-tree dimension is served
+ /// via the star-tree operator (non-null result from
`createStarTreeBasedProjectOperator`) and reads a single
+ /// aggregated record per matching path — same as a `DICTIONARY`-encoded dim
would.
+ @Test
+ public void testStarTreeAcceleratesEqualityFilterOnRawWithDictionary() {
+ QueryContext queryContext = QueryContextConverterUtils.getQueryContext(
+ String.format("SELECT COUNT(*) FROM %s WHERE %s = 'v3'", TABLE_NAME,
COLUMN));
+
+ FilterPlanNode filterPlanNode = new FilterPlanNode(new
SegmentContext(_segment), queryContext);
+ filterPlanNode.run();
+
+ BaseProjectOperator<?> operator =
StarTreeUtils.createStarTreeBasedProjectOperator(_segment, queryContext,
+ queryContext.getAggregationFunctions(), queryContext.getFilter(),
+ filterPlanNode.getPredicateEvaluators());
+ assertNotNull(operator, "Star-tree plan expected for EQ on RAW+dict
dimension");
+
+ // Traversal must not throw; before the fix,
StarTreeFilterOperator.getMatchingDictIds threw UOE here.
+ ValueBlock block = operator.nextBlock();
+ assertNotNull(block, "Star-tree traversal returned no block");
+
+ // Star-tree yields one aggregated document per matching path — same
behavior as DICTIONARY-encoded columns.
+ // (For an EQ on a single dim, matchLeafRecords=MAX_VALUE, and 1 unique
matching value, exactly one leaf matches.)
+ assertEquals(block.getNumDocs(), 1,
+ "Expected exactly one aggregated star-tree document for the EQ filter;
got " + block.getNumDocs());
+ }
+}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
index e4492bb1669..cd4bb07bbd9 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
@@ -43,6 +43,7 @@ import
org.apache.pinot.segment.local.aggregator.ValueAggregator;
import
org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
import
org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
+import org.apache.pinot.segment.local.segment.readers.PinotSegmentColumnReader;
import org.apache.pinot.segment.local.startree.v2.builder.MultipleTreesBuilder;
import org.apache.pinot.segment.spi.AggregationFunctionType;
import org.apache.pinot.segment.spi.Constants;
@@ -85,15 +86,15 @@ abstract class BaseStarTreeV2Test<R, A> {
private static final Random RANDOM = new Random();
private static final File TEMP_DIR = new File(FileUtils.getTempDirectory(),
"BaseStarTreeV2Test");
- private static final String TABLE_NAME = "testTable";
+ protected static final String TABLE_NAME = "testTable";
private static final String SEGMENT_NAME = "testSegment";
private static final int NUM_SEGMENT_RECORDS = 100_000;
private static final int MAX_LEAF_RECORDS = RANDOM.nextInt(100) + 1;
// Using column names with '__' to make sure regular table columns with '__'
in the name aren't wrongly interpreted
// as AggregationFunctionColumnPair
- private static final String DIMENSION1 = "d1__COLUMN_NAME";
- private static final String DIMENSION2 = "DISTINCTCOUNTRAWHLL__d2";
+ protected static final String DIMENSION1 = "d1__COLUMN_NAME";
+ protected static final String DIMENSION2 = "DISTINCTCOUNTRAWHLL__d2";
private static final int DIMENSION_CARDINALITY = 100;
private static final String AGG_COL = "m";
@@ -152,7 +153,7 @@ abstract class BaseStarTreeV2Test<R, A> {
dimensionFieldSpec ->
dimensionFieldSpec.setSingleValueField(isAggColSingleValueField()));
}
Schema schema = schemaBuilder.build();
- TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).build();
+ TableConfig tableConfig = createTableConfig();
List<GenericRow> segmentRecords = new ArrayList<>(NUM_SEGMENT_RECORDS);
for (int i = 0; i < NUM_SEGMENT_RECORDS; i++) {
@@ -311,9 +312,12 @@ abstract class BaseStarTreeV2Test<R, A> {
nonStarTreeAggregationColumnDictionaries.add(dataSource.getDictionary());
}
}
- List<ForwardIndexReader> nonStarTreeGroupByColumnReaders = new
ArrayList<>(numGroupByColumns);
+ // Use PinotSegmentColumnReader so a RAW forward-index group-by column
with a separated dictionary resolves
+ // its dict ids via the dictionary — mirroring the same lookup the
star-tree builder performs at build time
+ // (see BaseSingleTreeBuilder). Behavior is identical for
dictionary-encoded columns.
+ List<PinotSegmentColumnReader> nonStarTreeGroupByColumnReaders = new
ArrayList<>(numGroupByColumns);
for (String groupByColumn : groupByColumns) {
-
nonStarTreeGroupByColumnReaders.add(_indexSegment.getDataSource(groupByColumn).getForwardIndex());
+ nonStarTreeGroupByColumnReaders.add(new
PinotSegmentColumnReader(_indexSegment, groupByColumn));
}
Map<List<Integer>, List<Object>> nonStarTreeResult =
computeNonStarTreeResult(nonStarTreeFilterPlanNode,
nonStarTreeAggregationColumnReaders,
@@ -401,14 +405,13 @@ abstract class BaseStarTreeV2Test<R, A> {
private Map<List<Integer>, List<Object>>
computeNonStarTreeResult(FilterPlanNode nonStarTreeFilterPlanNode,
List<ForwardIndexReader> aggregationColumnReaders, List<Dictionary>
aggregationColumnDictionaries,
- List<ForwardIndexReader> groupByColumnReaders)
+ List<PinotSegmentColumnReader> groupByColumnReaders)
throws IOException {
Map<List<Integer>, List<Object>> result = new HashMap<>();
int numAggregations = aggregationColumnReaders.size();
int numGroupByColumns = groupByColumnReaders.size();
List<ForwardIndexReaderContext> aggregationColumnReaderContexts = new
ArrayList<>(numAggregations);
- List<ForwardIndexReaderContext> groupByColumnReaderContexts = new
ArrayList<>(numGroupByColumns);
try {
for (ForwardIndexReader aggregationColumnReader :
aggregationColumnReaders) {
if (aggregationColumnReader != null) {
@@ -417,9 +420,6 @@ abstract class BaseStarTreeV2Test<R, A> {
aggregationColumnReaderContexts.add(null);
}
}
- for (ForwardIndexReader groupByColumnReader : groupByColumnReaders) {
- groupByColumnReaderContexts.add(groupByColumnReader.createContext());
- }
BlockDocIdIterator docIdIterator =
nonStarTreeFilterPlanNode.run().nextBlock().getBlockDocIdSet().iterator();
int docId;
@@ -427,7 +427,7 @@ abstract class BaseStarTreeV2Test<R, A> {
// Array of dictionary ids (zero-length array for non-group-by queries)
List<Integer> group = new ArrayList<>(numGroupByColumns);
for (int i = 0; i < numGroupByColumns; i++) {
- group.add(groupByColumnReaders.get(i).getDictId(docId,
groupByColumnReaderContexts.get(i)));
+ group.add(groupByColumnReaders.get(i).getDictId(docId));
}
List<Object> values = result.computeIfAbsent(group, k -> new
ArrayList<>(numAggregations));
if (values.isEmpty()) {
@@ -465,10 +465,8 @@ abstract class BaseStarTreeV2Test<R, A> {
readerContext.close();
}
}
- for (ForwardIndexReaderContext readerContext :
groupByColumnReaderContexts) {
- if (readerContext != null) {
- readerContext.close();
- }
+ for (PinotSegmentColumnReader groupByColumnReader :
groupByColumnReaders) {
+ groupByColumnReader.close();
}
}
}
@@ -507,6 +505,12 @@ abstract class BaseStarTreeV2Test<R, A> {
return true;
}
+ /// Can be overridden to customize the table config (e.g. to attach
`FieldConfig`s that force a specific
+ /// forward-index encoding or dictionary configuration for the dimension
columns).
+ protected TableConfig createTableConfig() {
+ return new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).build();
+ }
+
abstract ValueAggregator<R, A> getValueAggregator();
abstract DataType getRawValueType();
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/RawWithDictionaryStarTreeV2Test.java
b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/RawWithDictionaryStarTreeV2Test.java
new file mode 100644
index 00000000000..f0ebd508188
--- /dev/null
+++
b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/RawWithDictionaryStarTreeV2Test.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.startree.v2;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.util.List;
+import java.util.Random;
+import org.apache.pinot.segment.local.aggregator.SumValueAggregator;
+import org.apache.pinot.segment.local.aggregator.ValueAggregator;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+
+import static org.testng.Assert.assertEquals;
+
+
+/// Exercises the full [BaseStarTreeV2Test] query matrix against star-tree
dimensions configured with a `RAW`
+/// forward index and a separated dictionary — the exact configuration handled
by
+/// [org.apache.pinot.core.startree.StarTreeUtils#toDictionaryBased] (Apache
Pinot PR #19153). Both
+/// `DIMENSION1` and `DIMENSION2` are switched to `RAW` + separated dictionary
so every filter predicate,
+/// including the `GROUP BY DIMENSION2` case, goes through the fix.
+///
+/// Before that fix,
[org.apache.pinot.core.startree.operator.StarTreeFilterOperator] invoked
+/// `getMatchingDictIds()` on a raw-value evaluator during tree traversal and
threw
+/// `UnsupportedOperationException`. Running the inherited `testQueries()` /
`testUnsupportedFilters()`
+/// suites confirms every AND/OR/NOT/nested filter combination now produces
the same aggregated result
+/// via the star-tree as via a plain scan.
+public class RawWithDictionaryStarTreeV2Test extends
BaseStarTreeV2Test<Object, Double> {
+
+ @Override
+ ValueAggregator<Object, Double> getValueAggregator() {
+ return new SumValueAggregator();
+ }
+
+ @Override
+ DataType getRawValueType() {
+ return DataType.INT;
+ }
+
+ @Override
+ Object getRandomRawValue(Random random) {
+ return random.nextInt();
+ }
+
+ @Override
+ protected void assertAggregatedValue(Double starTreeResult, Double
nonStarTreeResult) {
+ assertEquals(starTreeResult, nonStarTreeResult, 1e-5);
+ }
+
+ @Override
+ protected TableConfig createTableConfig() {
+ return new TableConfigBuilder(TableType.OFFLINE)
+ .setTableName(TABLE_NAME)
+ .setFieldConfigList(List.of(rawWithDictionary(DIMENSION1),
rawWithDictionary(DIMENSION2)))
+ .build();
+ }
+
+ /// Builds a `FieldConfig` that stores the column as a `RAW` forward index
while keeping a dictionary
+ /// alongside — the "separated dictionary" configuration star-tree must be
able to consume.
+ private static FieldConfig rawWithDictionary(String column) {
+ ObjectNode indexes = JsonUtils.newObjectNode();
+ ObjectNode forwardCfg = JsonUtils.newObjectNode();
+ forwardCfg.put("encodingType", "RAW");
+ indexes.set("forward", forwardCfg);
+ ObjectNode dictionaryCfg = JsonUtils.newObjectNode();
+ dictionaryCfg.put("disabled", false);
+ indexes.set("dictionary", dictionaryCfg);
+ return new FieldConfig.Builder(column)
+ .withEncodingType(FieldConfig.EncodingType.RAW)
+ .withIndexes(indexes)
+ .build();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
