This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 69e57f657d5 Allow timestamp-index derived columns in star-tree
dimensionsSplitOrder validation (#18774)
69e57f657d5 is described below
commit 69e57f657d5fe41f1386a7815c081eedcacbefd1
Author: Soumya Himanish Mohapatra
<[email protected]>
AuthorDate: Fri Aug 14 02:20:17 2026 +0530
Allow timestamp-index derived columns in star-tree dimensionsSplitOrder
validation (#18774)
---
.../pinot/core/startree/v2/BaseStarTreeV2Test.java | 34 ++++-
.../startree/v2/TimestampIndexStarTreeV2Test.java | 149 +++++++++++++++++++++
.../segment/local/utils/TableConfigUtils.java | 21 ++-
.../segment/local/utils/TableConfigUtilsTest.java | 38 ++++++
4 files changed, 233 insertions(+), 9 deletions(-)
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
index 6bba131d59d..4896b6865dd 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
@@ -97,7 +97,7 @@ abstract class BaseStarTreeV2Test<R, A> {
// as AggregationFunctionColumnPair
protected static final String DIMENSION1 = "d1__COLUMN_NAME";
protected static final String DIMENSION2 = "DISTINCTCOUNTRAWHLL__d2";
- private static final int DIMENSION_CARDINALITY = 100;
+ protected static final int DIMENSION_CARDINALITY = 100;
private static final String AGG_COL = "m";
// Supported filters
@@ -146,8 +146,8 @@ abstract class BaseStarTreeV2Test<R, A> {
_aggregatedValueType = _valueAggregator.getAggregatedValueType();
_aggregation = getAggregation(_valueAggregator.getAggregationType());
- Schema.SchemaBuilder schemaBuilder = new
Schema.SchemaBuilder().addSingleValueDimension(DIMENSION1, DataType.INT)
- .addSingleValueDimension(DIMENSION2, DataType.INT);
+ Schema.SchemaBuilder schemaBuilder = new Schema.SchemaBuilder();
+ addDimensionFields(schemaBuilder);
DataType rawValueType = getRawValueType();
// Raw value type will be null for COUNT aggregation function
if (rawValueType != null) {
@@ -160,8 +160,7 @@ abstract class BaseStarTreeV2Test<R, A> {
List<GenericRow> segmentRecords = new ArrayList<>(NUM_SEGMENT_RECORDS);
for (int i = 0; i < NUM_SEGMENT_RECORDS; i++) {
GenericRow segmentRecord = new GenericRow();
- segmentRecord.putValue(DIMENSION1,
RANDOM.nextInt(DIMENSION_CARDINALITY));
- segmentRecord.putValue(DIMENSION2,
RANDOM.nextInt(DIMENSION_CARDINALITY));
+ addDimensionValues(segmentRecord, RANDOM);
if (rawValueType != null) {
segmentRecord.putValue(AGG_COL, getRandomRawValue(RANDOM));
}
@@ -175,7 +174,7 @@ abstract class BaseStarTreeV2Test<R, A> {
driver.init(segmentGeneratorConfig, new
GenericRowRecordReader(segmentRecords));
driver.build();
- StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(Arrays.asList(DIMENSION1, DIMENSION2), null, null,
+ StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(getDimensionsSplitOrder(), null, null,
List.of(new StarTreeAggregationConfig(AGG_COL,
_valueAggregator.getAggregationType().getName(), null,
getCompressionCodec(),
true, getIndexVersion(), null, null)), MAX_LEAF_RECORDS);
@@ -254,7 +253,7 @@ abstract class BaseStarTreeV2Test<R, A> {
assertNull(predicateEvaluatorsMap);
}
- private void testQuery(String query)
+ protected void testQuery(String query)
throws IOException {
QueryContext queryContext =
QueryContextConverterUtils.getQueryContext(query);
@@ -513,6 +512,27 @@ abstract class BaseStarTreeV2Test<R, A> {
return new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).build();
}
+ protected void addDimensionFields(Schema.SchemaBuilder schemaBuilder) {
+ schemaBuilder.addSingleValueDimension(DIMENSION1,
DataType.INT).addSingleValueDimension(DIMENSION2, DataType.INT);
+ }
+
+ protected void addDimensionValues(GenericRow segmentRecord, Random random) {
+ segmentRecord.putValue(DIMENSION1, random.nextInt(DIMENSION_CARDINALITY));
+ segmentRecord.putValue(DIMENSION2, random.nextInt(DIMENSION_CARDINALITY));
+ }
+
+ protected List<String> getDimensionsSplitOrder() {
+ return List.of(DIMENSION1, DIMENSION2);
+ }
+
+ protected String getAggregation() {
+ return _aggregation;
+ }
+
+ protected StarTreeV2 getStarTreeV2() {
+ return _starTreeV2;
+ }
+
abstract ValueAggregator<R, A> getValueAggregator();
abstract DataType getRawValueType();
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/TimestampIndexStarTreeV2Test.java
b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/TimestampIndexStarTreeV2Test.java
new file mode 100644
index 00000000000..b524e26a279
--- /dev/null
+++
b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/TimestampIndexStarTreeV2Test.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.startree.v2;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.pinot.common.request.context.ExpressionContext;
+import org.apache.pinot.core.query.aggregation.function.AggregationFunction;
+import org.apache.pinot.core.query.request.context.QueryContext;
+import
org.apache.pinot.core.query.request.context.utils.QueryContextConverterUtils;
+import org.apache.pinot.core.startree.StarTreeUtils;
+import org.apache.pinot.segment.local.aggregator.SumValueAggregator;
+import org.apache.pinot.segment.local.aggregator.ValueAggregator;
+import
org.apache.pinot.segment.spi.index.startree.AggregationFunctionColumnPair;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.config.table.TimestampConfig;
+import org.apache.pinot.spi.config.table.TimestampIndexGranularity;
+import org.apache.pinot.spi.data.DateTimeFieldSpec;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.readers.GenericRow;
+import org.apache.pinot.spi.utils.TimestampIndexUtils;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+
+public class TimestampIndexStarTreeV2Test extends BaseStarTreeV2Test<Object,
Double> {
+ private static final String TIMESTAMP_COLUMN = "tsCol";
+ private static final String TIMESTAMP_COLUMN_WITH_GRANULARITY =
+ TimestampIndexUtils.getColumnWithGranularity(TIMESTAMP_COLUMN,
TimestampIndexGranularity.MILLISECOND);
+
+ @Override
+ ValueAggregator<Object, Double> getValueAggregator() {
+ return new SumValueAggregator();
+ }
+
+ @Override
+ DataType getRawValueType() {
+ return DataType.INT;
+ }
+
+ @Override
+ Object getRandomRawValue(Random random) {
+ return random.nextInt();
+ }
+
+ @Override
+ void assertAggregatedValue(Double starTreeResult, Double nonStarTreeResult) {
+ assertEquals(starTreeResult, nonStarTreeResult, 1e-5);
+ }
+
+ @Override
+ protected void addDimensionFields(Schema.SchemaBuilder schemaBuilder) {
+ super.addDimensionFields(schemaBuilder);
+ schemaBuilder.addField(
+ new DateTimeFieldSpec(TIMESTAMP_COLUMN, DataType.TIMESTAMP,
DateTimeFieldSpec.TimeFormat.TIMESTAMP.name(),
+ "1:MILLISECONDS"));
+ }
+
+ @Override
+ protected void addDimensionValues(GenericRow segmentRecord, Random random) {
+ super.addDimensionValues(segmentRecord, random);
+ segmentRecord.putValue(TIMESTAMP_COLUMN, (long)
random.nextInt(DIMENSION_CARDINALITY));
+ }
+
+ @Override
+ protected TableConfig createTableConfig() {
+ return new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
+ .setFieldConfigList(List.of(new
FieldConfig.Builder(TIMESTAMP_COLUMN).withTimestampConfig(
+ new
TimestampConfig(List.of(TimestampIndexGranularity.MILLISECOND))).build()))
+ .build();
+ }
+
+ @Override
+ protected List<String> getDimensionsSplitOrder() {
+ return List.of(DIMENSION1, DIMENSION2, TIMESTAMP_COLUMN_WITH_GRANULARITY);
+ }
+
+ @Test
+ public void testStarTreeBuiltOnTimestampIndexColumn() {
+
assertTrue(getStarTreeV2().getMetadata().getDimensionsSplitOrder().contains(TIMESTAMP_COLUMN_WITH_GRANULARITY));
+
assertEquals(getStarTreeV2().getDataSource(TIMESTAMP_COLUMN_WITH_GRANULARITY).getDictionary().length(),
+ DIMENSION_CARDINALITY);
+ }
+
+ @Test
+ public void testStarTreeFitForQueryOnTimestampIndexColumn() {
+ QueryContext queryContext = QueryContextConverterUtils.getQueryContext(
+ String.format("SELECT %s FROM %s WHERE %s < 10 GROUP BY %s",
getAggregation(), TABLE_NAME,
+ TIMESTAMP_COLUMN_WITH_GRANULARITY,
TIMESTAMP_COLUMN_WITH_GRANULARITY));
+ AggregationFunction[] aggregationFunctions =
queryContext.getAggregationFunctions();
+ assertNotNull(aggregationFunctions);
+ AggregationFunctionColumnPair[] aggregationFunctionColumnPairs =
+ StarTreeUtils.extractAggregationFunctionPairs(aggregationFunctions);
+ assertNotNull(aggregationFunctionColumnPairs);
+ List<Pair<AggregationFunction, AggregationFunctionColumnPair>>
aggregations =
+ new ArrayList<>(aggregationFunctions.length);
+ for (int i = 0; i < aggregationFunctions.length; i++) {
+ aggregations.add(Pair.of(aggregationFunctions[i],
aggregationFunctionColumnPairs[i]));
+ }
+ List<ExpressionContext> groupByExpressions =
queryContext.getGroupByExpressions();
+ assertNotNull(groupByExpressions);
+ assertTrue(StarTreeUtils.isFitForStarTree(getStarTreeV2().getMetadata(),
aggregations,
+ groupByExpressions.toArray(new ExpressionContext[0]),
Set.of(TIMESTAMP_COLUMN_WITH_GRANULARITY)));
+ assertFalse(StarTreeUtils.isFitForStarTree(getStarTreeV2().getMetadata(),
aggregations,
+ groupByExpressions.toArray(new ExpressionContext[0]),
Set.of(TIMESTAMP_COLUMN)));
+ }
+
+ @Test
+ public void testQueriesOnTimestampIndexColumn()
+ throws IOException {
+ String query = String.format("SELECT %s FROM %s", getAggregation(),
TABLE_NAME);
+ testQuery(query + String.format(" WHERE %s = 0",
TIMESTAMP_COLUMN_WITH_GRANULARITY));
+ testQuery(query + String.format(" WHERE %s < 10",
TIMESTAMP_COLUMN_WITH_GRANULARITY));
+ testQuery(query + String.format(" WHERE %1$s > 10 OR %1$s < 50",
TIMESTAMP_COLUMN_WITH_GRANULARITY));
+ testQuery(query + String.format(" WHERE NOT %s > 10",
TIMESTAMP_COLUMN_WITH_GRANULARITY));
+ testQuery(
+ query + String.format(" WHERE %s < 10 AND %s > 50",
TIMESTAMP_COLUMN_WITH_GRANULARITY, DIMENSION1));
+ testQuery(query + String.format(" WHERE %s > 90 AND NOT %s < 25",
TIMESTAMP_COLUMN_WITH_GRANULARITY, DIMENSION2));
+ testQuery(query + " GROUP BY " + TIMESTAMP_COLUMN_WITH_GRANULARITY);
+ }
+}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
index fc9158af589..fc2dcd1e84b 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
@@ -114,6 +114,7 @@ import org.apache.pinot.spi.utils.IngestionConfigUtils;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.PinotMd5Mode;
import org.apache.pinot.spi.utils.TimeUtils;
+import org.apache.pinot.spi.utils.TimestampIndexUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1824,7 +1825,8 @@ public final class TableConfigUtils {
// Star-tree index config is not managed by FieldIndexConfigs, and we need
to validate it separately.
List<StarTreeIndexConfig> starTreeIndexConfigs =
indexingConfig.getStarTreeIndexConfigs();
if (CollectionUtils.isNotEmpty(starTreeIndexConfigs)) {
- validateStarTreeIndexConfigs(starTreeIndexConfigs, indexConfigsMap,
schema);
+ validateStarTreeIndexConfigs(starTreeIndexConfigs, indexConfigsMap,
schema,
+ TimestampIndexUtils.extractColumnsWithGranularity(tableConfig));
}
// TIMESTAMP index is not managed by FieldIndexConfigs, and we need to
validate it separately.
@@ -1952,14 +1954,23 @@ public final class TableConfigUtils {
/// - 'dimensionsSplitOrder' contains all dimensions in
'skipStarNodeCreationForDimensions'
/// - Either functionColumnPairs or aggregationConfigs must be specified,
but not both
/// - All referenced columns exist in the schema and are single-valued
+ ///
+ /// `timestampIndexColumns` holds the TIMESTAMP-index derived columns (e.g.
`$ts$DAY`) declared via
+ /// [TimestampConfig#getGranularities()]. These are materialized as
dictionary-encoded single-value TIMESTAMP
+ /// columns at segment generation time (see
[TimestampIndexUtils#applyTimestampIndex(TableConfig, Schema)]), so
+ /// they are absent from the schema at config-validation time and are
accepted here without a schema lookup.
private static void validateStarTreeIndexConfigs(List<StarTreeIndexConfig>
starTreeIndexConfigs,
- Map<String, FieldIndexConfigs> indexConfigsMap, Schema schema) {
+ Map<String, FieldIndexConfigs> indexConfigsMap, Schema schema,
Set<String> timestampIndexColumns) {
Set<String> dimensionColumns = new HashSet<>();
for (StarTreeIndexConfig starTreeIndexConfig : starTreeIndexConfigs) {
// Validate dimension columns are dictionary encoded
List<String> dimensionsSplitOrder =
starTreeIndexConfig.getDimensionsSplitOrder();
assert CollectionUtils.isNotEmpty(dimensionsSplitOrder);
for (String dimension : dimensionsSplitOrder) {
+ if (timestampIndexColumns.contains(dimension)) {
+ dimensionColumns.add(dimension);
+ continue;
+ }
FieldIndexConfigs indexConfigs = indexConfigsMap.get(dimension);
Preconditions.checkState(indexConfigs != null,
"Failed to find dimension column: %s specified in star-tree index
config in schema", dimension);
@@ -2046,6 +2057,9 @@ public final class TableConfigUtils {
}
for (String column : Iterables.concat(dimensionColumns,
aggregatedColumns)) {
+ if (timestampIndexColumns.contains(column)) {
+ continue;
+ }
FieldSpec fieldSpec = schema.getFieldSpecFor(column);
Preconditions.checkState(fieldSpec != null,
"Failed to find column: %s specified in star-tree index config in
schema", column);
@@ -2054,6 +2068,9 @@ public final class TableConfigUtils {
}
for (String column : dimensionColumns) {
+ if (timestampIndexColumns.contains(column)) {
+ continue;
+ }
FieldSpec fieldSpec = schema.getFieldSpecFor(column);
Preconditions.checkState(fieldSpec.isSingleValueField(),
"Star-tree dimension columns must be single-value, but found
multi-value column: %s", column);
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java
index 310bbfadbc2..af4d7a85875 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java
@@ -50,6 +50,8 @@ import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.config.table.TagOverrideConfig;
import org.apache.pinot.spi.config.table.TenantConfig;
import org.apache.pinot.spi.config.table.TierConfig;
+import org.apache.pinot.spi.config.table.TimestampConfig;
+import org.apache.pinot.spi.config.table.TimestampIndexGranularity;
import org.apache.pinot.spi.config.table.UpsertConfig;
import org.apache.pinot.spi.config.table.assignment.InstanceAssignmentConfig;
import org.apache.pinot.spi.config.table.assignment.InstancePartitionsType;
@@ -2335,6 +2337,42 @@ public class TableConfigUtilsTest {
}
}
+ @Test
+ public void testValidateStarTreeIndexWithTimestampIndexDerivedColumns() {
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
+ .addDateTime("OrderDate", DataType.TIMESTAMP, "TIMESTAMP",
"1:MILLISECONDS")
+ .addMetric("value", DataType.LONG)
+ .build();
+
+ // Derived TIMESTAMP-index columns ($OrderDate$DAY, ...) are declared via
TimestampConfig granularities and are
+ // materialized only at segment generation time, so they are absent from
the schema here. The star-tree config
+ // referencing them in dimensionsSplitOrder must still validate.
+ FieldConfig timestampFieldConfig = new
FieldConfig.Builder("OrderDate").withTimestampConfig(
+ new TimestampConfig(List.of(TimestampIndexGranularity.DAY,
TimestampIndexGranularity.WEEK,
+ TimestampIndexGranularity.MONTH))).build();
+ StarTreeIndexConfig starTreeIndexConfig = new StarTreeIndexConfig(
+ List.of("$OrderDate$DAY", "$OrderDate$WEEK", "$OrderDate$MONTH"),
null, List.of("COUNT__*"), null, 10000);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
+ .setFieldConfigList(List.of(timestampFieldConfig))
+ .setStarTreeIndexConfigs(List.of(starTreeIndexConfig))
+ .build();
+ TableConfigUtils.validate(tableConfig, schema);
+
+ // A derived-looking column whose granularity was NOT declared in
TimestampConfig must still be rejected.
+ StarTreeIndexConfig undeclaredGranularity = new StarTreeIndexConfig(
+ List.of("$OrderDate$HOUR"), null, List.of("COUNT__*"), null, 10000);
+ TableConfig invalidTableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
+ .setFieldConfigList(List.of(timestampFieldConfig))
+ .setStarTreeIndexConfigs(List.of(undeclaredGranularity))
+ .build();
+ try {
+ TableConfigUtils.validate(invalidTableConfig, schema);
+ fail("Should fail for star-tree dimension referencing an undeclared
timestamp-index granularity");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
@Test
public void testValidateStarTreeIndexDuplicateFunctionColumnPair() {
Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
