This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c9391fa7cbd Merge SSE and MSE timestamp index, and re-enable the
disabled tests (#17335)
c9391fa7cbd is described below
commit c9391fa7cbd134fb578c05760a683adf0abc7bc0
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Tue Dec 9 13:47:20 2025 -0800
Merge SSE and MSE timestamp index, and re-enable the disabled tests (#17335)
---
...a => OfflineTimestampIndexIntegrationTest.java} | 62 ++++++++--
.../tests/custom/TimestampIndexSseTest.java | 135 ---------------------
2 files changed, 52 insertions(+), 145 deletions(-)
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/TimestampIndexMseTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/OfflineTimestampIndexIntegrationTest.java
similarity index 81%
rename from
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/TimestampIndexMseTest.java
rename to
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/OfflineTimestampIndexIntegrationTest.java
index 8e4f3fa6201..180b1abf1ad 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/TimestampIndexMseTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/OfflineTimestampIndexIntegrationTest.java
@@ -23,6 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.regex.Pattern;
import org.apache.pinot.integration.tests.BaseClusterIntegrationTest;
import org.apache.pinot.integration.tests.ClusterIntegrationTestUtils;
import org.apache.pinot.integration.tests.ExplainIntegrationTestTrait;
@@ -33,7 +34,7 @@ import
org.apache.pinot.spi.config.table.TimestampIndexGranularity;
import org.apache.pinot.spi.config.table.ingestion.IngestionConfig;
import org.apache.pinot.spi.config.table.ingestion.TransformConfig;
import org.apache.pinot.spi.data.DateTimeFieldSpec;
-import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.CommonConstants;
@@ -44,7 +45,8 @@ import org.testng.annotations.Test;
import static org.testng.Assert.assertTrue;
-public class TimestampIndexMseTest extends BaseClusterIntegrationTest
implements ExplainIntegrationTestTrait {
+public class OfflineTimestampIndexIntegrationTest extends
BaseClusterIntegrationTest
+ implements ExplainIntegrationTestTrait {
@BeforeClass
public void setUp()
@@ -84,7 +86,7 @@ public class TimestampIndexMseTest extends
BaseClusterIntegrationTest implements
protected Schema createSchema()
throws IOException {
Schema schema = super.createSchema();
- schema.addField(new DateTimeFieldSpec("ts", FieldSpec.DataType.TIMESTAMP,
"TIMESTAMP", "1:DAYS"));
+ schema.addField(new DateTimeFieldSpec("ts", DataType.TIMESTAMP,
"TIMESTAMP", "1:DAYS"));
return schema;
}
@@ -104,7 +106,20 @@ public class TimestampIndexMseTest extends
BaseClusterIntegrationTest implements
}
@Test
- public void timestampIndexSubstitutedInProjections() {
+ public void testProjectionSse() {
+ setUseMultiStageQueryEngine(false);
+ explainSse("SELECT datetrunc('SECOND', ts) FROM mytable",
+ "[BROKER_REDUCE(limit:10), 1, 0]",
+ "[COMBINE_SELECT, 2, 1]",
+ "[PLAN_START(numSegmentsForThisPlan:1), -1, -1]",
+ "[SELECT(selectList:$ts$SECOND), 3, 2]",
+ "[PROJECT($ts$SECOND), 4, 3]",
+ "[DOC_ID_SET, 5, 4]",
+ Pattern.compile("\\[FILTER_MATCH_ENTIRE_SEGMENT\\(docs:[0-9]+\\), 6,
5]"));
+ }
+
+ @Test
+ public void testProjectionMse() {
setUseMultiStageQueryEngine(true);
explain("SELECT datetrunc('SECOND', ts) FROM mytable",
"Execution Plan\n"
@@ -118,8 +133,22 @@ public class TimestampIndexMseTest extends
BaseClusterIntegrationTest implements
+ " FilterMatchEntireSegment(numDocs=[115545])\n");
}
- @Test(enabled = false)
- public void timestampIndexSubstitutedInAggregateFilter() {
+ @Test
+ public void testAggregateFilterSse() {
+ setUseMultiStageQueryEngine(false);
+ explainSse("SELECT sum(case when datetrunc('SECOND', ts) > 1577836801000
then 2 else 0 end) FROM mytable",
+ "[BROKER_REDUCE(limit:10), 1, 0]",
+ "[COMBINE_AGGREGATE, 2, 1]",
+ "[PLAN_START(numSegmentsForThisPlan:1), -1, -1]",
+
"[AGGREGATE(aggregations:sum(case(greater_than($ts$SECOND,'1577836801000'),'2','0'))),
3, 2]",
+ "[TRANSFORM(case(greater_than($ts$SECOND,'1577836801000'),'2','0')),
4, 3]",
+ "[PROJECT($ts$SECOND), 5, 4]",
+ "[DOC_ID_SET, 6, 5]",
+ Pattern.compile("\\[FILTER_MATCH_ENTIRE_SEGMENT\\(docs:[0-9]+\\), 7,
6]"));
+ }
+
+ @Test
+ public void testAggregateFilterMse() {
setUseMultiStageQueryEngine(true);
explain("SELECT sum(case when datetrunc('SECOND', ts) > 1577836801000 then
2 else 0 end) FROM mytable",
"Execution Plan\n"
@@ -140,8 +169,8 @@ public class TimestampIndexMseTest extends
BaseClusterIntegrationTest implements
+ "
FilterMatchEntireSegment(numDocs=[115545])\n");
}
- @Test(enabled = false)
- public void timestampIndexCompoundAggregateFilter()
+ @Test
+ public void testCompoundAggregateFilterMse()
throws Exception {
setUseMultiStageQueryEngine(true);
String query =
@@ -171,7 +200,20 @@ public class TimestampIndexMseTest extends
BaseClusterIntegrationTest implements
}
@Test
- public void timestampIndexSubstitutedInGroupBy() {
+ public void testGroupBySse() {
+ setUseMultiStageQueryEngine(false);
+ explainSse("SELECT count(*) FROM mytable group by datetrunc('SECOND', ts)",
+ "[BROKER_REDUCE(limit:10), 1, 0]",
+ "[COMBINE_GROUP_BY, 2, 1]",
+ "[PLAN_START(numSegmentsForThisPlan:1), -1, -1]",
+ "[GROUP_BY(groupKeys:$ts$SECOND, aggregations:count(*)), 3, 2]",
+ "[PROJECT($ts$SECOND), 4, 3]",
+ "[DOC_ID_SET, 5, 4]",
+ Pattern.compile("\\[FILTER_MATCH_ENTIRE_SEGMENT\\(docs:[0-9]+\\), 6,
5]"));
+ }
+
+ @Test
+ public void testGroupByMse() {
setUseMultiStageQueryEngine(true);
explain("SELECT count(*) FROM mytable group by datetrunc('SECOND', ts)",
"Execution Plan\n"
@@ -188,7 +230,7 @@ public class TimestampIndexMseTest extends
BaseClusterIntegrationTest implements
}
@Test
- public void timestampIndexSubstitutedInJoinMSE() {
+ public void testJoinMse() {
setUseMultiStageQueryEngine(true);
explain("SELECT 1 "
+ "FROM mytable as a1 "
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/TimestampIndexSseTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/TimestampIndexSseTest.java
deleted file mode 100644
index 5cc490ac33b..00000000000
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/TimestampIndexSseTest.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.integration.tests.custom;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.regex.Pattern;
-import org.apache.pinot.integration.tests.BaseClusterIntegrationTest;
-import org.apache.pinot.integration.tests.ClusterIntegrationTestUtils;
-import org.apache.pinot.integration.tests.ExplainIntegrationTestTrait;
-import org.apache.pinot.spi.config.table.FieldConfig;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.config.table.TimestampConfig;
-import org.apache.pinot.spi.config.table.TimestampIndexGranularity;
-import org.apache.pinot.spi.config.table.ingestion.IngestionConfig;
-import org.apache.pinot.spi.config.table.ingestion.TransformConfig;
-import org.apache.pinot.spi.data.DateTimeFieldSpec;
-import org.apache.pinot.spi.data.FieldSpec.DataType;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.util.TestUtils;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-
-public class TimestampIndexSseTest extends BaseClusterIntegrationTest
implements ExplainIntegrationTestTrait {
-
- @BeforeClass
- public void setUp()
- throws Exception {
- TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
-
- // Start the Pinot cluster
- startZk();
- startController();
- startBroker();
- startServers(2);
-
- // Create and upload the schema and table config
- Schema schema = createSchema();
- addSchema(schema);
- TableConfig tableConfig = createOfflineTableConfig();
- addTableConfig(tableConfig);
-
- // Unpack the Avro files
- List<File> avroFiles = unpackAvroData(_tempDir);
-
- // Create and upload segments
- ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, tableConfig,
schema, 0, _segmentDir, _tarDir);
- uploadSegments(getTableName(), _tarDir);
-
- // Wait for all documents loaded
- waitForAllDocsLoaded(600_000L);
- }
-
- @Override
- protected Schema createSchema()
- throws IOException {
- Schema schema = super.createSchema();
- schema.addField(new DateTimeFieldSpec("ts", DataType.TIMESTAMP,
"TIMESTAMP", "1:DAYS"));
- return schema;
- }
-
- @Override
- protected IngestionConfig getIngestionConfig() {
- IngestionConfig ingestionConfig = new IngestionConfig();
- List<TransformConfig> transformConfigs = new ArrayList<>();
- transformConfigs.add(new TransformConfig("ts", "1577836800000 + ArrTime *
1000"));
- ingestionConfig.setTransformConfigs(transformConfigs);
- return ingestionConfig;
- }
-
- @Override
- protected List<FieldConfig> getFieldConfigs() {
- return List.of(new FieldConfig.Builder("ts").withTimestampConfig(
- new
TimestampConfig(List.of(TimestampIndexGranularity.SECOND))).build());
- }
-
- @Test
- public void timestampIndexSubstitutedInProjections() {
- setUseMultiStageQueryEngine(false);
- explainSse("SELECT datetrunc('SECOND', ts) FROM mytable",
- "[BROKER_REDUCE(limit:10), 1, 0]",
- "[COMBINE_SELECT, 2, 1]",
- "[PLAN_START(numSegmentsForThisPlan:1), -1, -1]",
- "[SELECT(selectList:$ts$SECOND), 3, 2]",
- "[PROJECT($ts$SECOND), 4, 3]",
- "[DOC_ID_SET, 5, 4]",
- Pattern.compile("\\[FILTER_MATCH_ENTIRE_SEGMENT\\(docs:[0-9]+\\), 6,
5]"));
- }
-
- @Test
- public void timestampIndexSubstitutedInAggregateFilter() {
- setUseMultiStageQueryEngine(false);
- explainSse("SELECT sum(case when datetrunc('SECOND', ts) > 1577836801000
then 2 else 0 end) FROM mytable",
- "[BROKER_REDUCE(limit:10), 1, 0]",
- "[COMBINE_AGGREGATE, 2, 1]",
- "[PLAN_START(numSegmentsForThisPlan:1), -1, -1]",
-
"[AGGREGATE(aggregations:sum(case(greater_than($ts$SECOND,'1577836801000'),'2','0'))),
3, 2]",
- "[TRANSFORM(case(greater_than($ts$SECOND,'1577836801000'),'2','0')),
4, 3]",
- "[PROJECT($ts$SECOND), 5, 4]",
- "[DOC_ID_SET, 6, 5]",
- Pattern.compile("\\[FILTER_MATCH_ENTIRE_SEGMENT\\(docs:[0-9]+\\), 7,
6]"));
- }
-
- @Test
- public void timestampIndexSubstitutedInGroupBy() {
- setUseMultiStageQueryEngine(false);
- explainSse("SELECT count(*) FROM mytable group by datetrunc('SECOND', ts)",
- "[BROKER_REDUCE(limit:10), 1, 0]",
- "[COMBINE_GROUP_BY, 2, 1]",
- "[PLAN_START(numSegmentsForThisPlan:1), -1, -1]",
- "[GROUP_BY(groupKeys:$ts$SECOND, aggregations:count(*)), 3, 2]",
- "[PROJECT($ts$SECOND), 4, 3]",
- "[DOC_ID_SET, 5, 4]",
- Pattern.compile("\\[FILTER_MATCH_ENTIRE_SEGMENT\\(docs:[0-9]+\\), 6,
5]"));
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
