This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 9d9f4fcf44 Enhance and clean up star-tree tests (#15587)
9d9f4fcf44 is described below
commit 9d9f4fcf4445014bfa27be4bc60e3746153412f6
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Thu Apr 17 18:39:05 2025 -0600
Enhance and clean up star-tree tests (#15587)
---
.../BaseTableDataManagerNeedRefreshTest.java | 97 +++++-----
.../tests/OfflineClusterIntegrationTest.java | 208 ++++++++++-----------
.../v2/builder/StarTreeBuilderUtilsTest.java | 139 +++++++-------
.../v2/builder/StarTreeV2BuilderConfigTest.java | 63 ++++---
.../startree/AggregationFunctionColumnPair.java | 1 +
5 files changed, 266 insertions(+), 242 deletions(-)
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerNeedRefreshTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerNeedRefreshTest.java
index 451d5ed9cf..d802acf07a 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerNeedRefreshTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerNeedRefreshTest.java
@@ -22,7 +22,6 @@ import java.io.File;
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.Arrays;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
@@ -200,7 +199,7 @@ public class BaseTableDataManagerNeedRefreshTest {
throws Exception {
TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(DEFAULT_TABLE_NAME)
.setNullHandlingEnabled(true)
- .setNoDictionaryColumns(Collections.singletonList(TEXT_INDEX_COLUMN))
+ .setNoDictionaryColumns(List.of(TEXT_INDEX_COLUMN))
.build();
Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension(TEXT_INDEX_COLUMN,
FieldSpec.DataType.STRING)
.addSingleValueDimension(JSON_INDEX_COLUMN, FieldSpec.DataType.JSON)
@@ -409,8 +408,9 @@ public class BaseTableDataManagerNeedRefreshTest {
public void addStartreeIndex() {
// Test 1 : Adding a StarTree index should trigger segment refresh.
- StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(Collections.singletonList("Carrier"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ StarTreeIndexConfig starTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("Carrier"), null,
List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null,
+ 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
assertTrue(BASE_TABLE_DATA_MANAGER.isSegmentStale(new
IndexLoadingConfig(tableConfig, SCHEMA),
IMMUTABLE_SEGMENT_DATA_MANAGER).isStale());
@@ -423,15 +423,17 @@ public class BaseTableDataManagerNeedRefreshTest {
// trigger segment refresh.
// Create a segment with StarTree index on Carrier.
- StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(Collections.singletonList("Carrier"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ StarTreeIndexConfig starTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("Carrier"), null,
List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null,
+ 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
ImmutableSegmentDataManager segmentDataManager =
createImmutableSegmentDataManager(new IndexLoadingConfig(tableConfig,
SCHEMA), _testName, generateRows());
// Create a StarTree index on Distance.
- StarTreeIndexConfig newStarTreeIndexConfig = new
StarTreeIndexConfig(Collections.singletonList("Distance"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ StarTreeIndexConfig newStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("Distance"), null,
List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null,
+ 100);
TableConfig newTableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(newStarTreeIndexConfig)).build();
assertTrue(
@@ -446,14 +448,15 @@ public class BaseTableDataManagerNeedRefreshTest {
// refresh.
// Create a segment with StarTree index on Carrier.
- StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(Collections.singletonList("Carrier"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ StarTreeIndexConfig starTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("Carrier"), null,
List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null,
+ 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
ImmutableSegmentDataManager segmentDataManager =
createImmutableSegmentDataManager(new IndexLoadingConfig(tableConfig,
SCHEMA), _testName, generateRows());
StarTreeIndexConfig newStarTreeIndexConfig = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
TableConfig newTableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(newStarTreeIndexConfig)).build();
assertTrue(
@@ -469,14 +472,14 @@ public class BaseTableDataManagerNeedRefreshTest {
// Create a segment with StarTree index on Carrier, Distance.
StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
ImmutableSegmentDataManager segmentDataManager =
createImmutableSegmentDataManager(new IndexLoadingConfig(tableConfig,
SCHEMA), _testName, generateRows());
// Create a StarTree index.
StarTreeIndexConfig newStarTreeIndexConfig = new
StarTreeIndexConfig(Arrays.asList("Distance", "Carrier"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
TableConfig newTableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(newStarTreeIndexConfig)).build();
assertTrue(
@@ -491,7 +494,7 @@ public class BaseTableDataManagerNeedRefreshTest {
// Create a segment with StarTree index on Carrier, Distance.
StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
ImmutableSegmentDataManager segmentDataManager =
createImmutableSegmentDataManager(new IndexLoadingConfig(tableConfig,
SCHEMA), _testName, generateRows());
@@ -499,7 +502,7 @@ public class BaseTableDataManagerNeedRefreshTest {
// Create a StarTree index.
StarTreeIndexConfig newStarTreeIndexConfig =
new StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"),
Arrays.asList("Carrier", "Distance"),
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
TableConfig newTableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(newStarTreeIndexConfig)).build();
assertTrue(
@@ -515,14 +518,14 @@ public class BaseTableDataManagerNeedRefreshTest {
StarTreeIndexConfig starTreeIndexConfig =
new StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"),
Arrays.asList("Carrier", "Distance"),
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
ImmutableSegmentDataManager segmentDataManager =
createImmutableSegmentDataManager(new IndexLoadingConfig(tableConfig,
SCHEMA), _testName, generateRows());
StarTreeIndexConfig newStarTreeIndexConfig =
new StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"),
Arrays.asList("Distance", "Carrier"),
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
TableConfig newTableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(newStarTreeIndexConfig)).build();
assertFalse(
@@ -537,13 +540,13 @@ public class BaseTableDataManagerNeedRefreshTest {
StarTreeIndexConfig starTreeIndexConfig =
new StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"),
Arrays.asList("Carrier", "Distance"),
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
ImmutableSegmentDataManager segmentDataManager =
createImmutableSegmentDataManager(new IndexLoadingConfig(tableConfig,
SCHEMA), _testName, generateRows());
StarTreeIndexConfig newStarTreeIndexConfig = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
TableConfig newTableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(newStarTreeIndexConfig)).build();
assertTrue(
@@ -557,13 +560,13 @@ public class BaseTableDataManagerNeedRefreshTest {
// Test 8: Adding a new StarTree index with an added metrics aggregation
function should trigger segment refresh.
StarTreeIndexConfig starTreeIndex = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndex)).build();
ImmutableSegmentDataManager segmentDataManager =
createImmutableSegmentDataManager(new IndexLoadingConfig(tableConfig,
SCHEMA), _testName, generateRows());
StarTreeIndexConfig newStarTreeIndexConfig = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null,
- Arrays.asList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName(),
"MAX__Distance"), null, 100);
+ Arrays.asList(AggregationFunctionColumnPair.COUNT_STAR_NAME,
"MAX__Distance"), null, 100);
TableConfig newTableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(newStarTreeIndexConfig)).build();
assertTrue(
@@ -578,13 +581,13 @@ public class BaseTableDataManagerNeedRefreshTest {
// trigger segment refresh.
StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null,
- Arrays.asList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName(),
"MAX__Distance"), null, 100);
+ Arrays.asList(AggregationFunctionColumnPair.COUNT_STAR_NAME,
"MAX__Distance"), null, 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
ImmutableSegmentDataManager segmentDataManager =
createImmutableSegmentDataManager(new IndexLoadingConfig(tableConfig,
SCHEMA), _testName, generateRows());
StarTreeIndexConfig newStarTreeIndexConfig = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null,
- Arrays.asList("MAX__Distance",
AggregationFunctionColumnPair.COUNT_STAR.toColumnName()), null, 100);
+ Arrays.asList("MAX__Distance",
AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
TableConfig newTableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(newStarTreeIndexConfig)).build();
assertFalse(
@@ -598,7 +601,7 @@ public class BaseTableDataManagerNeedRefreshTest {
// Test 10: removing an aggregation function through aggregation config
should trigger segment refresh.
StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null,
- Arrays.asList("MAX__Distance",
AggregationFunctionColumnPair.COUNT_STAR.toColumnName()), null, 100);
+ Arrays.asList("MAX__Distance",
AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
ImmutableSegmentDataManager segmentDataManager =
@@ -628,8 +631,7 @@ public class BaseTableDataManagerNeedRefreshTest {
// Create a StarTree index.
StarTreeIndexConfig newStarTreeIndexConfig = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
- Collections.singletonList(aggregationConfig), 100);
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME),
List.of(aggregationConfig), 100);
TableConfig newTableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(newStarTreeIndexConfig)).build();
assertTrue(
@@ -645,8 +647,7 @@ public class BaseTableDataManagerNeedRefreshTest {
StarTreeAggregationConfig aggregationConfig = new
StarTreeAggregationConfig("Distance", "MAX");
StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
- Collections.singletonList(aggregationConfig), 100);
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME),
List.of(aggregationConfig), 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
ImmutableSegmentDataManager segmentDataManager =
createImmutableSegmentDataManager(new IndexLoadingConfig(tableConfig,
SCHEMA), _testName, generateRows());
@@ -665,14 +666,16 @@ public class BaseTableDataManagerNeedRefreshTest {
@Test
void testStarTreeIndexMaxLeafNode()
throws Exception {
- StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(Collections.singletonList("Carrier"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ StarTreeIndexConfig starTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("Carrier"), null,
List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null,
+ 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
ImmutableSegmentDataManager segmentDataManager =
createImmutableSegmentDataManager(new IndexLoadingConfig(tableConfig,
SCHEMA), _testName, generateRows());
- StarTreeIndexConfig newStarTreeIndexConfig = new
StarTreeIndexConfig(Collections.singletonList("Carrier"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 10);
+ StarTreeIndexConfig newStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("Carrier"), null,
List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null,
+ 10);
TableConfig newTableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(newStarTreeIndexConfig)).build();
assertTrue(
@@ -683,8 +686,9 @@ public class BaseTableDataManagerNeedRefreshTest {
@Test
void testStarTreeIndexRemove()
throws Exception {
- StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(Collections.singletonList("Carrier"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ StarTreeIndexConfig starTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("Carrier"), null,
List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null,
+ 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
ImmutableSegmentDataManager segmentDataManager =
createImmutableSegmentDataManager(new IndexLoadingConfig(tableConfig,
SCHEMA), _testName, generateRows());
@@ -696,14 +700,16 @@ public class BaseTableDataManagerNeedRefreshTest {
throws Exception {
// Test 15: Add multiple StarTree Indexes should trigger segment refresh.
- StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(Collections.singletonList("Carrier"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ StarTreeIndexConfig starTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("Carrier"), null,
List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null,
+ 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
ImmutableSegmentDataManager segmentDataManager =
createImmutableSegmentDataManager(new IndexLoadingConfig(tableConfig,
SCHEMA), _testName, generateRows());
- StarTreeIndexConfig newStarTreeIndexConfig = new
StarTreeIndexConfig(Collections.singletonList("Distance"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ StarTreeIndexConfig newStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("Distance"), null,
List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null,
+ 100);
TableConfig newTableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig,
newStarTreeIndexConfig)).build();
assertTrue(
@@ -716,8 +722,9 @@ public class BaseTableDataManagerNeedRefreshTest {
throws Exception {
// Test 16: Enabling default StarTree index should trigger a segment
refresh.
- StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(Collections.singletonList("Carrier"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ StarTreeIndexConfig starTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("Carrier"), null,
List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null,
+ 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
ImmutableSegmentDataManager segmentDataManager =
createImmutableSegmentDataManager(new IndexLoadingConfig(tableConfig,
SCHEMA), _testName, generateRows());
@@ -734,8 +741,9 @@ public class BaseTableDataManagerNeedRefreshTest {
throws Exception {
// Test 17: Attempting to trigger segment refresh again should not be
successful.
- StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(Collections.singletonList("Carrier"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ StarTreeIndexConfig starTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("Carrier"), null,
List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null,
+ 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
IndexLoadingConfig indexLoadingConfig = new
IndexLoadingConfig(tableConfig, SCHEMA);
ImmutableSegmentDataManager segmentDataManager =
@@ -748,8 +756,9 @@ public class BaseTableDataManagerNeedRefreshTest {
throws Exception {
// Test 18: Disabling default StarTree index should trigger a segment
refresh.
- StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(Collections.singletonList("Carrier"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ StarTreeIndexConfig starTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("Carrier"), null,
List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null,
+ 100);
TableConfig tableConfig =
getTableConfigBuilder().setStarTreeIndexConfigs(List.of(starTreeIndexConfig)).build();
tableConfig.getIndexingConfig().setEnableDefaultStarTree(true);
ImmutableSegmentDataManager segmentDataManager =
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
index f020fa0276..4dba0fb6e4 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
@@ -34,7 +34,6 @@ import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -137,29 +136,29 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
// For inverted index triggering test
private static final List<String> UPDATED_INVERTED_INDEX_COLUMNS =
- Arrays.asList("FlightNum", "Origin", "Quarter", "DivActualElapsedTime");
+ List.of("FlightNum", "Origin", "Quarter", "DivActualElapsedTime");
private static final String TEST_UPDATED_INVERTED_INDEX_QUERY =
"SELECT COUNT(*) FROM mytable WHERE DivActualElapsedTime = 305";
// For range index triggering test
- private static final List<String> UPDATED_RANGE_INDEX_COLUMNS =
Collections.singletonList("DivActualElapsedTime");
+ private static final List<String> UPDATED_RANGE_INDEX_COLUMNS =
List.of("DivActualElapsedTime");
private static final String TEST_UPDATED_RANGE_INDEX_QUERY =
"SELECT COUNT(*) FROM mytable WHERE DivActualElapsedTime > 305";
// For bloom filter triggering test
- private static final List<String> UPDATED_BLOOM_FILTER_COLUMNS =
Collections.singletonList("Carrier");
+ private static final List<String> UPDATED_BLOOM_FILTER_COLUMNS =
List.of("Carrier");
private static final String TEST_UPDATED_BLOOM_FILTER_QUERY = "SELECT
COUNT(*) FROM mytable WHERE Carrier = 'CA'";
// For star-tree triggering test
private static final StarTreeIndexConfig STAR_TREE_INDEX_CONFIG_1 =
- new StarTreeIndexConfig(Collections.singletonList("Carrier"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ new StarTreeIndexConfig(List.of("Carrier"), null,
List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null,
+ 100);
private static final String TEST_STAR_TREE_QUERY_1 = "SELECT COUNT(*) FROM
mytable WHERE Carrier = 'UA'";
private static final String TEST_STAR_TREE_QUERY_1_FILTER_INVERT =
"SELECT COUNT(*) FILTER (WHERE Carrier = 'UA') FROM mytable";
private static final StarTreeIndexConfig STAR_TREE_INDEX_CONFIG_2 =
- new StarTreeIndexConfig(Collections.singletonList("DestState"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ new StarTreeIndexConfig(List.of("DestState"), null,
List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null,
+ 100);
private static final String TEST_STAR_TREE_QUERY_2 = "SELECT COUNT(*) FROM
mytable WHERE DestState = 'CA'";
// For default columns test
@@ -202,9 +201,9 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
@Override
protected List<FieldConfig> getFieldConfigs() {
- return Collections.singletonList(
- new FieldConfig("DivAirports", FieldConfig.EncodingType.DICTIONARY,
Collections.emptyList(),
- CompressionCodec.MV_ENTRY_DICT, null));
+ return List.of(
+ new FieldConfig("DivAirports", FieldConfig.EncodingType.DICTIONARY,
List.of(), CompressionCodec.MV_ENTRY_DICT,
+ null));
}
@Override
@@ -1819,121 +1818,112 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
@Test
public void testStarTreeTriggering()
throws Exception {
- long numTotalDocs = getCountStarResult();
+ int numTotalDocs = (int) getCountStarResult();
long tableSizeWithDefaultIndex = getTableSize(getTableName());
// Test the first query
JsonNode firstQueryResponse = postQuery(TEST_STAR_TREE_QUERY_1);
int firstQueryResult =
firstQueryResponse.get("resultTable").get("rows").get(0).get(0).asInt();
- assertEquals(firstQueryResponse.get("totalDocs").asLong(), numTotalDocs);
// Initially 'numDocsScanned' should be the same as 'COUNT(*)' result
- assertEquals(firstQueryResponse.get("numDocsScanned").asInt(),
firstQueryResult);
+ verifySingleValueResponse(firstQueryResponse, firstQueryResult,
numTotalDocs, firstQueryResult);
// Verify that inverting the filter to be a filtered agg shows the
identical results
- firstQueryResponse = postQuery(TEST_STAR_TREE_QUERY_1_FILTER_INVERT);
-
assertEquals(firstQueryResponse.get("resultTable").get("rows").get(0).get(0).asInt(),
firstQueryResult);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_1_FILTER_INVERT),
firstQueryResult, numTotalDocs,
+ firstQueryResult);
- // Update table config and trigger reload
+ // Update table config without enabling dynamic star-tree creation and
trigger reload, should have no effect
TableConfig tableConfig = getOfflineTableConfig();
IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
-
indexingConfig.setStarTreeIndexConfigs(Collections.singletonList(STAR_TREE_INDEX_CONFIG_1));
+ indexingConfig.setStarTreeIndexConfigs(List.of(STAR_TREE_INDEX_CONFIG_1));
+ indexingConfig.setEnableDynamicStarTreeCreation(false);
+ updateTableConfig(tableConfig);
+ reloadAllSegments(TEST_STAR_TREE_QUERY_1, false, numTotalDocs);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_1),
firstQueryResult, numTotalDocs, firstQueryResult);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_1_FILTER_INVERT),
firstQueryResult, numTotalDocs,
+ firstQueryResult);
+
+ // Set enableDynamicStarTreeCreation to true and trigger reload
indexingConfig.setEnableDynamicStarTreeCreation(true);
updateTableConfig(tableConfig);
reloadAllSegments(TEST_STAR_TREE_QUERY_1, false, numTotalDocs);
// With star-tree, 'numDocsScanned' should be the same as number of
segments (1 per segment)
-
assertEquals(postQuery(TEST_STAR_TREE_QUERY_1).get("numDocsScanned").asLong(),
NUM_SEGMENTS);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_1),
firstQueryResult, numTotalDocs, NUM_SEGMENTS);
// Verify that inverting the filter to be a filtered agg shows the
identical results
- firstQueryResponse = postQuery(TEST_STAR_TREE_QUERY_1_FILTER_INVERT);
-
assertEquals(firstQueryResponse.get("resultTable").get("rows").get(0).get(0).asInt(),
firstQueryResult);
- assertEquals(firstQueryResponse.get("totalDocs").asLong(), numTotalDocs);
- assertEquals(firstQueryResponse.get("numDocsScanned").asInt(),
NUM_SEGMENTS);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_1_FILTER_INVERT),
firstQueryResult, numTotalDocs,
+ NUM_SEGMENTS);
// Reload again should have no effect
reloadAllSegments(TEST_STAR_TREE_QUERY_1, false, numTotalDocs);
- firstQueryResponse = postQuery(TEST_STAR_TREE_QUERY_1);
-
assertEquals(firstQueryResponse.get("resultTable").get("rows").get(0).get(0).asInt(),
firstQueryResult);
- assertEquals(firstQueryResponse.get("totalDocs").asLong(), numTotalDocs);
- assertEquals(firstQueryResponse.get("numDocsScanned").asInt(),
NUM_SEGMENTS);
- // Verify that inverting the filter to be a filtered agg shows the
identical results
- firstQueryResponse = postQuery(TEST_STAR_TREE_QUERY_1_FILTER_INVERT);
-
assertEquals(firstQueryResponse.get("resultTable").get("rows").get(0).get(0).asInt(),
firstQueryResult);
- assertEquals(firstQueryResponse.get("totalDocs").asLong(), numTotalDocs);
- assertEquals(firstQueryResponse.get("numDocsScanned").asInt(),
NUM_SEGMENTS);
-
- // Enforce a sleep here since segment reload is async and there is another
back-to-back reload below.
- // Otherwise, there is no way to tell whether the 1st reload on server
side is finished,
- // which may hit the race condition that the 1st reload finishes after the
2nd reload is fully done.
- // 10 seconds are still better than hitting race condition which will time
out after 10 minutes.
- Thread.sleep(10_000L);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_1),
firstQueryResult, numTotalDocs, NUM_SEGMENTS);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_1_FILTER_INVERT),
firstQueryResult, numTotalDocs,
+ NUM_SEGMENTS);
// Should be able to use the star-tree with an additional match-all
predicate on another dimension
- firstQueryResponse = postQuery(TEST_STAR_TREE_QUERY_1 + " AND
DaysSinceEpoch > 16070");
-
assertEquals(firstQueryResponse.get("resultTable").get("rows").get(0).get(0).asInt(),
firstQueryResult);
- assertEquals(firstQueryResponse.get("totalDocs").asLong(), numTotalDocs);
- assertEquals(firstQueryResponse.get("numDocsScanned").asInt(),
NUM_SEGMENTS);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_1 + " AND
DaysSinceEpoch > 16070"), firstQueryResult,
+ numTotalDocs, NUM_SEGMENTS);
// Test the second query
JsonNode secondQueryResponse = postQuery(TEST_STAR_TREE_QUERY_2);
int secondQueryResult =
secondQueryResponse.get("resultTable").get("rows").get(0).get(0).asInt();
- assertEquals(secondQueryResponse.get("totalDocs").asLong(), numTotalDocs);
// Initially 'numDocsScanned' should be the same as 'COUNT(*)' result
- assertEquals(secondQueryResponse.get("numDocsScanned").asInt(),
secondQueryResult);
+ verifySingleValueResponse(secondQueryResponse, secondQueryResult,
numTotalDocs, secondQueryResult);
- // Update table config with a different star-tree index config and trigger
reload
-
indexingConfig.setStarTreeIndexConfigs(Collections.singletonList(STAR_TREE_INDEX_CONFIG_2));
+ // Update table config without enabling dynamic star-tree creation and
trigger reload, should have no effect
+ indexingConfig.setStarTreeIndexConfigs(List.of(STAR_TREE_INDEX_CONFIG_2));
+ indexingConfig.setEnableDynamicStarTreeCreation(false);
+ updateTableConfig(tableConfig);
+ reloadAllSegments(TEST_STAR_TREE_QUERY_2, false, numTotalDocs);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_2),
secondQueryResult, numTotalDocs, secondQueryResult);
+
+ // Set enableDynamicStarTreeCreation to true and trigger reload
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
updateTableConfig(tableConfig);
reloadAllSegments(TEST_STAR_TREE_QUERY_2, false, numTotalDocs);
// With star-tree, 'numDocsScanned' should be the same as number of
segments (1 per segment)
-
assertEquals(postQuery(TEST_STAR_TREE_QUERY_2).get("numDocsScanned").asLong(),
NUM_SEGMENTS);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_2),
secondQueryResult, numTotalDocs, NUM_SEGMENTS);
// First query should not be able to use the star-tree
- firstQueryResponse = postQuery(TEST_STAR_TREE_QUERY_1);
- assertEquals(firstQueryResponse.get("numDocsScanned").asInt(),
firstQueryResult);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_1),
firstQueryResult, numTotalDocs, firstQueryResult);
// Reload again should have no effect
reloadAllSegments(TEST_STAR_TREE_QUERY_2, false, numTotalDocs);
- firstQueryResponse = postQuery(TEST_STAR_TREE_QUERY_1);
-
assertEquals(firstQueryResponse.get("resultTable").get("rows").get(0).get(0).asInt(),
firstQueryResult);
- assertEquals(firstQueryResponse.get("totalDocs").asLong(), numTotalDocs);
- assertEquals(firstQueryResponse.get("numDocsScanned").asInt(),
firstQueryResult);
- secondQueryResponse = postQuery(TEST_STAR_TREE_QUERY_2);
-
assertEquals(secondQueryResponse.get("resultTable").get("rows").get(0).get(0).asInt(),
secondQueryResult);
- assertEquals(secondQueryResponse.get("totalDocs").asLong(), numTotalDocs);
- assertEquals(secondQueryResponse.get("numDocsScanned").asInt(),
NUM_SEGMENTS);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_1),
firstQueryResult, numTotalDocs, firstQueryResult);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_2),
secondQueryResult, numTotalDocs, NUM_SEGMENTS);
// Should be able to use the star-tree with an additional match-all
predicate on another dimension
- secondQueryResponse = postQuery(TEST_STAR_TREE_QUERY_2 + " AND
DaysSinceEpoch > 16070");
-
assertEquals(secondQueryResponse.get("resultTable").get("rows").get(0).get(0).asInt(),
secondQueryResult);
- assertEquals(secondQueryResponse.get("totalDocs").asLong(), numTotalDocs);
- assertEquals(secondQueryResponse.get("numDocsScanned").asInt(),
NUM_SEGMENTS);
-
- // Enforce a sleep here since segment reload is async and there is another
back-to-back reload below.
- // Otherwise, there is no way to tell whether the 1st reload on server
side is finished,
- // which may hit the race condition that the 1st reload finishes after the
2nd reload is fully done.
- // 10 seconds are still better than hitting race condition which will time
out after 10 minutes.
- Thread.sleep(10_000L);
-
- // Remove the star-tree index config and trigger reload
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_2 + " AND
DaysSinceEpoch > 16070"), secondQueryResult,
+ numTotalDocs, NUM_SEGMENTS);
+
+ // Remove the star-tree index config without enabling dynamic star-tree
creation and trigger reload, should have no
+ // effect
indexingConfig.setStarTreeIndexConfigs(null);
+ indexingConfig.setEnableDynamicStarTreeCreation(false);
+ updateTableConfig(tableConfig);
+ reloadAllSegments(TEST_STAR_TREE_QUERY_2, false, numTotalDocs);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_1),
firstQueryResult, numTotalDocs, firstQueryResult);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_2),
secondQueryResult, numTotalDocs, NUM_SEGMENTS);
+
+ // Set enableDynamicStarTreeCreation to true and trigger reload
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
updateTableConfig(tableConfig);
reloadAllSegments(TEST_STAR_TREE_QUERY_2, false, numTotalDocs);
// Without star-tree, 'numDocsScanned' should be the same as the
'COUNT(*)' result
-
assertEquals(postQuery(TEST_STAR_TREE_QUERY_2).get("numDocsScanned").asLong(),
secondQueryResult);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_2),
secondQueryResult, numTotalDocs, secondQueryResult);
assertEquals(getTableSize(getTableName()), tableSizeWithDefaultIndex);
// First query should not be able to use the star-tree
- firstQueryResponse = postQuery(TEST_STAR_TREE_QUERY_1);
- assertEquals(firstQueryResponse.get("numDocsScanned").asInt(),
firstQueryResult);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_1),
firstQueryResult, numTotalDocs, firstQueryResult);
// Reload again should have no effect
reloadAllSegments(TEST_STAR_TREE_QUERY_2, false, numTotalDocs);
- firstQueryResponse = postQuery(TEST_STAR_TREE_QUERY_1);
-
assertEquals(firstQueryResponse.get("resultTable").get("rows").get(0).get(0).asInt(),
firstQueryResult);
- assertEquals(firstQueryResponse.get("totalDocs").asLong(), numTotalDocs);
- assertEquals(firstQueryResponse.get("numDocsScanned").asInt(),
firstQueryResult);
- secondQueryResponse = postQuery(TEST_STAR_TREE_QUERY_2);
-
assertEquals(secondQueryResponse.get("resultTable").get("rows").get(0).get(0).asInt(),
secondQueryResult);
- assertEquals(secondQueryResponse.get("totalDocs").asLong(), numTotalDocs);
- assertEquals(secondQueryResponse.get("numDocsScanned").asInt(),
secondQueryResult);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_1),
firstQueryResult, numTotalDocs, firstQueryResult);
+ verifySingleValueResponse(postQuery(TEST_STAR_TREE_QUERY_2),
secondQueryResult, numTotalDocs, secondQueryResult);
+ }
+
+ private void verifySingleValueResponse(JsonNode queryResponse, int
expectedResult, int expectedTotalDocs,
+ int expectedDocsScanned) {
+
assertEquals(queryResponse.get("resultTable").get("rows").get(0).get(0).asInt(),
expectedResult);
+ assertEquals(queryResponse.get("totalDocs").asInt(), expectedTotalDocs);
+ assertEquals(queryResponse.get("numDocsScanned").asInt(),
expectedDocsScanned);
}
/**
@@ -2067,17 +2057,18 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
addSchema(schema);
TableConfig tableConfig = getOfflineTableConfig();
- List<TransformConfig> transformConfigs =
- Arrays.asList(new TransformConfig("NewAddedDerivedHoursSinceEpoch",
"DaysSinceEpoch * 24"),
- new TransformConfig("NewAddedDerivedTimestamp", "DaysSinceEpoch *
24 * 3600 * 1000"),
- new TransformConfig("NewAddedDerivedSVBooleanDimension",
"ActualElapsedTime > 0"),
- new TransformConfig("NewAddedDerivedMVStringDimension",
"split(DestCityName, ', ')"),
- new TransformConfig("NewAddedDerivedDivAirportSeqIDs",
"DivAirportSeqIDs"),
- new TransformConfig("NewAddedDerivedDivAirportSeqIDsString",
"DivAirportSeqIDs"),
- new TransformConfig("NewAddedRawDerivedStringDimension",
"reverse(DestCityName)"),
- new TransformConfig("NewAddedRawDerivedMVIntDimension",
"array(ActualElapsedTime)"),
- new TransformConfig("NewAddedDerivedMVDoubleDimension",
"array(ArrDelayMinutes)"),
- new TransformConfig("NewAddedDerivedNullString", "caseWhen(true,
null, null)"));
+ List<TransformConfig> transformConfigs = List.of(
+ new TransformConfig("NewAddedDerivedHoursSinceEpoch", "DaysSinceEpoch
* 24"),
+ new TransformConfig("NewAddedDerivedTimestamp", "DaysSinceEpoch * 24 *
3600 * 1000"),
+ new TransformConfig("NewAddedDerivedSVBooleanDimension",
"ActualElapsedTime > 0"),
+ new TransformConfig("NewAddedDerivedMVStringDimension",
"split(DestCityName, ', ')"),
+ new TransformConfig("NewAddedDerivedDivAirportSeqIDs",
"DivAirportSeqIDs"),
+ new TransformConfig("NewAddedDerivedDivAirportSeqIDsString",
"DivAirportSeqIDs"),
+ new TransformConfig("NewAddedRawDerivedStringDimension",
"reverse(DestCityName)"),
+ new TransformConfig("NewAddedRawDerivedMVIntDimension",
"array(ActualElapsedTime)"),
+ new TransformConfig("NewAddedDerivedMVDoubleDimension",
"array(ArrDelayMinutes)"),
+ new TransformConfig("NewAddedDerivedNullString", "caseWhen(true, null,
null)")
+ );
IngestionConfig ingestionConfig = new IngestionConfig();
ingestionConfig.setTransformConfigs(transformConfigs);
tableConfig.setIngestionConfig(ingestionConfig);
@@ -2087,10 +2078,11 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
List<FieldConfig> fieldConfigList = tableConfig.getFieldConfigList();
assertNotNull(fieldConfigList);
fieldConfigList.add(
- new FieldConfig("NewAddedDerivedDivAirportSeqIDs",
FieldConfig.EncodingType.DICTIONARY, Collections.emptyList(),
+ new FieldConfig("NewAddedDerivedDivAirportSeqIDs",
FieldConfig.EncodingType.DICTIONARY, List.of(),
+ CompressionCodec.MV_ENTRY_DICT, null));
+ fieldConfigList.add(
+ new FieldConfig("NewAddedDerivedDivAirportSeqIDsString",
FieldConfig.EncodingType.DICTIONARY, List.of(),
CompressionCodec.MV_ENTRY_DICT, null));
- fieldConfigList.add(new
FieldConfig("NewAddedDerivedDivAirportSeqIDsString",
FieldConfig.EncodingType.DICTIONARY,
- Collections.emptyList(), CompressionCodec.MV_ENTRY_DICT, null));
updateTableConfig(tableConfig);
// Trigger reload
@@ -2421,8 +2413,8 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
// Add expression override
TableConfig tableConfig = getOfflineTableConfig();
- tableConfig.setQueryConfig(new QueryConfig(null, null, null,
- Collections.singletonMap("DaysSinceEpoch * 24",
"NewAddedDerivedHoursSinceEpoch"), null, null));
+ tableConfig.setQueryConfig(
+ new QueryConfig(null, null, null, Map.of("DaysSinceEpoch * 24",
"NewAddedDerivedHoursSinceEpoch"), null, null));
updateTableConfig(tableConfig);
TestUtils.waitForCondition(aVoid -> {
@@ -2919,8 +2911,8 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
throws Exception {
setUseMultiStageQueryEngine(useMultiStageQueryEngine);
List<String> origins =
- Arrays.asList("ATL", "ORD", "DFW", "DEN", "LAX", "IAH", "SFO", "PHX",
"LAS", "EWR", "MCO", "BOS", "SLC", "SEA",
- "MSP", "CLT", "LGA", "DTW", "JFK", "BWI");
+ List.of("ATL", "ORD", "DFW", "DEN", "LAX", "IAH", "SFO", "PHX", "LAS",
"EWR", "MCO", "BOS", "SLC", "SEA", "MSP",
+ "CLT", "LGA", "DTW", "JFK", "BWI");
StringBuilder caseStatementBuilder = new StringBuilder("CASE ");
for (int i = 0; i < origins.size(); i++) {
// WHEN Origin = 'ATL' THEN 1
@@ -3365,7 +3357,8 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
int daysSinceEpoch = 16138;
int hoursSinceEpoch = 16138 * 24;
int secondsSinceEpoch = 16138 * 24 * 60 * 60;
- List<String> baseQueries = Arrays.asList("SELECT * FROM mytable limit
10000",
+ List<String> baseQueries = List.of(
+ "SELECT * FROM mytable limit 10000",
"SELECT DaysSinceEpoch, timeConvert(DaysSinceEpoch,'DAYS','SECONDS')
FROM mytable limit 10000",
"SELECT DaysSinceEpoch, timeConvert(DaysSinceEpoch,'DAYS','SECONDS')
FROM mytable order by DaysSinceEpoch "
+ "limit 10000",
@@ -3378,7 +3371,8 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
+ " limit 10000",
"SELECT MAX(timeConvert(DaysSinceEpoch,'DAYS','SECONDS')) FROM mytable
limit 10000",
"SELECT COUNT(*) FROM mytable GROUP BY
dateTimeConvert(DaysSinceEpoch,'1:DAYS:EPOCH','1:HOURS:EPOCH',"
- + "'1:HOURS') limit 10000");
+ + "'1:HOURS') limit 10000"
+ );
List<String> queries = new ArrayList<>();
baseQueries.forEach(q -> queries.add(q.replace("mytable",
"MYTABLE").replace("DaysSinceEpoch", "DAYSSinceEpOch")));
baseQueries.forEach(
@@ -3396,7 +3390,8 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
int daysSinceEpoch = 16138;
int hoursSinceEpoch = 16138 * 24;
int secondsSinceEpoch = 16138 * 24 * 60 * 60;
- List<String> baseQueries = Arrays.asList("SELECT * FROM mytable",
+ List<String> baseQueries = List.of(
+ "SELECT * FROM mytable",
"SELECT DaysSinceEpoch, timeConvert(DaysSinceEpoch,'DAYS','SECONDS')
FROM mytable",
"SELECT DaysSinceEpoch, timeConvert(DaysSinceEpoch,'DAYS','SECONDS')
FROM mytable order by DaysSinceEpoch "
+ "limit 10000",
@@ -3406,8 +3401,8 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
"SELECT count(*) FROM mytable WHERE
timeConvert(DaysSinceEpoch,'DAYS','HOURS') = " + hoursSinceEpoch,
"SELECT count(*) FROM mytable WHERE
timeConvert(DaysSinceEpoch,'DAYS','SECONDS') = " + secondsSinceEpoch,
"SELECT MAX(timeConvert(DaysSinceEpoch,'DAYS','SECONDS')) FROM
mytable",
- "SELECT COUNT(*) FROM mytable GROUP BY
dateTimeConvert(DaysSinceEpoch,'1:DAYS:EPOCH','1:HOURS:EPOCH',"
- + "'1:HOURS')");
+ "SELECT COUNT(*) FROM mytable GROUP BY
dateTimeConvert(DaysSinceEpoch,'1:DAYS:EPOCH','1:HOURS:EPOCH','1:HOURS')"
+ );
List<String> queries = new ArrayList<>();
baseQueries.forEach(q -> queries.add(q.replace("DaysSinceEpoch",
"mytable.DAYSSinceEpOch")));
baseQueries.forEach(q -> queries.add(q.replace("DaysSinceEpoch",
"mytable.DAYSSinceEpOch")));
@@ -3425,7 +3420,8 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
int daysSinceEpoch = 16138;
int hoursSinceEpoch = 16138 * 24;
int secondsSinceEpoch = 16138 * 24 * 60 * 60;
- List<String> baseQueries = Arrays.asList("SELECT * FROM mytable",
+ List<String> baseQueries = List.of(
+ "SELECT * FROM mytable",
"SELECT DaysSinceEpoch, timeConvert(DaysSinceEpoch,'DAYS','SECONDS')
FROM mytable",
"SELECT DaysSinceEpoch, timeConvert(DaysSinceEpoch,'DAYS','SECONDS')
FROM mytable order by DaysSinceEpoch "
+ "limit 10000",
@@ -3435,8 +3431,8 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
"SELECT count(*) FROM mytable WHERE
timeConvert(DaysSinceEpoch,'DAYS','HOURS') = " + hoursSinceEpoch,
"SELECT count(*) FROM mytable WHERE
timeConvert(DaysSinceEpoch,'DAYS','SECONDS') = " + secondsSinceEpoch,
"SELECT MAX(timeConvert(DaysSinceEpoch,'DAYS','SECONDS')) FROM
mytable",
- "SELECT COUNT(*) FROM mytable GROUP BY
dateTimeConvert(DaysSinceEpoch,'1:DAYS:EPOCH','1:HOURS:EPOCH',"
- + "'1:HOURS')");
+ "SELECT COUNT(*) FROM mytable GROUP BY
dateTimeConvert(DaysSinceEpoch,'1:DAYS:EPOCH','1:HOURS:EPOCH','1:HOURS')"
+ );
List<String> queries = new ArrayList<>();
baseQueries.forEach(
q -> queries.add(q.replace("mytable",
"MYTABLE").replace("DaysSinceEpoch", "MYTABLE.DAYSSinceEpOch")));
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeBuilderUtilsTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeBuilderUtilsTest.java
index ff83134dbd..9ed0194e20 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeBuilderUtilsTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeBuilderUtilsTest.java
@@ -23,8 +23,6 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -65,32 +63,32 @@ public class StarTreeBuilderUtilsTest {
@Test
public void testAreStarTreeBuilderConfigListsEqual() {
// Create StartTreeIndexConfigs to test for unequal starTree configs.
- StarTreeIndexConfig starTreeIndexConfig1 = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ StarTreeIndexConfig starTreeIndexConfig1 = new
StarTreeIndexConfig(List.of("Carrier", "Distance"), null,
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
// Different skip star node creation.
StarTreeIndexConfig starTreeIndexConfig2 =
- new StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"),
Collections.singletonList("Distance"),
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ new StarTreeIndexConfig(List.of("Carrier", "Distance"),
List.of("Distance"),
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
// Different dimension split order.
- StarTreeIndexConfig starTreeIndexConfig3 = new
StarTreeIndexConfig(Arrays.asList("Distance", "Carrier"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ StarTreeIndexConfig starTreeIndexConfig3 = new
StarTreeIndexConfig(List.of("Distance", "Carrier"), null,
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
// Different max leaf records.
- StarTreeIndexConfig starTreeIndexConfig4 = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 200);
+ StarTreeIndexConfig starTreeIndexConfig4 = new
StarTreeIndexConfig(List.of("Carrier", "Distance"), null,
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 200);
// Create StartTreeAggregationConfigs with StarTreeAggregationConfig.
StarTreeAggregationConfig starTreeAggregationConfig1 = new
StarTreeAggregationConfig("Distance", "MAX");
// Different AggregationConfig.
- StarTreeIndexConfig starTreeIndexConfig5 = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null, null,
- Collections.singletonList(starTreeAggregationConfig1), 100);
+ StarTreeIndexConfig starTreeIndexConfig5 =
+ new StarTreeIndexConfig(List.of("Carrier", "Distance"), null, null,
List.of(starTreeAggregationConfig1), 100);
// Create StarTreeIndexConfig for equality check.
- StarTreeIndexConfig starTreeIndexConfig6 = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ StarTreeIndexConfig starTreeIndexConfig6 = new
StarTreeIndexConfig(List.of("Carrier", "Distance"), null,
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
// test unequal builder config size.
List<StarTreeV2BuilderConfig> config1 = new ArrayList<>();
@@ -139,11 +137,15 @@ public class StarTreeBuilderUtilsTest {
// Create Schema and SegmentMetadata for testing.
Schema schema = new Schema.SchemaBuilder().addSingleValueDimension("d1",
FieldSpec.DataType.INT)
- .addSingleValueDimension("d2",
FieldSpec.DataType.LONG).addSingleValueDimension("d3", FieldSpec.DataType.FLOAT)
- .addSingleValueDimension("d4",
FieldSpec.DataType.DOUBLE).addMultiValueDimension("d5", FieldSpec.DataType.INT)
- .addMetric("m1", FieldSpec.DataType.DOUBLE).addMetric("m2",
FieldSpec.DataType.BYTES)
+ .addSingleValueDimension("d2", FieldSpec.DataType.LONG)
+ .addSingleValueDimension("d3", FieldSpec.DataType.FLOAT)
+ .addSingleValueDimension("d4", FieldSpec.DataType.DOUBLE)
+ .addMultiValueDimension("d5", FieldSpec.DataType.INT)
+ .addMetric("m1", FieldSpec.DataType.DOUBLE)
+ .addMetric("m2", FieldSpec.DataType.BYTES)
.addTime(new TimeGranularitySpec(FieldSpec.DataType.LONG,
TimeUnit.MILLISECONDS, "t"), null)
- .addDateTime("dt", FieldSpec.DataType.LONG, "1:MILLISECONDS:EPOCH",
"1:HOURS").build();
+ .addDateTime("dt", FieldSpec.DataType.LONG, "1:MILLISECONDS:EPOCH",
"1:HOURS")
+ .build();
SegmentMetadataImpl segmentMetadata = mock(SegmentMetadataImpl.class);
when(segmentMetadata.getSchema()).thenReturn(schema);
// Included
@@ -176,31 +178,30 @@ public class StarTreeBuilderUtilsTest {
// // Create a list of string with column name, hasDictionary and
cardinality.
List<List<String>> columnList =
- Arrays.asList(Arrays.asList("d1", "true", "200"), Arrays.asList("d2",
"true", "400"),
- Arrays.asList("d3", "true", "20000"), Arrays.asList("d4", "false",
"100"),
- Arrays.asList("d5", "true", "100"), Arrays.asList("m1", "false",
"-1"), Arrays.asList("m2", "true", "100"),
- Arrays.asList("t", "true", "20000"), Arrays.asList("dt", "true",
"30000"));
+ List.of(List.of("d1", "true", "200"), List.of("d2", "true", "400"),
List.of("d3", "true", "20000"),
+ List.of("d4", "false", "100"), List.of("d5", "true", "100"),
List.of("m1", "false", "-1"),
+ List.of("m2", "true", "100"), List.of("t", "true", "20000"),
List.of("dt", "true", "30000"));
// Convert the list of string to JsonNode with appropriate key names and
root node.
JsonNode segmentMetadataAsJsonNode =
convertStringListToJsonNode(columnList);
// Create StartTreeIndexConfig for testing.
- StarTreeIndexConfig starTreeIndexConfig1 = new
StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"), null,
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ StarTreeIndexConfig starTreeIndexConfig1 = new
StarTreeIndexConfig(List.of("Carrier", "Distance"), null,
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
StarTreeIndexConfig starTreeIndexConfig2 =
- new StarTreeIndexConfig(Arrays.asList("Carrier", "Distance"),
Collections.singletonList("Distance"),
-
Collections.singletonList(AggregationFunctionColumnPair.COUNT_STAR.toColumnName()),
null, 100);
+ new StarTreeIndexConfig(List.of("Carrier", "Distance"),
List.of("Distance"),
+ List.of(AggregationFunctionColumnPair.COUNT_STAR_NAME), null, 100);
// Create StartTreeV2BuilderConfig from segmentMetadataImpl.
List<StarTreeV2BuilderConfig> builderConfig1 =
-
StarTreeBuilderUtils.generateBuilderConfigs(Arrays.asList(starTreeIndexConfig1,
starTreeIndexConfig2), true,
+
StarTreeBuilderUtils.generateBuilderConfigs(List.of(starTreeIndexConfig1,
starTreeIndexConfig2), true,
segmentMetadata);
// Create StartTreeV2BuilderConfig from JsonNode.
List<StarTreeV2BuilderConfig> builderConfig2 =
-
StarTreeBuilderUtils.generateBuilderConfigs(Arrays.asList(starTreeIndexConfig1,
starTreeIndexConfig2), true,
- schema, segmentMetadataAsJsonNode);
+
StarTreeBuilderUtils.generateBuilderConfigs(List.of(starTreeIndexConfig1,
starTreeIndexConfig2), true, schema,
+ segmentMetadataAsJsonNode);
// They should be equal.
assertEquals(builderConfig1, builderConfig2);
@@ -215,16 +216,16 @@ public class StarTreeBuilderUtilsTest {
StarTreeV2Metadata.writeMetadata(metadataProperties, 1, List.of("col1"),
aggregationSpecs, 100, Set.of());
StarTreeV2Metadata existingStarTreeMetadata = new
StarTreeV2Metadata(metadataProperties);
- StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(List.of("col1"), null, null,
- List.of(new StarTreeAggregationConfig("col2", "DISTINCTCOUNTHLL",
Map.of(Constants.HLL_LOG2M_KEY, 16),
- null, null, null, null, null)), 100);
+ StarTreeIndexConfig starTreeIndexConfig = new
StarTreeIndexConfig(List.of("col1"), null, null, List.of(
+ new StarTreeAggregationConfig("col2", "DISTINCTCOUNTHLL",
Map.of(Constants.HLL_LOG2M_KEY, 16), null, null, null,
+ null, null)), 100);
assertFalse(StarTreeBuilderUtils.shouldModifyExistingStarTrees(
List.of(StarTreeV2BuilderConfig.fromIndexConfig(starTreeIndexConfig)),
List.of(existingStarTreeMetadata)));
// Change log2m value
- starTreeIndexConfig = new StarTreeIndexConfig(List.of("col1"), null, null,
- List.of(new StarTreeAggregationConfig("col2", "DISTINCTCOUNTHLL",
Map.of(Constants.HLL_LOG2M_KEY, 8),
- null, null, null, null, null)), 100);
+ starTreeIndexConfig = new StarTreeIndexConfig(List.of("col1"), null, null,
List.of(
+ new StarTreeAggregationConfig("col2", "DISTINCTCOUNTHLL",
Map.of(Constants.HLL_LOG2M_KEY, 8), null, null, null,
+ null, null)), 100);
assertTrue(StarTreeBuilderUtils.shouldModifyExistingStarTrees(
List.of(StarTreeV2BuilderConfig.fromIndexConfig(starTreeIndexConfig)),
List.of(existingStarTreeMetadata)));
}
@@ -232,72 +233,76 @@ public class StarTreeBuilderUtilsTest {
@Test
public void testExpressionContextFromFunctionParameters() {
// DISTINCTCOUNTHLL
- DistinctCountHLLValueAggregator hllValueAggregator =
(DistinctCountHLLValueAggregator)
-
ValueAggregatorFactory.getValueAggregator(AggregationFunctionType.DISTINCTCOUNTHLL,
-
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTHLL,
- Map.of(Constants.HLL_LOG2M_KEY, "10")));
+ DistinctCountHLLValueAggregator hllValueAggregator =
+ (DistinctCountHLLValueAggregator)
ValueAggregatorFactory.getValueAggregator(
+ AggregationFunctionType.DISTINCTCOUNTHLL,
+
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTHLL,
+ Map.of(Constants.HLL_LOG2M_KEY, "10")));
assertEquals(10, hllValueAggregator.getLog2m());
- hllValueAggregator = (DistinctCountHLLValueAggregator)
-
ValueAggregatorFactory.getValueAggregator(AggregationFunctionType.DISTINCTCOUNTHLL,
-
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTHLL,
- Map.of()));
+ hllValueAggregator = (DistinctCountHLLValueAggregator)
ValueAggregatorFactory.getValueAggregator(
+ AggregationFunctionType.DISTINCTCOUNTHLL,
+
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTHLL,
+ Map.of()));
// Verify default value used
assertEquals(8, hllValueAggregator.getLog2m());
// DISTINCTCOUNTHLLPLUS
- DistinctCountHLLPlusValueAggregator hllPlusValueAggregator =
(DistinctCountHLLPlusValueAggregator)
-
ValueAggregatorFactory.getValueAggregator(AggregationFunctionType.DISTINCTCOUNTHLLPLUS,
-
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTHLLPLUS,
- Map.of(Constants.HLLPLUS_ULL_P_KEY, "10",
Constants.HLLPLUS_SP_KEY, 20)));
+ DistinctCountHLLPlusValueAggregator hllPlusValueAggregator =
+ (DistinctCountHLLPlusValueAggregator)
ValueAggregatorFactory.getValueAggregator(
+ AggregationFunctionType.DISTINCTCOUNTHLLPLUS,
+
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTHLLPLUS,
+ Map.of(Constants.HLLPLUS_ULL_P_KEY, "10",
Constants.HLLPLUS_SP_KEY, 20)));
assertEquals(10, hllPlusValueAggregator.getP());
assertEquals(20, hllPlusValueAggregator.getSp());
- hllPlusValueAggregator = (DistinctCountHLLPlusValueAggregator)
-
ValueAggregatorFactory.getValueAggregator(AggregationFunctionType.DISTINCTCOUNTHLLPLUS,
+ hllPlusValueAggregator = (DistinctCountHLLPlusValueAggregator)
ValueAggregatorFactory.getValueAggregator(
+ AggregationFunctionType.DISTINCTCOUNTHLLPLUS,
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTHLLPLUS,
Map.of("garbage", "value")));
// Verify default values used
assertEquals(14, hllPlusValueAggregator.getP());
assertEquals(0, hllPlusValueAggregator.getSp());
- hllPlusValueAggregator = (DistinctCountHLLPlusValueAggregator)
-
ValueAggregatorFactory.getValueAggregator(AggregationFunctionType.DISTINCTCOUNTHLLPLUS,
+ hllPlusValueAggregator = (DistinctCountHLLPlusValueAggregator)
ValueAggregatorFactory.getValueAggregator(
+ AggregationFunctionType.DISTINCTCOUNTHLLPLUS,
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTHLLPLUS,
Map.of(Constants.HLLPLUS_ULL_P_KEY, "10")));
assertEquals(10, hllPlusValueAggregator.getP());
assertEquals(0, hllPlusValueAggregator.getSp());
- hllPlusValueAggregator = (DistinctCountHLLPlusValueAggregator)
-
ValueAggregatorFactory.getValueAggregator(AggregationFunctionType.DISTINCTCOUNTHLLPLUS,
-
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTHLLPLUS,
- Map.of(Constants.HLLPLUS_SP_KEY, 20)));
+ hllPlusValueAggregator = (DistinctCountHLLPlusValueAggregator)
ValueAggregatorFactory.getValueAggregator(
+ AggregationFunctionType.DISTINCTCOUNTHLLPLUS,
+
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTHLLPLUS,
+ Map.of(Constants.HLLPLUS_SP_KEY, 20)));
assertEquals(14, hllPlusValueAggregator.getP());
assertEquals(20, hllPlusValueAggregator.getSp());
// DISTINCTCOUNTULL
- DistinctCountULLValueAggregator ullValueAggregator =
(DistinctCountULLValueAggregator)
-
ValueAggregatorFactory.getValueAggregator(AggregationFunctionType.DISTINCTCOUNTULL,
-
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTULL,
- Map.of(Constants.HLLPLUS_ULL_P_KEY, "10")));
+ DistinctCountULLValueAggregator ullValueAggregator =
+ (DistinctCountULLValueAggregator)
ValueAggregatorFactory.getValueAggregator(
+ AggregationFunctionType.DISTINCTCOUNTULL,
+
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTULL,
+ Map.of(Constants.HLLPLUS_ULL_P_KEY, "10")));
assertEquals(10, ullValueAggregator.getP());
- ullValueAggregator = (DistinctCountULLValueAggregator)
-
ValueAggregatorFactory.getValueAggregator(AggregationFunctionType.DISTINCTCOUNTULL,
+ ullValueAggregator = (DistinctCountULLValueAggregator)
ValueAggregatorFactory.getValueAggregator(
+ AggregationFunctionType.DISTINCTCOUNTULL,
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTULL,
Map.of("garbage", "value")));
// Verify default value used
assertEquals(12, ullValueAggregator.getP());
// DISTINCTCOUNTCPCSKETCH
- DistinctCountCPCSketchValueAggregator cpcSketchValueAggregator =
(DistinctCountCPCSketchValueAggregator)
-
ValueAggregatorFactory.getValueAggregator(AggregationFunctionType.DISTINCTCOUNTCPCSKETCH,
-
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTCPCSKETCH,
- Map.of(Constants.CPCSKETCH_LGK_KEY, 20)));
+ DistinctCountCPCSketchValueAggregator cpcSketchValueAggregator =
+ (DistinctCountCPCSketchValueAggregator)
ValueAggregatorFactory.getValueAggregator(
+ AggregationFunctionType.DISTINCTCOUNTCPCSKETCH,
+
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTCPCSKETCH,
+ Map.of(Constants.CPCSKETCH_LGK_KEY, 20)));
assertEquals(20, cpcSketchValueAggregator.getLgK());
- cpcSketchValueAggregator = (DistinctCountCPCSketchValueAggregator)
-
ValueAggregatorFactory.getValueAggregator(AggregationFunctionType.DISTINCTCOUNTCPCSKETCH,
+ cpcSketchValueAggregator = (DistinctCountCPCSketchValueAggregator)
ValueAggregatorFactory.getValueAggregator(
+ AggregationFunctionType.DISTINCTCOUNTCPCSKETCH,
StarTreeBuilderUtils.expressionContextFromFunctionParameters(AggregationFunctionType.DISTINCTCOUNTCPCSKETCH,
Map.of()));
// Verify default value used
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfigTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfigTest.java
index cba984e5b1..b40cba7c06 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfigTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfigTest.java
@@ -22,10 +22,9 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
-import java.util.Arrays;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.pinot.segment.spi.AggregationFunctionType;
import org.apache.pinot.segment.spi.ColumnMetadata;
@@ -50,12 +49,17 @@ public class StarTreeV2BuilderConfigTest {
@Test
public void testDefaultConfig() {
- Schema schema = new Schema.SchemaBuilder().addSingleValueDimension("d1",
DataType.INT)
- .addSingleValueDimension("d2",
DataType.LONG).addSingleValueDimension("d3", DataType.FLOAT)
- .addSingleValueDimension("d4",
DataType.DOUBLE).addMultiValueDimension("d5", DataType.INT)
- .addMetric("m1", DataType.DOUBLE).addMetric("m2", DataType.BYTES)
+ Schema schema = new Schema.SchemaBuilder().setSchemaName("testTable")
+ .addSingleValueDimension("d1", DataType.INT)
+ .addSingleValueDimension("d2", DataType.LONG)
+ .addSingleValueDimension("d3", DataType.FLOAT)
+ .addSingleValueDimension("d4", DataType.DOUBLE)
+ .addMultiValueDimension("d5", DataType.INT)
+ .addMetric("m1", DataType.DOUBLE)
+ .addMetric("m2", DataType.BYTES)
.addTime(new TimeGranularitySpec(DataType.LONG, TimeUnit.MILLISECONDS,
"t"), null)
- .addDateTime("dt", DataType.LONG, "1:MILLISECONDS:EPOCH",
"1:HOURS").build();
+ .addDateTime("dt", DataType.LONG, "1:MILLISECONDS:EPOCH", "1:HOURS")
+ .build();
SegmentMetadataImpl segmentMetadata = mock(SegmentMetadataImpl.class);
when(segmentMetadata.getSchema()).thenReturn(schema);
// Included
@@ -88,31 +92,41 @@ public class StarTreeV2BuilderConfigTest {
StarTreeV2BuilderConfig defaultConfig =
StarTreeV2BuilderConfig.generateDefaultConfig(segmentMetadata);
// Sorted by cardinality in descending order, followed by the time column
- assertEquals(defaultConfig.getDimensionsSplitOrder(), Arrays.asList("d2",
"d1", "dt", "t"));
+ assertEquals(defaultConfig.getDimensionsSplitOrder(), List.of("d2", "d1",
"dt", "t"));
// No column should be skipped for star-node creation
assertTrue(defaultConfig.getSkipStarNodeCreationForDimensions().isEmpty());
// Should have COUNT(*) and SUM(m1) as the function column pairs
- assertEquals(defaultConfig.getFunctionColumnPairs(), new HashSet<>(
- Arrays.asList(AggregationFunctionColumnPair.COUNT_STAR,
- new AggregationFunctionColumnPair(AggregationFunctionType.SUM,
"m1"))));
+ assertEquals(defaultConfig.getFunctionColumnPairs(),
Set.of(AggregationFunctionColumnPair.COUNT_STAR,
+ new AggregationFunctionColumnPair(AggregationFunctionType.SUM, "m1")));
assertEquals(defaultConfig.getMaxLeafRecords(),
StarTreeV2BuilderConfig.DEFAULT_MAX_LEAF_RECORDS);
}
@Test
public void testDefaultConfigFromJsonNodeSegmentMetadata() {
- Schema schema = new Schema.SchemaBuilder().addSingleValueDimension("d1",
DataType.INT)
- .addSingleValueDimension("d2",
DataType.LONG).addSingleValueDimension("d3", DataType.FLOAT)
- .addSingleValueDimension("d4",
DataType.DOUBLE).addMultiValueDimension("d5", DataType.INT)
- .addMetric("m1", DataType.DOUBLE).addMetric("m2", DataType.BYTES)
+ Schema schema = new Schema.SchemaBuilder().setSchemaName("testTable")
+ .addSingleValueDimension("d1", DataType.INT)
+ .addSingleValueDimension("d2", DataType.LONG)
+ .addSingleValueDimension("d3", DataType.FLOAT)
+ .addSingleValueDimension("d4", DataType.DOUBLE)
+ .addMultiValueDimension("d5", DataType.INT)
+ .addMetric("m1", DataType.DOUBLE)
+ .addMetric("m2", DataType.BYTES)
.addTime(new TimeGranularitySpec(DataType.LONG, TimeUnit.MILLISECONDS,
"t"), null)
- .addDateTime("dt", DataType.LONG, "1:MILLISECONDS:EPOCH",
"1:HOURS").build();
+ .addDateTime("dt", DataType.LONG, "1:MILLISECONDS:EPOCH", "1:HOURS")
+ .build();
// Create a list of string with column name, hasDictionary and cardinality.
- List<List<String>> columnList =
- Arrays.asList(Arrays.asList("d1", "true", "200"), Arrays.asList("d2",
"true", "400"),
- Arrays.asList("d3", "true", "20000"), Arrays.asList("d4", "false",
"100"),
- Arrays.asList("d5", "true", "100"), Arrays.asList("m1", "false",
"-1"), Arrays.asList("m2", "true", "100"),
- Arrays.asList("t", "true", "20000"), Arrays.asList("dt", "true",
"30000"));
+ List<List<String>> columnList = List.of(
+ List.of("d1", "true", "200"),
+ List.of("d2", "true", "400"),
+ List.of("d3", "true", "20000"),
+ List.of("d4", "false", "100"),
+ List.of("d5", "true", "100"),
+ List.of("m1", "false", "-1"),
+ List.of("m2", "true", "100"),
+ List.of("t", "true", "20000"),
+ List.of("dt", "true", "30000")
+ );
// Convert the list of string to JsonNode with appropriate key names.
JsonNode segmentMetadataAsJsonNode =
convertStringListToJsonNode(columnList);
@@ -121,13 +135,12 @@ public class StarTreeV2BuilderConfigTest {
StarTreeV2BuilderConfig defaultConfig =
StarTreeV2BuilderConfig.generateDefaultConfig(schema,
segmentMetadataAsJsonNode);
// Sorted by cardinality in descending order, followed by the time column
- assertEquals(defaultConfig.getDimensionsSplitOrder(), Arrays.asList("d2",
"d1", "dt", "t"));
+ assertEquals(defaultConfig.getDimensionsSplitOrder(), List.of("d2", "d1",
"dt", "t"));
// No column should be skipped for star-node creation
assertTrue(defaultConfig.getSkipStarNodeCreationForDimensions().isEmpty());
// Should have COUNT(*) and SUM(m1) as the function column pairs
- assertEquals(defaultConfig.getFunctionColumnPairs(), new HashSet<>(
- Arrays.asList(AggregationFunctionColumnPair.COUNT_STAR,
- new AggregationFunctionColumnPair(AggregationFunctionType.SUM,
"m1"))));
+ assertEquals(defaultConfig.getFunctionColumnPairs(),
Set.of(AggregationFunctionColumnPair.COUNT_STAR,
+ new AggregationFunctionColumnPair(AggregationFunctionType.SUM, "m1")));
assertEquals(defaultConfig.getMaxLeafRecords(),
StarTreeV2BuilderConfig.DEFAULT_MAX_LEAF_RECORDS);
}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/startree/AggregationFunctionColumnPair.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/startree/AggregationFunctionColumnPair.java
index 31c9c744c5..3344d35259 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/startree/AggregationFunctionColumnPair.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/startree/AggregationFunctionColumnPair.java
@@ -28,6 +28,7 @@ public class AggregationFunctionColumnPair implements
Comparable<AggregationFunc
public static final String STAR = "*";
public static final AggregationFunctionColumnPair COUNT_STAR =
new AggregationFunctionColumnPair(AggregationFunctionType.COUNT, STAR);
+ public static final String COUNT_STAR_NAME = COUNT_STAR.toColumnName();
private final AggregationFunctionType _functionType;
private final String _column;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
